From 0a6a90415a5b77e482a121812f0a0bbf34ebbdbd Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Tue, 16 Jul 2024 06:58:35 +0300 Subject: [PATCH] Refactor initial memory paging --- src/arch/aarch64/device/serial.rs | 8 +- src/arch/aarch64/init/device_tree/mod.rs | 125 ------- src/arch/aarch64/rmm.rs | 380 +------------------- src/arch/aarch64/start.rs | 52 ++- src/arch/x86/paging/mod.rs | 1 + src/arch/x86/rmm.rs | 400 +-------------------- src/arch/x86/start.rs | 26 +- src/arch/x86_64/paging/mod.rs | 1 + src/arch/x86_64/rmm.rs | 359 +----------------- src/arch/x86_64/start.rs | 22 +- src/dtb/mod.rs | 98 ++++- src/main.rs | 3 + src/memory/mod.rs | 17 + src/startup/memory.rs | 440 +++++++++++++++++++++++ src/startup/mod.rs | 1 + syscall | 2 +- 16 files changed, 651 insertions(+), 1284 deletions(-) create mode 100644 src/startup/memory.rs create mode 100644 src/startup/mod.rs diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs index e5a543a5cb..0543fa54a6 100644 --- a/src/arch/aarch64/device/serial.rs +++ b/src/arch/aarch64/device/serial.rs @@ -1,10 +1,10 @@ use alloc::boxed::Box; use spin::Mutex; -use crate::{device::uart_pl011::SerialPort, init::device_tree, interrupt::irq::trigger}; +use crate::{device::uart_pl011::SerialPort, interrupt::irq::trigger}; use super::irqchip::{register_irq, InterruptHandler, IRQ_CHIP}; -use crate::dtb::DTB_BINARY; +use crate::dtb::{diag_uart_range, DTB_BINARY}; use alloc::vec::Vec; use byteorder::{ByteOrder, BE}; use fdt::Fdt; @@ -25,13 +25,13 @@ impl InterruptHandler for Com1Irq { } } -pub unsafe fn init_early(dtb_base: usize, dtb_size: usize) { +pub unsafe fn init_early(dtb: &Fdt) { if COM1.lock().is_some() { // Hardcoded UART return; } - if let Some((phys, _size, skip_init, cts)) = device_tree::diag_uart_range(dtb_base, dtb_size) { + if let Some((phys, _size, skip_init, cts)) = diag_uart_range(dtb) { let virt = crate::PHYS_OFFSET + phys; { let mut serial_port = SerialPort::new(virt, skip_init, cts); diff --git a/src/arch/aarch64/init/device_tree/mod.rs b/src/arch/aarch64/init/device_tree/mod.rs index 86c64744f2..45afe736cb 100644 --- a/src/arch/aarch64/init/device_tree/mod.rs +++ b/src/arch/aarch64/init/device_tree/mod.rs @@ -1,24 +1,7 @@ use byteorder::{ByteOrder, BE}; -use core::slice; use fdt::{node::NodeProperty, Fdt}; use log::debug; -#[derive(Copy, Clone, Debug, Default)] -#[repr(C)] -pub struct MemoryArea { - pub base_addr: u64, - pub length: u64, - pub _type: u32, - pub acpi: u32, -} - -pub static mut MEMORY_MAP: [MemoryArea; 512] = [MemoryArea { - base_addr: 0, - length: 0, - _type: 0, - acpi: 0, -}; 512]; - pub fn travel_interrupt_ctrl(fdt: &Fdt) { let root_intr_parent = fdt .root() @@ -53,111 +36,3 @@ pub fn travel_interrupt_ctrl(fdt: &Fdt) { } } } - -#[allow(unused)] -fn memory_ranges(dt: &Fdt, ranges: &mut [(usize, usize); 10]) -> usize { - let mut index = 0; - for chunk in dt.memory().regions() { - if index >= ranges.len() || chunk.size.is_none() { - break; - } - ranges[index] = (chunk.starting_address as usize, chunk.size.unwrap()); - index += 1; - } - index -} - -fn dev_memory_ranges(dt: &Fdt, ranges: &mut [(usize, usize); 10]) -> usize { - // work around for qemu-arm64 - // dev mem: 128MB - 1GB, see https://github.com/qemu/qemu/blob/master/hw/arm/virt.c for details - let root_node = dt.root(); - let is_qemu_virt = root_node.model().contains("linux,dummy-virt"); - - if is_qemu_virt { - ranges[0] = (0x08000000, 0x08000000); - ranges[1] = (0x10000000, 0x30000000); - return 2; - } - - let soc_node = dt.find_node("/soc").unwrap(); - let reg = soc_node.ranges().unwrap(); - - let mut index = 0; - for chunk in reg { - if index >= ranges.len() { - break; - } - debug!( - "dev mem 0x{:08x} 0x{:08x} 0x{:08x} 0x{:08x}", - chunk.child_bus_address_hi, - chunk.child_bus_address, - chunk.parent_bus_address, - chunk.size - ); - - ranges[index] = (chunk.parent_bus_address, chunk.size); - index += 1; - } - index -} - -pub fn diag_uart_range(dtb_base: usize, dtb_size: usize) -> Option<(usize, usize, bool, bool)> { - let data = unsafe { slice::from_raw_parts(dtb_base as *const u8, dtb_size) }; - let dt = Fdt::new(data).unwrap(); - - let stdout_path = dt.chosen().stdout().unwrap(); - let uart_node = stdout_path.node(); - let skip_init = uart_node.property("skip-init").is_some(); - let cts_event_walkaround = uart_node.property("cts-event-walkaround").is_some(); - - let mut reg = uart_node.reg().unwrap(); - let memory = reg.nth(0).unwrap(); - - Some(( - memory.starting_address as usize, - memory.size.unwrap(), - skip_init, - cts_event_walkaround, - )) -} - -#[allow(unused)] -pub fn fill_env_data(dtb_base: usize, dtb_size: usize, env_base: usize) -> usize { - let data = unsafe { slice::from_raw_parts(dtb_base as *const u8, dtb_size) }; - let dt = Fdt::new(data).unwrap(); - - if let Some(bootargs) = dt.chosen().bootargs() { - let bootargs_len = bootargs.len(); - - let env_base_slice = - unsafe { slice::from_raw_parts_mut(env_base as *mut u8, bootargs_len) }; - env_base_slice[..bootargs_len].clone_from_slice(bootargs.as_bytes()); - - bootargs_len - } else { - 0 - } -} - -pub fn fill_memory_map(dtb_base: usize, dtb_size: usize) { - let data = unsafe { slice::from_raw_parts(dtb_base as *const u8, dtb_size) }; - let dt = Fdt::new(data).unwrap(); - - let mut ranges: [(usize, usize); 10] = [(0, 0); 10]; - - //in uefi boot mode, ignore memory node, just read the device memory range - //let nranges = memory_ranges(&dt, &mut ranges); - let nranges = dev_memory_ranges(&dt, &mut ranges); - - for index in 0..nranges { - let (base, size) = ranges[index]; - unsafe { - MEMORY_MAP[index] = MemoryArea { - base_addr: base as u64, - length: size as u64, - _type: 2, - acpi: 0, - }; - } - } -} diff --git a/src/arch/aarch64/rmm.rs b/src/arch/aarch64/rmm.rs index 8a1e3016a9..453382c44c 100644 --- a/src/arch/aarch64/rmm.rs +++ b/src/arch/aarch64/rmm.rs @@ -1,34 +1,6 @@ -use core::{cell::SyncUnsafeCell, cmp, mem, slice}; -use rmm::{ - Arch, BumpAllocator, MemoryArea, PageFlags, PageMapper, PhysicalAddress, TableKind, - VirtualAddress, KILOBYTE, MEGABYTE, -}; +use rmm::{Arch, PageFlags, VirtualAddress}; -use crate::{init::device_tree::MEMORY_MAP, paging::entry::EntryFlags}; - -use super::CurrentRmmArch as RmmA; - -// Keep synced with OsMemoryKind in bootloader -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[repr(u64)] -#[allow(dead_code)] -pub enum BootloaderMemoryKind { - Null = 0, - Free = 1, - Reclaim = 2, - Reserved = 3, -} - -// Keep synced with OsMemoryEntry in bootloader -#[derive(Clone, Copy, Debug)] -#[repr(C, packed)] -pub struct BootloaderMemoryEntry { - pub base: u64, - pub size: u64, - pub kind: BootloaderMemoryKind, -} - -unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { +pub unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { use crate::kernel_executable_offsets::*; let virt_addr = virt.data(); @@ -43,351 +15,3 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { PageFlags::new().write(true) } } - -unsafe fn inner( - areas: &'static [MemoryArea], - kernel_base: usize, - kernel_size_aligned: usize, - stack_base: usize, - stack_size_aligned: usize, - env_base: usize, - env_size_aligned: usize, - acpi_base: usize, - acpi_size_aligned: usize, - initfs_base: usize, - initfs_size_aligned: usize, -) { - type A = RmmA; - - // First, calculate how much memory we have - let mut size = 0; - for area in areas.iter() { - if area.size > 0 { - log::debug!("{:X?}", area); - size += area.size; - } - } - - log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE); - - // Create a basic allocator for the first pages - let mut bump_allocator = BumpAllocator::::new(areas, 0); - - { - let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) - .expect("failed to create Mapper"); - - // Map all physical areas at PHYS_OFFSET - for area in areas.iter() { - for i in 0..area.size / A::PAGE_SIZE { - let phys = area.base.add(i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - // Map kernel at KERNEL_OFFSET and identity map too - for i in 0..kernel_size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE); - let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - - let virt = A::phys_to_virt(phys); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - - let mut identity_map = |base, size_aligned| { - // Map with identity mapping - for i in 0..size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - }; - - identity_map(stack_base, stack_size_aligned); - identity_map(env_base, env_size_aligned); - identity_map(acpi_base, acpi_size_aligned); - identity_map(initfs_base, initfs_size_aligned); - - //TODO: this is another hack to map our UART - /* - match crate::device::serial::COM1.lock().as_ref().map(|x| x.base()) { - Some(serial_base) => { - let flush = mapper.map_phys( - VirtualAddress::new(serial_base), - PhysicalAddress::new(serial_base - crate::PHYS_OFFSET), - PageFlags::new().write(true) - ).expect("failed to map frame"); - flush.ignore(); // Not the active table - }, - None => (), - } - */ - - //map dev mem - for mem in MEMORY_MAP { - if mem._type == 2 { - let size_aligned = - ((mem.length as usize + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let base = mem.base_addr as usize; - for i in 0..size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - // use the same mair_el1 value with bootloader, - // mair_el1 == 0x00000000000044FF - // set mem_attr == device memory - let flags = page_flags::(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - } - - // Ensure graphical debug region remains paged - #[cfg(feature = "graphical_debug")] - { - use crate::devices::graphical_debug::FRAMEBUFFER; - - let (phys, virt, size) = *FRAMEBUFFER.lock(); - - let pages = (size + A::PAGE_SIZE - 1) / A::PAGE_SIZE; - for i in 0..pages { - let phys = PhysicalAddress::new(phys + i * A::PAGE_SIZE); - let virt = VirtualAddress::new(virt + i * A::PAGE_SIZE); - let flags = PageFlags::new().write(true); - //TODO: Write combining flag - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - log::debug!("Table: {:X}", mapper.table().phys().data()); - for i in 0..A::PAGE_ENTRIES { - if let Some(entry) = mapper.table().entry(i) { - if entry.present() { - log::debug!("{}: {:X}", i, entry.data()); - } - } - } - - // Use the new table - mapper.make_current(); - } - - // Create the physical memory map - let offset = bump_allocator.offset(); - log::info!( - "Permanently used: {} KB", - (offset + (KILOBYTE - 1)) / KILOBYTE - ); - - crate::memory::init_mm(bump_allocator); -} - -static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new( - [MemoryArea { - base: PhysicalAddress::new(0), - size: 0, - }; 512], -); -static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); - -pub unsafe fn init( - kernel_base: usize, - kernel_size: usize, - stack_base: usize, - stack_size: usize, - env_base: usize, - env_size: usize, - acpi_base: usize, - acpi_size: usize, - areas_base: usize, - areas_size: usize, - initfs_base: usize, - initfs_size: usize, -) { - type A = RmmA; - - let real_base = 0; - let real_size = 0x100000; - let real_end = real_base + real_size; - - let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let kernel_end = kernel_base + kernel_size_aligned; - - let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let stack_end = stack_base + stack_size_aligned; - - let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let env_end = env_base + env_size_aligned; - - let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let acpi_end = acpi_base + acpi_size_aligned; - - let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let initfs_end = initfs_base + initfs_size_aligned; - - let bootloader_areas = slice::from_raw_parts( - areas_base as *const BootloaderMemoryEntry, - areas_size / mem::size_of::(), - ); - - let areas = &mut *AREAS.get(); - - // Copy memory map from bootloader location, and page align it - let mut area_i = 0; - for bootloader_area in bootloader_areas.iter() { - if { bootloader_area.kind } != BootloaderMemoryKind::Free { - // Not a free area - continue; - } - - let mut base = bootloader_area.base as usize; - let mut size = bootloader_area.size as usize; - - log::debug!("{:X}:{:X}", base, size); - - // Page align base - let base_offset = (A::PAGE_SIZE - (base & A::PAGE_OFFSET_MASK)) & A::PAGE_OFFSET_MASK; - if base_offset > size { - // Area is too small to page align base - continue; - } - base += base_offset; - size -= base_offset; - - // Page align size - size &= !A::PAGE_OFFSET_MASK; - log::debug!(" => {:X}:{:X}", base, size); - - let mut new_base = base; - - // Ensure real-mode areas are not used - if base < real_end && base + size > real_base { - log::warn!( - "{:X}:{:X} overlaps with real mode {:X}:{:X}", - base, - size, - real_base, - real_size - ); - new_base = cmp::max(new_base, real_end); - } - - // Ensure kernel areas are not used - if base < kernel_end && base + size > kernel_base { - log::warn!( - "{:X}:{:X} overlaps with kernel {:X}:{:X}", - base, - size, - kernel_base, - kernel_size - ); - new_base = cmp::max(new_base, kernel_end); - } - - // Ensure stack areas are not used - if base < stack_end && base + size > stack_base { - log::warn!( - "{:X}:{:X} overlaps with stack {:X}:{:X}", - base, - size, - stack_base, - stack_size - ); - new_base = cmp::max(new_base, stack_end); - } - - // Ensure env areas are not used - if base < env_end && base + size > env_base { - log::warn!( - "{:X}:{:X} overlaps with env {:X}:{:X}", - base, - size, - env_base, - env_size - ); - new_base = cmp::max(new_base, env_end); - } - - // Ensure acpi areas are not used - if base < acpi_end && base + size > acpi_base { - log::warn!( - "{:X}:{:X} overlaps with acpi {:X}:{:X}", - base, - size, - acpi_base, - acpi_size - ); - new_base = cmp::max(new_base, acpi_end); - } - if base < initfs_end && base + size > initfs_base { - log::warn!( - "{:X}:{:X} overlaps with initfs {:X}:{:X}", - base, - size, - initfs_base, - initfs_size - ); - new_base = cmp::max(new_base, initfs_end); - } - - if new_base != base { - let end = base + size; - let new_size = end.checked_sub(new_base).unwrap_or(0); - log::info!( - "{:X}:{:X} moved to {:X}:{:X}", - base, - size, - new_base, - new_size - ); - base = new_base; - size = new_size; - } - - if size == 0 { - // Area is zero sized - continue; - } - - areas[area_i].base = PhysicalAddress::new(base); - areas[area_i].size = size; - area_i += 1; - } - AREA_COUNT.get().write(area_i as u16); - - inner( - areas, - kernel_base, - kernel_size_aligned, - stack_base, - stack_size_aligned, - env_base, - env_size_aligned, - acpi_base, - acpi_size_aligned, - initfs_base, - initfs_size_aligned, - ); -} diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index b482525838..2ca3da7d36 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -8,8 +8,17 @@ use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; #[cfg(feature = "graphical_debug")] use crate::devices::graphical_debug; -use crate::{allocator, device, dtb, init::device_tree, paging}; +use fdt::Fdt; use log::info; +use rmm::PhysicalAddress; + +use crate::{ + allocator, device, dtb, + dtb::register_dev_memory_ranges, + memory::{Frame, PAGE_SIZE}, + paging, + startup::memory::{register_bootloader_areas, register_memory_region, BootloaderMemoryKind}, +}; /// Test of zero values in BSS. static mut BSS_TEST_ZERO: usize = 0; @@ -22,7 +31,7 @@ pub static CPU_COUNT: AtomicU32 = AtomicU32::new(0); pub static AP_READY: AtomicBool = AtomicBool::new(false); static BSP_READY: AtomicBool = AtomicBool::new(false); -#[repr(C, packed)] +#[repr(C, packed(8))] pub struct KernelArgs { kernel_base: usize, kernel_size: usize, @@ -56,9 +65,16 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst); KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst); - if args.dtb_base != 0 { + let dtb = if args.dtb_base != 0 { + let data = unsafe { slice::from_raw_parts(args.dtb_base as *const u8, args.dtb_size) }; + Fdt::new(data).ok() + } else { + None + }; + + if let Some(dt) = &dtb { // Try to find serial port prior to logging - device::serial::init_early(crate::PHYS_OFFSET + args.dtb_base, args.dtb_size); + device::serial::init_early(dt); } // Convert env to slice @@ -124,30 +140,46 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { tmp = out(reg) _, ); - if args.dtb_base != 0 { - //Try to read device memory map - device_tree::fill_memory_map(crate::PHYS_OFFSET + args.dtb_base, args.dtb_size); + if let Some(dt) = &dtb { + //in uefi boot mode, ignore memory node, just read the device memory range + //register_memory_ranges(dt); + + register_dev_memory_ranges(dt); } + register_bootloader_areas(args.areas_base, args.areas_size); + /* NOT USED WITH UEFI let env_size = device_tree::fill_env_data(crate::PHYS_OFFSET + dtb_base, dtb_size, env_base); */ // Initialize RMM - crate::arch::rmm::init( + register_memory_region( args.kernel_base, args.kernel_size, + BootloaderMemoryKind::Kernel, + ); + register_memory_region( args.stack_base, args.stack_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.env_base, args.env_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.dtb_base, args.dtb_size, - args.areas_base, - args.areas_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.bootstrap_base, args.bootstrap_size, + BootloaderMemoryKind::IdentityMap, ); + crate::startup::memory::init(None, None); // Initialize paging paging::init(); diff --git a/src/arch/x86/paging/mod.rs b/src/arch/x86/paging/mod.rs index 61c6413ed4..d2f9662251 100644 --- a/src/arch/x86/paging/mod.rs +++ b/src/arch/x86/paging/mod.rs @@ -14,6 +14,7 @@ pub mod entry { const NO_CACHE = 1 << 4; const HUGE_PAGE = 1 << 7; const GLOBAL = 1 << 8; + const DEV_MEM = 0; } } } diff --git a/src/arch/x86/rmm.rs b/src/arch/x86/rmm.rs index 6d7414bcd1..453382c44c 100644 --- a/src/arch/x86/rmm.rs +++ b/src/arch/x86/rmm.rs @@ -1,32 +1,6 @@ -use core::{cell::SyncUnsafeCell, cmp, mem, slice}; -use rmm::{ - Arch, BumpAllocator, FrameAllocator, MemoryArea, PageEntry, PageFlags, PageMapper, - PhysicalAddress, TableKind, VirtualAddress, KILOBYTE, MEGABYTE, -}; +use rmm::{Arch, PageFlags, VirtualAddress}; -use super::CurrentRmmArch as RmmA; - -// Keep synced with OsMemoryKind in bootloader -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[repr(u64)] -#[allow(dead_code)] -pub enum BootloaderMemoryKind { - Null = 0, - Free = 1, - Reclaim = 2, - Reserved = 3, -} - -// Keep synced with OsMemoryEntry in bootloader -#[derive(Clone, Copy, Debug)] -#[repr(C, packed)] -pub struct BootloaderMemoryEntry { - pub base: u64, - pub size: u64, - pub kind: BootloaderMemoryKind, -} - -unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { +pub unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { use crate::kernel_executable_offsets::*; let virt_addr = virt.data(); @@ -41,373 +15,3 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { PageFlags::new().write(true) } } - -unsafe fn inner( - areas: &'static [MemoryArea], - kernel_base: usize, - kernel_size_aligned: usize, - stack_base: usize, - stack_size_aligned: usize, - env_base: usize, - env_size_aligned: usize, - acpi_base: usize, - acpi_size_aligned: usize, - initfs_base: usize, - initfs_size_aligned: usize, -) { - type A = RmmA; - - // First, calculate how much memory we have - let mut size = 0; - for area in areas.iter() { - if area.size > 0 { - log::debug!("{:X?}", area); - size += area.size; - } - } - - log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE); - - // Create a basic allocator for the first pages - let mut bump_allocator = BumpAllocator::::new(areas, 0); - - { - let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) - .expect("failed to create Mapper"); - - // Pre-allocate all kernel PD entries so that when the page table is copied, - // these entries are synced between processes - for i in 512..1024 { - let phys = mapper - .allocator_mut() - .allocate_one() - .expect("failed to map page table"); - let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE; - mapper - .table() - .set_entry(i, PageEntry::new(phys.data(), flags)); - } - - // Map all physical areas at PHYS_OFFSET - for area in areas.iter() { - for i in 0..area.size / A::PAGE_SIZE { - let phys = area.base.add(i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - // Map kernel at KERNEL_OFFSET and identity map too - for i in 0..kernel_size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE); - let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - - let virt = A::phys_to_virt(phys); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - - let mut identity_map = |base, size_aligned| { - // Map with identity mapping - for i in 0..size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - }; - - identity_map(stack_base, stack_size_aligned); - identity_map(env_base, env_size_aligned); - identity_map(acpi_base, acpi_size_aligned); - identity_map(initfs_base, initfs_size_aligned); - - // Ensure graphical debug region remains paged - #[cfg(feature = "graphical_debug")] - { - use super::paging::entry::EntryFlags; - use crate::devices::graphical_debug::FRAMEBUFFER; - - let (phys, virt, size) = *FRAMEBUFFER.lock(); - - let pages = (size + A::PAGE_SIZE - 1) / A::PAGE_SIZE; - for i in 0..pages { - let phys = PhysicalAddress::new(phys + i * A::PAGE_SIZE); - let virt = VirtualAddress::new(virt + i * A::PAGE_SIZE); - let flags = PageFlags::new().write(true).write_combining(true); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - log::debug!("Table: {:X}", mapper.table().phys().data()); - for i in 0..A::PAGE_ENTRIES { - if let Some(entry) = mapper.table().entry(i) { - if entry.present() { - log::debug!("{}: {:X}", i, entry.data()); - } - } - } - - // Use the new table - mapper.make_current(); - } - - // Create the physical memory map - let offset = bump_allocator.offset(); - log::info!( - "Permanently used: {} KB", - (offset + (KILOBYTE - 1)) / KILOBYTE - ); - - crate::memory::init_mm(bump_allocator); -} - -static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new( - [MemoryArea { - base: PhysicalAddress::new(0), - size: 0, - }; 512], -); -static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); - -pub unsafe fn init( - kernel_base: usize, - kernel_size: usize, - stack_base: usize, - stack_size: usize, - env_base: usize, - env_size: usize, - acpi_base: usize, - acpi_size: usize, - areas_base: usize, - areas_size: usize, - initfs_base: usize, - initfs_size: usize, -) { - type A = RmmA; - - let real_base = 0; - let real_size = 0x100000; - let real_end = real_base + real_size; - - let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let kernel_end = kernel_base + kernel_size_aligned; - - let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let stack_end = stack_base + stack_size_aligned; - - let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let env_end = env_base + env_size_aligned; - - let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let acpi_end = acpi_base + acpi_size_aligned; - - let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; - let initfs_end = initfs_base + initfs_size_aligned; - - let areas = &mut *AREAS.get(); - - let bootloader_areas = slice::from_raw_parts( - areas_base as *const BootloaderMemoryEntry, - areas_size / mem::size_of::(), - ); - - // Copy memory map from bootloader location, and page align it - let mut area_i = 0; - for bootloader_area in bootloader_areas.iter() { - if { bootloader_area.kind } != BootloaderMemoryKind::Free { - // Not a free area - continue; - } - - let mut base = bootloader_area.base as usize; - let mut size = bootloader_area.size as usize; - - log::debug!("{:X}:{:X}", base, size); - - // Page align base - let base_offset = (A::PAGE_SIZE - (base & A::PAGE_OFFSET_MASK)) & A::PAGE_OFFSET_MASK; - if base_offset > size { - // Area is too small to page align base - continue; - } - base += base_offset; - size -= base_offset; - - // Page align size - size &= !A::PAGE_OFFSET_MASK; - log::debug!(" => {:X}:{:X}", base, size); - - let mut new_base = base; - - // Ensure real-mode areas are not used - if base < real_end && base + size > real_base { - log::warn!( - "{:X}:{:X} overlaps with real mode {:X}:{:X}", - base, - size, - real_base, - real_size - ); - new_base = cmp::max(new_base, real_end); - } - - // Ensure kernel areas are not used - if base < kernel_end && base + size > kernel_base { - log::warn!( - "{:X}:{:X} overlaps with kernel {:X}:{:X}", - base, - size, - kernel_base, - kernel_size - ); - new_base = cmp::max(new_base, kernel_end); - } - - // Ensure stack areas are not used - if base < stack_end && base + size > stack_base { - log::warn!( - "{:X}:{:X} overlaps with stack {:X}:{:X}", - base, - size, - stack_base, - stack_size - ); - new_base = cmp::max(new_base, stack_end); - } - - // Ensure env areas are not used - if base < env_end && base + size > env_base { - log::warn!( - "{:X}:{:X} overlaps with env {:X}:{:X}", - base, - size, - env_base, - env_size - ); - new_base = cmp::max(new_base, env_end); - } - - // Ensure acpi areas are not used - if base < acpi_end && base + size > acpi_base { - log::warn!( - "{:X}:{:X} overlaps with acpi {:X}:{:X}", - base, - size, - acpi_base, - acpi_size - ); - new_base = cmp::max(new_base, acpi_end); - } - - // Ensure initfs areas are not used - if base < initfs_end && base + size > initfs_base { - log::warn!( - "{:X}:{:X} overlaps with initfs {:X}:{:X}", - base, - size, - initfs_base, - initfs_size - ); - new_base = cmp::max(new_base, initfs_end); - } - - if new_base != base { - let end = base + size; - let new_size = end.checked_sub(new_base).unwrap_or(0); - log::info!( - "{:X}:{:X} moved to {:X}:{:X}", - base, - size, - new_base, - new_size - ); - base = new_base; - size = new_size; - } - - // Ensure area fits within physmap (1GiB) - //TODO: let memory areas >1GiB be used - let physmap_size = 0x40000000; - if base >= physmap_size { - log::warn!("{:X}:{:X} outside of physmap, ignoring", base, size); - size = 0; // Skip area - } else if base + size > physmap_size { - let new_size = physmap_size.checked_sub(base).unwrap_or(0); - log::warn!( - "{:X}:{:X} outside of physmap, moved to {:X}:{:X}", - base, - size, - base, - new_size - ); - size = new_size; - } - - // Combine areas that overlap - for other_i in 0..area_i { - let other = &areas[other_i]; - let other_base = other.base.data(); - let other_end = other_base + other.size; - if base < other_end && base + size > other_base { - let new_base = cmp::min(base, other_base); - let new_size = cmp::max(base + size, other_end) - .checked_sub(new_base) - .unwrap_or(0); - log::warn!( - "{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", - base, - size, - other_base, - other.size, - new_base, - new_size - ); - areas[other_i].base = PhysicalAddress::new(new_base); - areas[other_i].size = new_size; - size = 0; // Skip area - } - } - - if size == 0 { - // Area is zero sized, skip - continue; - } - - areas[area_i].base = PhysicalAddress::new(base); - areas[area_i].size = size; - area_i += 1; - } - AREA_COUNT.get().write(area_i as u16); - - inner( - areas, - kernel_base, - kernel_size_aligned, - stack_base, - stack_size_aligned, - env_base, - env_size_aligned, - acpi_base, - acpi_size_aligned, - initfs_base, - initfs_size_aligned, - ); -} diff --git a/src/arch/x86/start.rs b/src/arch/x86/start.rs index 90dcdbaa52..b89f372873 100644 --- a/src/arch/x86/start.rs +++ b/src/arch/x86/start.rs @@ -4,6 +4,7 @@ /// defined in other files inside of the `arch` module use core::slice; use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use log::info; #[cfg(feature = "acpi")] use crate::acpi; @@ -14,8 +15,8 @@ use crate::{ cpu_set::LogicalCpuId, device, gdt, idt, interrupt, paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind}, + startup::memory::{register_bootloader_areas, register_memory_region, BootloaderMemoryKind}, }; -use log::info; /// Test of zero values in BSS. static mut BSS_TEST_ZERO: usize = 0; @@ -31,7 +32,7 @@ pub static CPU_COUNT: AtomicU32 = AtomicU32::new(0); pub static AP_READY: AtomicBool = AtomicBool::new(false); static BSP_READY: AtomicBool = AtomicBool::new(false); -#[repr(C, packed)] +#[repr(C, packed(8))] pub struct KernelArgs { kernel_base: u64, kernel_size: u64, @@ -137,20 +138,35 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { idt::init(); // Initialize RMM - crate::arch::rmm::init( + register_bootloader_areas(args.areas_base as usize, args.areas_size as usize); + + register_memory_region( args.kernel_base as usize, args.kernel_size as usize, + BootloaderMemoryKind::Kernel, + ); + register_memory_region( args.stack_base as usize, args.stack_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.env_base as usize, args.env_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.acpi_rsdp_base as usize, args.acpi_rsdp_size as usize, - args.areas_base as usize, - args.areas_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.bootstrap_base as usize, args.bootstrap_size as usize, + BootloaderMemoryKind::IdentityMap, ); + crate::startup::memory::init(Some(0x100000), Some(0x40000000)); + // Initialize paging paging::init(); diff --git a/src/arch/x86_64/paging/mod.rs b/src/arch/x86_64/paging/mod.rs index a45a34cbf8..441da03c35 100644 --- a/src/arch/x86_64/paging/mod.rs +++ b/src/arch/x86_64/paging/mod.rs @@ -16,6 +16,7 @@ pub mod entry { const NO_CACHE = 1 << 4; const HUGE_PAGE = 1 << 7; const GLOBAL = 1 << 8; + const DEV_MEM = 0; } } } diff --git a/src/arch/x86_64/rmm.rs b/src/arch/x86_64/rmm.rs index 3fcfa32966..9d78a2f9f5 100644 --- a/src/arch/x86_64/rmm.rs +++ b/src/arch/x86_64/rmm.rs @@ -1,31 +1,6 @@ -use core::{cell::SyncUnsafeCell, cmp, mem, slice}; -use rmm::{ - Arch, BumpAllocator, MemoryArea, PageFlags, PageMapper, PhysicalAddress, TableKind, - VirtualAddress, KILOBYTE, MEGABYTE, -}; +use rmm::{Arch, PageFlags, VirtualAddress}; -use super::CurrentRmmArch as RmmA; - -// Keep synced with OsMemoryKind in bootloader -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[repr(u64)] -pub enum BootloaderMemoryKind { - Null = 0, - Free = 1, - Reclaim = 2, - Reserved = 3, -} - -// Keep synced with OsMemoryEntry in bootloader -#[derive(Clone, Copy, Debug)] -#[repr(C, packed)] -pub struct BootloaderMemoryEntry { - pub base: u64, - pub size: u64, - pub kind: BootloaderMemoryKind, -} - -unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { +pub unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { use crate::kernel_executable_offsets::*; let virt_addr = virt.data(); @@ -41,333 +16,3 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { }) .global(cfg!(not(feature = "pti"))) } - -unsafe fn inner( - areas: &'static [MemoryArea], - kernel_base: usize, - kernel_size_aligned: usize, - stack_base: usize, - stack_size_aligned: usize, - env_base: usize, - env_size_aligned: usize, - acpi_base: usize, - acpi_size_aligned: usize, - initfs_base: usize, - initfs_size_aligned: usize, -) { - type A = RmmA; - - // First, calculate how much memory we have - let mut size = 0; - for area in areas.iter() { - if area.size > 0 { - log::debug!("{:X?}", area); - size += area.size; - } - } - - log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE); - - // Create a basic allocator for the first pages - let mut bump_allocator = BumpAllocator::::new(areas, 0); - - { - let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) - .expect("failed to create Mapper"); - - // Map all physical areas at PHYS_OFFSET - for area in areas.iter() { - for i in 0..area.size / A::PAGE_SIZE { - let phys = area.base.add(i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - // Map kernel at KERNEL_OFFSET and map linearly too - for i in 0..kernel_size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE); - let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - - let virt = A::phys_to_virt(phys); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - - let mut identity_map = |base, size_aligned| { - // Map with identity mapping - for i in 0..size_aligned / A::PAGE_SIZE { - let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - }; - - identity_map(stack_base, stack_size_aligned); - identity_map(env_base, env_size_aligned); - identity_map(acpi_base, acpi_size_aligned); - identity_map(initfs_base, initfs_size_aligned); - - // Ensure graphical debug region remains paged - #[cfg(feature = "graphical_debug")] - { - use super::paging::entry::EntryFlags; - use crate::devices::graphical_debug::FRAMEBUFFER; - - let (phys, virt, size) = *FRAMEBUFFER.lock(); - - let pages = (size + A::PAGE_SIZE - 1) / A::PAGE_SIZE; - for i in 0..pages { - let phys = PhysicalAddress::new(phys + i * A::PAGE_SIZE); - let virt = VirtualAddress::new(virt + i * A::PAGE_SIZE); - let flags = PageFlags::new().write(true).write_combining(true); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - log::debug!("Table: {:X}", mapper.table().phys().data()); - for i in 0..A::PAGE_ENTRIES { - if let Some(entry) = mapper.table().entry(i) { - if entry.present() { - log::debug!("{}: {:X}", i, entry.data()); - } - } - } - - // Use the new table - mapper.make_current(); - } - - // Create the physical memory map - let offset = bump_allocator.offset(); - log::info!( - "Permanently used: {} KB", - (offset + (KILOBYTE - 1)) / KILOBYTE - ); - - crate::memory::init_mm(bump_allocator); -} - -static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new( - [MemoryArea { - base: PhysicalAddress::new(0), - size: 0, - }; 512], -); -static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); - -pub fn areas() -> &'static [MemoryArea] { - // SAFETY: Both AREAS and AREA_COUNT are initialized once and then never changed. - // - // TODO: Memory hotplug? - unsafe { &(&*AREAS.get())[..AREA_COUNT.get().read().into()] } -} - -pub unsafe fn init( - kernel_base: usize, - kernel_size: usize, - stack_base: usize, - stack_size: usize, - env_base: usize, - env_size: usize, - acpi_base: usize, - acpi_size: usize, - areas_base: usize, - areas_size: usize, - initfs_base: usize, - initfs_size: usize, -) { - type A = RmmA; - - let real_base = 0; - let real_size = 0x100000; - let real_end = real_base + real_size; - - let kernel_size_aligned = kernel_size.next_multiple_of(A::PAGE_SIZE); - let kernel_end = kernel_base + kernel_size_aligned; - - let stack_size_aligned = stack_size.next_multiple_of(A::PAGE_SIZE); - let stack_end = stack_base + stack_size_aligned; - - let env_size_aligned = env_size.next_multiple_of(A::PAGE_SIZE); - let env_end = env_base + env_size_aligned; - - let acpi_size_aligned = acpi_size.next_multiple_of(A::PAGE_SIZE); - let acpi_end = acpi_base + acpi_size_aligned; - - let initfs_size_aligned = initfs_size.next_multiple_of(A::PAGE_SIZE); - let initfs_end = initfs_base + initfs_size_aligned; - - let bootloader_areas = slice::from_raw_parts( - areas_base as *const BootloaderMemoryEntry, - areas_size / mem::size_of::(), - ); - - // Copy memory map from bootloader location, and page align it - let mut area_i = 0; - let areas_raw = &mut *AREAS.get(); - - for bootloader_area in bootloader_areas.iter() { - if { bootloader_area.kind } != BootloaderMemoryKind::Free { - // Not a free area - continue; - } - - let mut base = bootloader_area.base as usize; - let mut size = bootloader_area.size as usize; - - log::info!("{:X}:{:X}", base, size); - - // Page align base - let base_offset = (A::PAGE_SIZE - (base & A::PAGE_OFFSET_MASK)) & A::PAGE_OFFSET_MASK; - if base_offset > size { - // Area is too small to page align base - continue; - } - base += base_offset; - size -= base_offset; - - // Page align size - size &= !A::PAGE_OFFSET_MASK; - log::debug!(" => {:X}:{:X}", base, size); - - let mut new_base = base; - - // Ensure real-mode areas are not used - if base < real_end && base + size > real_base { - log::warn!( - "{:X}:{:X} overlaps with real mode {:X}:{:X}", - base, - size, - real_base, - real_size - ); - new_base = cmp::max(new_base, real_end); - } - - // Ensure kernel areas are not used - if base < kernel_end && base + size > kernel_base { - log::warn!( - "{:X}:{:X} overlaps with kernel {:X}:{:X}", - base, - size, - kernel_base, - kernel_size - ); - new_base = cmp::max(new_base, kernel_end); - } - - // Ensure stack areas are not used - if base < stack_end && base + size > stack_base { - log::warn!( - "{:X}:{:X} overlaps with stack {:X}:{:X}", - base, - size, - stack_base, - stack_size - ); - new_base = cmp::max(new_base, stack_end); - } - - // Ensure env areas are not used - if base < env_end && base + size > env_base { - log::warn!( - "{:X}:{:X} overlaps with env {:X}:{:X}", - base, - size, - env_base, - env_size - ); - new_base = cmp::max(new_base, env_end); - } - - // Ensure acpi areas are not used - if base < acpi_end && base + size > acpi_base { - log::warn!( - "{:X}:{:X} overlaps with acpi {:X}:{:X}", - base, - size, - acpi_base, - acpi_size - ); - new_base = cmp::max(new_base, acpi_end); - } - - // Ensure initfs areas are not used - if base < initfs_end && base + size > initfs_base { - log::warn!( - "{:X}:{:X} overlaps with initfs {:X}:{:X}", - base, - size, - initfs_base, - initfs_size - ); - new_base = cmp::max(new_base, initfs_end); - } - - if new_base != base { - let end = base + size; - let new_size = end.checked_sub(new_base).unwrap_or(0); - log::info!( - "{:X}:{:X} moved to {:X}:{:X}", - base, - size, - new_base, - new_size - ); - base = new_base; - size = new_size; - } - - if size == 0 { - // Area is zero sized, skip - continue; - } - - areas_raw[area_i].base = PhysicalAddress::new(base); - areas_raw[area_i].size = size; - area_i += 1; - } - for i in area_i..areas_raw.len() { - areas_raw[i] = MemoryArea { - base: PhysicalAddress::new(!0), - size: 0, - }; - } - - areas_raw.sort_unstable_by_key(|area| area.base); - - AREA_COUNT.get().write(area_i as u16); - - inner( - areas(), - kernel_base, - kernel_size_aligned, - stack_base, - stack_size_aligned, - env_base, - env_size_aligned, - acpi_base, - acpi_size_aligned, - initfs_base, - initfs_size_aligned, - ); -} diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index 2e3b46018d..2e2563519e 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -17,6 +17,7 @@ use crate::{ cpu_set::LogicalCpuId, device, gdt, idt, interrupt, misc, paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind}, + startup::memory::{register_bootloader_areas, register_memory_region, BootloaderMemoryKind}, }; /// Test of zero values in BSS. @@ -33,7 +34,7 @@ pub static CPU_COUNT: AtomicU32 = AtomicU32::new(0); pub static AP_READY: AtomicBool = AtomicBool::new(false); static BSP_READY: AtomicBool = AtomicBool::new(false); -#[repr(C, packed)] +#[repr(C, packed(8))] pub struct KernelArgs { kernel_base: u64, kernel_size: u64, @@ -139,20 +140,33 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { idt::init(); // Initialize RMM - crate::arch::rmm::init( + register_bootloader_areas(args.areas_base as usize, args.areas_size as usize); + register_memory_region( args.kernel_base as usize, args.kernel_size as usize, + BootloaderMemoryKind::Kernel, + ); + register_memory_region( args.stack_base as usize, args.stack_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.env_base as usize, args.env_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.acpi_rsdp_base as usize, args.acpi_rsdp_size as usize, - args.areas_base as usize, - args.areas_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.bootstrap_base as usize, args.bootstrap_size as usize, + BootloaderMemoryKind::IdentityMap, ); + crate::startup::memory::init(Some(0x100000), None); // Initialize PAT paging::init(); diff --git a/src/dtb/mod.rs b/src/dtb/mod.rs index a13b92c7cc..f46ad9ec3c 100644 --- a/src/dtb/mod.rs +++ b/src/dtb/mod.rs @@ -1,6 +1,7 @@ -use core::slice; - +use crate::startup::memory::{register_memory_region, BootloaderMemoryKind}; use alloc::vec::Vec; +use core::slice; +use fdt::Fdt; use spin::once::Once; pub static DTB_BINARY: Once> = Once::new(); @@ -21,3 +22,96 @@ pub unsafe fn init(dtb: Option<(usize, usize)>) { println!("DTB_BINARY INIT TWICE!"); } } + +#[allow(unused)] +pub fn register_memory_ranges(dt: &Fdt) { + for chunk in dt.memory().regions() { + if let Some(size) = chunk.size { + register_memory_region( + chunk.starting_address as usize, + size, + BootloaderMemoryKind::Free, + ); + } + } +} + +pub fn register_dev_memory_ranges(dt: &Fdt) { + if cfg!(target_arch = "aarch64") { + // work around for qemu-arm64 + // dev mem: 128MB - 1GB, see https://github.com/qemu/qemu/blob/master/hw/arm/virt.c for details + let root_node = dt.root(); + let is_qemu_virt = root_node.model().contains("linux,dummy-virt"); + + if is_qemu_virt { + register_memory_region(0x08000000, 0x08000000, BootloaderMemoryKind::Device); + register_memory_region(0x10000000, 0x30000000, BootloaderMemoryKind::Device); + return; + } + } + + let soc_node = dt.find_node("/soc").unwrap(); + let reg = soc_node.ranges().unwrap(); + + for chunk in reg { + log::debug!( + "dev mem 0x{:08x} 0x{:08x} 0x{:08x} 0x{:08x}", + chunk.child_bus_address_hi, + chunk.child_bus_address, + chunk.parent_bus_address, + chunk.size + ); + + register_memory_region( + chunk.parent_bus_address, + chunk.size, + BootloaderMemoryKind::Device, + ); + } + + // also add all soc-internal devices please because they might not be shown in ranges + for device in soc_node.children() { + if let Some(reg) = device.reg() { + for entry in reg { + if let Some(size) = entry.size { + let addr = entry.starting_address as usize; + log::debug!("soc device {} 0x{:08x} 0x{:08x}", device.name, addr, size); + + register_memory_region(addr, size, BootloaderMemoryKind::Device); + } + } + } + } +} + +pub fn diag_uart_range(dtb: &Fdt) -> Option<(usize, usize, bool, bool)> { + let stdout_path = dtb.chosen().stdout().unwrap(); + let uart_node = stdout_path.node(); + let skip_init = uart_node.property("skip-init").is_some(); + let cts_event_walkaround = uart_node.property("cts-event-walkaround").is_some(); + + let mut reg = uart_node.reg().unwrap(); + let memory = reg.nth(0).unwrap(); + + Some(( + memory.starting_address as usize, + memory.size.unwrap(), + skip_init, + cts_event_walkaround, + )) +} + +#[allow(unused)] +pub fn fill_env_data(dt: &Fdt, env_base: usize) -> usize { + if let Some(bootargs) = dt.chosen().bootargs() { + let bootargs_len = bootargs.len(); + + let env_base_slice = + unsafe { slice::from_raw_parts_mut(env_base as *mut u8, bootargs_len) }; + env_base_slice[..bootargs_len].clone_from_slice(bootargs.as_bytes()); + + bootargs_len + } else { + 0 + } +} diff --git a/src/main.rs b/src/main.rs index 57815f0455..9b70818df7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -139,6 +139,9 @@ pub mod profiling; /// Schemes, filesystem handlers mod scheme; +/// Early init +mod startup; + /// Synchronization primitives mod sync; diff --git a/src/memory/mod.rs b/src/memory/mod.rs index bb8e056b99..f60947b9c4 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -25,6 +25,23 @@ use crate::{ }; use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage, TableKind, VirtualAddress}; +/// Available physical memory areas +pub(crate) static AREAS: SyncUnsafeCell<[rmm::MemoryArea; 512]> = SyncUnsafeCell::new( + [rmm::MemoryArea { + base: PhysicalAddress::new(0), + size: 0, + }; 512], +); +pub(crate) static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); + +// TODO: Share code +pub(crate) fn areas() -> &'static [rmm::MemoryArea] { + // SAFETY: Both AREAS and AREA_COUNT are initialized once and then never changed. + // + // TODO: Memory hotplug? + unsafe { &(&*AREAS.get())[..AREA_COUNT.get().read().into()] } +} + /// Get the number of frames available pub fn free_frames() -> usize { total_frames() - used_frames() diff --git a/src/startup/memory.rs b/src/startup/memory.rs new file mode 100644 index 0000000000..2d9107b473 --- /dev/null +++ b/src/startup/memory.rs @@ -0,0 +1,440 @@ +use crate::{ + arch::{consts::KERNEL_OFFSET, paging::entry::EntryFlags, rmm::page_flags, CurrentRmmArch}, + memory::PAGE_SIZE, +}; +use core::{ + cmp::{max, min}, + mem, slice, + slice::Iter, +}; +use rmm::{ + Arch, BumpAllocator, MemoryArea, PageFlags, PageMapper, PhysicalAddress, TableKind, + VirtualAddress, KILOBYTE, MEGABYTE, +}; +use crate::startup::memory::BootloaderMemoryKind::Null; + +// Keep synced with OsMemoryKind in bootloader +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u64)] +#[allow(dead_code)] +pub enum BootloaderMemoryKind { + Null = 0, + Free = 1, + Reclaim = 2, + Reserved = 3, + + // These are local to kernel + Kernel = 0x100, + Device = 0x101, + IdentityMap = 0x102, +} + +// Keep synced with OsMemoryEntry in bootloader +#[derive(Clone, Copy, Debug)] +#[repr(C, packed(8))] +struct BootloaderMemoryEntry { + pub base: u64, + pub size: u64, + pub kind: BootloaderMemoryKind, +} + +#[derive(Clone, Copy, Debug)] +struct MemoryEntry { + pub start: usize, + pub end: usize, + pub kind: BootloaderMemoryKind, +} + +impl MemoryEntry { + fn intersect(&self, other: &Self) -> Option { + let start = max(self.start, other.start); + let end = min(self.end, other.end); + if start < end { + Some(Self { + start, + end, + kind: self.kind, + }) + } else { + None + } + } + + fn combine(&self, other: &Self) -> Option { + if self.start <= other.end && self.end >= other.start { + Some(Self { + start: min(self.start, other.start), + end: max(self.end, other.end), + kind: self.kind, + }) + } else { + None + } + } +} + +struct MemoryMap { + entries: [MemoryEntry; 512], + size: usize, +} + +impl MemoryMap { + fn register(&mut self, base: usize, size: usize, kind: BootloaderMemoryKind) { + if self.size >= self.entries.len() { + panic!("Early memory map overflow!"); + } + let start = if kind == BootloaderMemoryKind::Free { + align_up(base) + } else { + align_down(base) + }; + let end = base.saturating_add(size); + let end = if kind == BootloaderMemoryKind::Free { + align_down(end) + } else { + align_up(end) + }; + if start < end { + self.entries[self.size] = MemoryEntry { start, end, kind }; + self.size += 1; + } + } + + fn iter(&self) -> Iter { + return self.entries[0..self.size].iter(); + } + + pub fn free(&self) -> impl Iterator { + self.iter().filter(|x| x.kind == BootloaderMemoryKind::Free) + } + + pub fn non_free(&self) -> impl Iterator { + self.iter().filter(|x| x.kind != BootloaderMemoryKind::Free) + } + + pub fn kernel(&self) -> Option<&MemoryEntry> { + self.iter().find(|x| x.kind == BootloaderMemoryKind::Kernel) + } + + pub fn devices(&self) -> impl Iterator { + self.iter() + .filter(|x| x.kind == BootloaderMemoryKind::Device) + } + + pub fn identity_mapped(&self) -> impl Iterator { + self.iter() + .filter(|x| x.kind == BootloaderMemoryKind::IdentityMap) + } +} + +static mut MEMORY_MAP: MemoryMap = MemoryMap { + entries: [MemoryEntry { + start: 0, + end: 0, + kind: BootloaderMemoryKind::Null, + }; 512], + size: 0, +}; + +fn align_up(x: usize) -> usize { + (x.saturating_add(PAGE_SIZE - 1) / PAGE_SIZE) * PAGE_SIZE +} +fn align_down(x: usize) -> usize { + x / PAGE_SIZE * PAGE_SIZE +} + +pub fn register_memory_region(base: usize, size: usize, kind: BootloaderMemoryKind) { + if kind != Null && size != 0 { + log::debug!("Registering {:?} memory {:X} size {:X}", kind, base, size); + unsafe { MEMORY_MAP.register(base, size, kind) } + } +} + +pub fn register_bootloader_areas(areas_base: usize, areas_size: usize) { + let bootloader_areas = unsafe { + slice::from_raw_parts( + areas_base as *const BootloaderMemoryEntry, + areas_size / mem::size_of::(), + ) + }; + for bootloader_area in bootloader_areas.iter() { + register_memory_region( + bootloader_area.base as usize, + bootloader_area.size as usize, + bootloader_area.kind, + ) + } +} + +unsafe fn add_memory(areas: &mut [MemoryArea], area_i: &mut usize, mut area: MemoryEntry) { + for reservation in MEMORY_MAP.non_free() { + if area.end > reservation.start && area.end <= reservation.end { + log::info!( + "Memory {:X}:{:X} overlaps with reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + area.end = reservation.start; + } + if area.start >= area.end { + return; + } + + if area.start >= reservation.start && area.start < reservation.end { + log::info!( + "Memory {:X}:{:X} overlaps with reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + area.start = reservation.end; + } + if area.start >= area.end { + return; + } + + if area.start <= reservation.start && area.end > reservation.start { + log::info!( + "Memory {:X}:{:X} contains reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + debug_assert!(area.start < reservation.start && reservation.end < area.end, + "Should've contained reservation entirely: memory block {:X}:{:X} reservation {:X}:{:X}", + area.start, area.end, + reservation.start, reservation.end + ); + // recurse on first part of split memory block + + add_memory( + areas, + area_i, + MemoryEntry { + end: reservation.start, + ..area + }, + ); + + // and continue with the second part + area.start = reservation.end; + } + debug_assert!( + area.intersect(reservation).is_none(), + "Intersects with reservation! memory block {:X}:{:X} reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + debug_assert!( + area.start < area.end, + "Empty memory block {:X}:{:X}", + area.start, + area.end + ); + } + + // Combine overlapping memory areas + let mut other_i = 0; + while other_i < *area_i { + let other = &areas[other_i]; + let other = MemoryEntry { + start: other.base.data(), + end: other.base.data().saturating_add(other.size), + kind: BootloaderMemoryKind::Free, + }; + if let Some(union) = area.combine(&other) { + log::debug!( + "{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", + area.start, + area.end, + other.start, + other.end, + union.start, + union.end + ); + area = union; + *area_i -= 1; // delete the original memory chunk + areas[other_i] = areas[*area_i]; + } else { + other_i += 1; + } + } + + areas[*area_i].base = PhysicalAddress::new(area.start); + areas[*area_i].size = area.end - area.start; + *area_i += 1; +} + +unsafe fn map_memory(areas: &[MemoryArea], mut bump_allocator: &mut BumpAllocator) { + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); + + #[cfg(target_arch = "i686")] + { + // Pre-allocate all kernel PD entries so that when the page table is copied, + // these entries are synced between processes + for i in 512..1024 { + let phys = mapper + .allocator_mut() + .allocate_one() + .expect("failed to map page table"); + let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE; + mapper + .table() + .set_entry(i, PageEntry::new(phys.data(), flags)); + } + } + + // Map all physical areas at PHYS_OFFSET + for area in areas.iter() { + for i in 0..area.size / PAGE_SIZE { + let phys = area.base.add(i * PAGE_SIZE); + let virt = A::phys_to_virt(phys); + let flags = page_flags::(virt); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + let kernel_area = MEMORY_MAP.kernel().unwrap(); + let kernel_base = kernel_area.start; + let kernel_size = kernel_area.end - kernel_area.start; + // Map kernel at KERNEL_OFFSET and identity map too + for i in 0..kernel_size / A::PAGE_SIZE { + let phys = PhysicalAddress::new(kernel_base + i * PAGE_SIZE); + let virt = VirtualAddress::new(KERNEL_OFFSET + i * PAGE_SIZE); + let flags = page_flags::(virt); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + + let virt = A::phys_to_virt(phys); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + + for area in MEMORY_MAP.identity_mapped() { + let base = area.start; + let size = area.end - area.start; + for i in 0..size / PAGE_SIZE { + let phys = PhysicalAddress::new(base + i * PAGE_SIZE); + let virt = A::phys_to_virt(phys); + let flags = page_flags::(virt); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + //map dev mem + for area in MEMORY_MAP.devices() { + let base = area.start; + let size = area.end - area.start; + for i in 0..size / PAGE_SIZE { + let phys = PhysicalAddress::new(base + i * PAGE_SIZE); + let virt = A::phys_to_virt(phys); + // use the same mair_el1 value with bootloader, + // mair_el1 == 0x00000000000044FF + // set mem_attr == device memory + let flags = page_flags::(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + // Ensure graphical debug region remains paged + #[cfg(feature = "graphical_debug")] + { + use crate::devices::graphical_debug::FRAMEBUFFER; + + let (phys, virt, size) = *FRAMEBUFFER.lock(); + + let pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + for i in 0..pages { + let phys = PhysicalAddress::new(phys + i * PAGE_SIZE); + let virt = VirtualAddress::new(virt + i * PAGE_SIZE); + let flags = PageFlags::new().write(true).write_combining(true); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + log::debug!("Table: {:X}", mapper.table().phys().data()); + for i in 0..A::PAGE_ENTRIES { + if let Some(entry) = mapper.table().entry(i) { + if entry.present() { + log::debug!("{}: {:X}", i, entry.data()); + } + } + } + + // Use the new table + mapper.make_current(); +} + +pub unsafe fn init(low_limit: Option, high_limit: Option) { + let physmem_limit = MemoryEntry { + start: align_up(low_limit.unwrap_or(0)), + end: align_down(high_limit.unwrap_or(usize::MAX)), + kind: BootloaderMemoryKind::Free, + }; + + let areas = &mut *crate::memory::AREAS.get(); + let mut area_i = 0; + + // Copy initial memory map, and page align it + for area in MEMORY_MAP.free() { + log::debug!("{:X}:{:X}", area.start, area.end); + + if let Some(area) = area.intersect(&physmem_limit) { + add_memory(areas, &mut area_i, area); + } + } + + areas[..area_i].sort_unstable_by_key(|area| area.base); + crate::memory::AREA_COUNT.get().write(area_i as u16); + + // free memory map in now ready + let areas = crate::memory::areas(); + + // First, calculate how much memory we have + let mut size = 0; + for area in areas.iter() { + if area.size > 0 { + log::debug!("{:X?}", area); + size += area.size; + } + } + + log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE); + + // Create a basic allocator for the first pages + let mut bump_allocator = BumpAllocator::::new(areas, 0); + + map_memory(areas, &mut bump_allocator); + + // Create the physical memory map + let offset = bump_allocator.offset(); + log::info!( + "Permanently used: {} KB", + (offset + (KILOBYTE - 1)) / KILOBYTE + ); + + crate::memory::init_mm(bump_allocator); +} diff --git a/src/startup/mod.rs b/src/startup/mod.rs new file mode 100644 index 0000000000..eb291915fd --- /dev/null +++ b/src/startup/mod.rs @@ -0,0 +1 @@ +pub mod memory; diff --git a/syscall b/syscall index c910533bcb..f3fc45a1b7 160000 --- a/syscall +++ b/syscall @@ -1 +1 @@ -Subproject commit c910533bcbb2c7f9e8ef3628b012bfc70c2c33c9 +Subproject commit f3fc45a1b74debc4a188276640fdfb8be47ccc25