From cea93f7647f6f914e6afdb834b89b56c5209ce60 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:00:28 +0200 Subject: [PATCH] cargo fix --edition & rustfmt Or to be precise: RUST_TARGET_PATH=$(pwd)/targets cargo fix --edition \ --target targets/x86_64-unknown-kernel.json \ --target targets/i686-unknown-kernel.json \ --target targets/aarch64-unknown-kernel.json \ --target targets/riscv64-unknown-kernel.json \ -Zbuild-std=core,alloc --allow-dirty --bin kernel cargo fmt --- Cargo.toml | 2 +- src/acpi/hpet.rs | 52 +- src/acpi/mod.rs | 154 +++--- src/allocator/linked_list.rs | 49 +- src/allocator/mod.rs | 52 +- .../device/cpu/registers/control_regs.rs | 156 ++++-- src/arch/aarch64/device/generic_timer.rs | 34 +- src/arch/aarch64/device/irqchip/gic.rs | 134 ++--- src/arch/aarch64/device/irqchip/gicv3.rs | 58 ++- .../aarch64/device/irqchip/irq_bcm2835.rs | 66 +-- .../aarch64/device/irqchip/irq_bcm2836.rs | 28 +- src/arch/aarch64/device/mod.rs | 56 +- src/arch/aarch64/device/rtc.rs | 2 +- src/arch/aarch64/device/serial.rs | 104 ++-- src/arch/aarch64/interrupt/exception.rs | 184 ++++--- src/arch/aarch64/interrupt/handler.rs | 30 +- src/arch/aarch64/interrupt/irq.rs | 18 +- src/arch/aarch64/interrupt/mod.rs | 32 +- src/arch/aarch64/interrupt/syscall.rs | 4 +- src/arch/aarch64/interrupt/trace.rs | 30 +- src/arch/aarch64/macros.rs | 4 +- src/arch/aarch64/misc.rs | 10 +- src/arch/aarch64/mod.rs | 12 +- src/arch/aarch64/paging/mod.rs | 16 +- src/arch/aarch64/start.rs | 326 ++++++------ src/arch/aarch64/stop.rs | 34 +- src/arch/riscv64/device/irqchip/hlic.rs | 56 +- src/arch/riscv64/device/mod.rs | 24 +- src/arch/riscv64/device/serial.rs | 107 ++-- src/arch/riscv64/interrupt/exception.rs | 211 ++++---- src/arch/riscv64/interrupt/handler.rs | 24 +- src/arch/riscv64/interrupt/mod.rs | 22 +- src/arch/riscv64/interrupt/trace.rs | 34 +- src/arch/riscv64/macros.rs | 4 +- src/arch/riscv64/misc.rs | 26 +- src/arch/riscv64/mod.rs | 10 +- src/arch/riscv64/start.rs | 305 +++++------ src/arch/x86/gdt.rs | 141 ++--- src/arch/x86/interrupt/handler.rs | 46 +- src/arch/x86/interrupt/irq.rs | 96 ++-- src/arch/x86/macros.rs | 8 +- src/arch/x86/mod.rs | 10 +- src/arch/x86_64/alternative.rs | 388 +++++++------- src/arch/x86_64/gdt.rs | 177 ++++--- src/arch/x86_64/interrupt/handler.rs | 26 +- src/arch/x86_64/interrupt/irq.rs | 90 ++-- src/arch/x86_64/interrupt/syscall.rs | 281 +++++----- src/arch/x86_64/macros.rs | 12 +- src/arch/x86_64/misc.rs | 34 +- src/arch/x86_64/mod.rs | 16 +- src/arch/x86_shared/device/hpet.rs | 164 +++--- src/arch/x86_shared/device/ioapic.rs | 314 ++++++------ src/arch/x86_shared/device/local_apic.rs | 224 ++++---- src/arch/x86_shared/device/mod.rs | 71 +-- src/arch/x86_shared/device/pic.rs | 56 +- src/arch/x86_shared/device/pit.rs | 26 +- src/arch/x86_shared/device/tsc.rs | 64 +-- src/arch/x86_shared/idt.rs | 245 ++++----- src/arch/x86_shared/interrupt/mod.rs | 16 +- src/arch/x86_shared/interrupt/trace.rs | 36 +- src/arch/x86_shared/paging/mod.rs | 56 +- src/arch/x86_shared/start.rs | 416 +++++++-------- src/arch/x86_shared/stop.rs | 86 ++-- src/common/mod.rs | 6 +- src/context/arch/aarch64.rs | 188 +++---- src/context/arch/riscv64.rs | 56 +- src/context/arch/x86.rs | 90 ++-- src/context/arch/x86_64.rs | 136 ++--- src/context/context.rs | 19 +- src/context/memory.rs | 169 +++--- src/context/switch.rs | 181 ++++--- src/devices/graphical_debug/display.rs | 58 ++- src/devices/uart_16550.rs | 4 +- src/dtb/irqchip.rs | 40 +- src/externs.rs | 210 ++++---- src/main.rs | 2 +- src/memory/mod.rs | 44 +- src/panic.rs | 118 ++--- src/percpu.rs | 64 +-- src/scheme/irq.rs | 58 ++- src/scheme/mod.rs | 36 +- src/scheme/proc.rs | 17 +- src/scheme/sys/context.rs | 26 +- src/scheme/sys/mod.rs | 2 +- src/scheme/user.rs | 19 +- src/startup/memory.rs | 482 +++++++++--------- src/sync/wait_queue.rs | 21 +- src/syscall/debug.rs | 2 +- src/syscall/process.rs | 10 +- src/syscall/usercopy.rs | 24 +- 90 files changed, 4080 insertions(+), 3571 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76ae9bac1f..88a2811890 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "kernel" version = "0.5.12" build = "build.rs" -edition = "2021" +edition = "2024" [build-dependencies] cc = "1.0" diff --git a/src/acpi/hpet.rs b/src/acpi/hpet.rs index c3c05c45a0..59f9fc86e5 100644 --- a/src/acpi/hpet.rs +++ b/src/acpi/hpet.rs @@ -63,16 +63,17 @@ impl Hpet { #[cfg(target_arch = "x86")] impl Hpet { pub unsafe fn map(&self) { - use crate::{ - memory::{Frame, KernelMapper}, - paging::{entry::EntryFlags, Page, VirtualAddress}, - }; - use rmm::PageFlags; + unsafe { + use crate::{ + memory::{Frame, KernelMapper}, + paging::{entry::EntryFlags, Page, VirtualAddress}, + }; + use rmm::PageFlags; - let frame = Frame::containing(PhysicalAddress::new(self.base_address.address as usize)); - let page = Page::containing_address(VirtualAddress::new(crate::HPET_OFFSET)); + let frame = Frame::containing(PhysicalAddress::new(self.base_address.address as usize)); + let page = Page::containing_address(VirtualAddress::new(crate::HPET_OFFSET)); - KernelMapper::lock() + KernelMapper::lock() .get_mut() .expect( "KernelMapper locked re-entrant while mapping memory for GenericAddressStructure", @@ -86,36 +87,45 @@ impl Hpet { ) .expect("failed to map memory for GenericAddressStructure") .flush(); + } } pub unsafe fn read_u64(&self, offset: usize) -> u64 { - read_volatile((crate::HPET_OFFSET + offset) as *const u64) + unsafe { read_volatile((crate::HPET_OFFSET + offset) as *const u64) } } pub unsafe fn write_u64(&mut self, offset: usize, value: u64) { - write_volatile((crate::HPET_OFFSET + offset) as *mut u64, value); + unsafe { + write_volatile((crate::HPET_OFFSET + offset) as *mut u64, value); + } } } #[cfg(not(target_arch = "x86"))] impl Hpet { pub unsafe fn map(&self) { - map_device_memory( - PhysicalAddress::new(self.base_address.address as usize), - PAGE_SIZE, - ); + unsafe { + map_device_memory( + PhysicalAddress::new(self.base_address.address as usize), + PAGE_SIZE, + ); + } } pub unsafe fn read_u64(&self, offset: usize) -> u64 { - read_volatile( - (self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *const u64, - ) + unsafe { + read_volatile( + (self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *const u64, + ) + } } pub unsafe fn write_u64(&mut self, offset: usize, value: u64) { - write_volatile( - (self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *mut u64, - value, - ); + unsafe { + write_volatile( + (self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *mut u64, + value, + ); + } } } diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 8548e11190..ae43c5a39c 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -28,17 +28,19 @@ mod spcr; mod xsdt; unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::paging::PageMapper) { - let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data())); - let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data())); + unsafe { + let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data())); + let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data())); - for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE { - let (_, flush) = mapper - .map_linearly( - base.add(page_idx * crate::memory::PAGE_SIZE), - PageFlags::new(), - ) - .expect("failed to linearly map SDT"); - flush.flush(); + for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE { + let (_, flush) = mapper + .map_linearly( + base.add(page_idx * crate::memory::PAGE_SIZE), + PageFlags::new(), + ) + .expect("failed to linearly map SDT"); + flush.flush(); + } } } @@ -93,83 +95,85 @@ pub static RXSDT_ENUM: Once = Once::new(); /// Parse the ACPI tables to gather CPU, interrupt, and timer information pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) { - { - let mut sdt_ptrs = SDT_POINTERS.write(); - *sdt_ptrs = Some(HashMap::new()); - } - - // Search for RSDP - let rsdp_opt = RSDP::get_rsdp(&mut KernelMapper::lock(), already_supplied_rsdp); - - if let Some(rsdp) = rsdp_opt { - info!("SDT address: {:#x}", rsdp.sdt_address()); - let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock()); - - for &c in rxsdt.signature.iter() { - print!("{}", c as char); + unsafe { + { + let mut sdt_ptrs = SDT_POINTERS.write(); + *sdt_ptrs = Some(HashMap::new()); } - println!(":"); - let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) { - let mut initialized = false; + // Search for RSDP + let rsdp_opt = RSDP::get_rsdp(&mut KernelMapper::lock(), already_supplied_rsdp); - let rsdt = RXSDT_ENUM.call_once(|| { - initialized = true; + if let Some(rsdp) = rsdp_opt { + info!("SDT address: {:#x}", rsdp.sdt_address()); + let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock()); - RxsdtEnum::Rsdt(rsdt) - }); + for &c in rxsdt.signature.iter() { + print!("{}", c as char); + } + println!(":"); - if !initialized { - log::error!("RXSDT_ENUM already initialized"); + let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) { + let mut initialized = false; + + let rsdt = RXSDT_ENUM.call_once(|| { + initialized = true; + + RxsdtEnum::Rsdt(rsdt) + }); + + if !initialized { + log::error!("RXSDT_ENUM already initialized"); + } + + rsdt + } else if let Some(xsdt) = Xsdt::new(rxsdt) { + let mut initialized = false; + + let xsdt = RXSDT_ENUM.call_once(|| { + initialized = true; + + RxsdtEnum::Xsdt(xsdt) + }); + if !initialized { + log::error!("RXSDT_ENUM already initialized"); + } + + xsdt + } else { + println!("UNKNOWN RSDT OR XSDT SIGNATURE"); + return; + }; + + // TODO: Don't touch ACPI tables in kernel? + + for sdt in rxsdt.iter() { + get_sdt(sdt, &mut KernelMapper::lock()); } - rsdt - } else if let Some(xsdt) = Xsdt::new(rxsdt) { - let mut initialized = false; + for sdt_address in rxsdt.iter() { + let sdt = &*((sdt_address + crate::PHYS_OFFSET) as *const Sdt); - let xsdt = RXSDT_ENUM.call_once(|| { - initialized = true; - - RxsdtEnum::Xsdt(xsdt) - }); - if !initialized { - log::error!("RXSDT_ENUM already initialized"); + let signature = get_sdt_signature(sdt); + if let Some(ref mut ptrs) = *(SDT_POINTERS.write()) { + ptrs.insert(signature, sdt); + } } - xsdt + //TODO: support this on any arch + #[cfg(target_arch = "aarch64")] + spcr::Spcr::init(); + // TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface + // to initialize enumerated processors to userspace? + Madt::init(); + // TODO: Let userspace setup HPET, and then provide an interface to specify which timer to + // use? + Hpet::init(); + #[cfg(target_arch = "aarch64")] + gtdt::Gtdt::init(); } else { - println!("UNKNOWN RSDT OR XSDT SIGNATURE"); - return; - }; - - // TODO: Don't touch ACPI tables in kernel? - - for sdt in rxsdt.iter() { - get_sdt(sdt, &mut KernelMapper::lock()); + println!("NO RSDP FOUND"); } - - for sdt_address in rxsdt.iter() { - let sdt = &*((sdt_address + crate::PHYS_OFFSET) as *const Sdt); - - let signature = get_sdt_signature(sdt); - if let Some(ref mut ptrs) = *(SDT_POINTERS.write()) { - ptrs.insert(signature, sdt); - } - } - - //TODO: support this on any arch - #[cfg(target_arch = "aarch64")] - spcr::Spcr::init(); - // TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface - // to initialize enumerated processors to userspace? - Madt::init(); - // TODO: Let userspace setup HPET, and then provide an interface to specify which timer to - // use? - Hpet::init(); - #[cfg(target_arch = "aarch64")] - gtdt::Gtdt::init(); - } else { - println!("NO RSDP FOUND"); } } diff --git a/src/allocator/linked_list.rs b/src/allocator/linked_list.rs index b1875434f4..889b8458c1 100644 --- a/src/allocator/linked_list.rs +++ b/src/allocator/linked_list.rs @@ -12,38 +12,45 @@ pub struct Allocator; impl Allocator { pub unsafe fn init(offset: usize, size: usize) { - *HEAP.lock() = Some(Heap::new(offset, size)); + unsafe { + *HEAP.lock() = Some(Heap::new(offset, size)); + } } } unsafe impl GlobalAlloc for Allocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - while let Some(ref mut heap) = *HEAP.lock() { - match heap.allocate_first_fit(layout) { - Err(()) => { - let size = heap.size(); - super::map_heap( - &mut KernelMapper::lock(), - crate::KERNEL_HEAP_OFFSET + size, - crate::KERNEL_HEAP_SIZE, - ); - heap.extend(crate::KERNEL_HEAP_SIZE); - } - other => { - return other - .ok() - .map_or(ptr::null_mut(), |allocation| allocation.as_ptr()) + unsafe { + while let Some(ref mut heap) = *HEAP.lock() { + match heap.allocate_first_fit(layout) { + Err(()) => { + let size = heap.size(); + super::map_heap( + &mut KernelMapper::lock(), + crate::KERNEL_HEAP_OFFSET + size, + crate::KERNEL_HEAP_SIZE, + ); + heap.extend(crate::KERNEL_HEAP_SIZE); + } + other => { + return other + .ok() + .map_or(ptr::null_mut(), |allocation| allocation.as_ptr()) + } } } + panic!("__rust_allocate: heap not initialized"); } - panic!("__rust_allocate: heap not initialized"); } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - if let Some(ref mut heap) = *HEAP.lock() { - heap.deallocate(NonNull::new_unchecked(ptr), layout) - } else { - panic!("__rust_deallocate: heap not initialized"); + unsafe { + match *HEAP.lock() { + Some(ref mut heap) => heap.deallocate(NonNull::new_unchecked(ptr), layout), + _ => { + panic!("__rust_deallocate: heap not initialized"); + } + } } } } diff --git a/src/allocator/mod.rs b/src/allocator/mod.rs index 36a8409304..98b9c9b345 100644 --- a/src/allocator/mod.rs +++ b/src/allocator/mod.rs @@ -17,35 +17,39 @@ mod linked_list; mod slab; unsafe fn map_heap(mapper: &mut KernelMapper, offset: usize, size: usize) { - let mapper = mapper - .get_mut() - .expect("failed to obtain exclusive access to KernelMapper while extending heap"); - let mut flush_all = PageFlushAll::new(); + unsafe { + let mapper = mapper + .get_mut() + .expect("failed to obtain exclusive access to KernelMapper while extending heap"); + let mut flush_all = PageFlushAll::new(); - let heap_start_page = Page::containing_address(VirtualAddress::new(offset)); - let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1)); - for page in Page::range_inclusive(heap_start_page, heap_end_page) { - let result = mapper - .map( - page.start_address(), - PageFlags::new() - .write(true) - .global(cfg!(not(feature = "pti"))), - ) - .expect("failed to map kernel heap"); - flush_all.consume(result); + let heap_start_page = Page::containing_address(VirtualAddress::new(offset)); + let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1)); + for page in Page::range_inclusive(heap_start_page, heap_end_page) { + let result = mapper + .map( + page.start_address(), + PageFlags::new() + .write(true) + .global(cfg!(not(feature = "pti"))), + ) + .expect("failed to map kernel heap"); + flush_all.consume(result); + } + + flush_all.flush(); } - - flush_all.flush(); } pub unsafe fn init() { - let offset = crate::KERNEL_HEAP_OFFSET; - let size = crate::KERNEL_HEAP_SIZE; + unsafe { + let offset = crate::KERNEL_HEAP_OFFSET; + let size = crate::KERNEL_HEAP_SIZE; - // Map heap pages - map_heap(&mut KernelMapper::lock(), offset, size); + // Map heap pages + map_heap(&mut KernelMapper::lock(), offset, size); - // Initialize global heap - Allocator::init(offset, size); + // Initialize global heap + Allocator::init(offset, size); + } } diff --git a/src/arch/aarch64/device/cpu/registers/control_regs.rs b/src/arch/aarch64/device/cpu/registers/control_regs.rs index 69bf670145..8622a91c30 100644 --- a/src/arch/aarch64/device/cpu/registers/control_regs.rs +++ b/src/arch/aarch64/device/cpu/registers/control_regs.rs @@ -13,129 +13,177 @@ bitflags! { } pub unsafe fn ttbr0_el1() -> u64 { - let ret: u64; - asm!("mrs {}, ttbr0_el1", out(reg) ret); - ret + unsafe { + let ret: u64; + asm!("mrs {}, ttbr0_el1", out(reg) ret); + ret + } } pub unsafe fn ttbr0_el1_write(val: u64) { - asm!("msr ttbr0_el1, {}", in(reg) val); + unsafe { + asm!("msr ttbr0_el1, {}", in(reg) val); + } } pub unsafe fn ttbr1_el1() -> u64 { - let ret: u64; - asm!("mrs {}, ttbr1_el1", out(reg) ret); - ret + unsafe { + let ret: u64; + asm!("mrs {}, ttbr1_el1", out(reg) ret); + ret + } } pub unsafe fn ttbr1_el1_write(val: u64) { - asm!("msr ttbr1_el1, {}", in(reg) val); + unsafe { + asm!("msr ttbr1_el1, {}", in(reg) val); + } } pub unsafe fn mair_el1() -> MairEl1 { - let ret: u64; - asm!("mrs {}, mair_el1", out(reg) ret); - MairEl1::from_bits_truncate(ret) + unsafe { + let ret: u64; + asm!("mrs {}, mair_el1", out(reg) ret); + MairEl1::from_bits_truncate(ret) + } } pub unsafe fn mair_el1_write(val: MairEl1) { - asm!("msr mair_el1, {}", in(reg) val.bits()); + unsafe { + asm!("msr mair_el1, {}", in(reg) val.bits()); + } } pub unsafe fn tpidr_el0() -> u64 { - let ret: u64; - asm!("mrs {}, tpidr_el0", out(reg) ret); - ret + unsafe { + let ret: u64; + asm!("mrs {}, tpidr_el0", out(reg) ret); + ret + } } pub unsafe fn tpidr_el0_write(val: u64) { - asm!("msr tpidr_el0, {}", in(reg) val); + unsafe { + asm!("msr tpidr_el0, {}", in(reg) val); + } } pub unsafe fn tpidr_el1() -> u64 { - let ret: u64; - asm!("mrs {}, tpidr_el1", out(reg) ret); - ret + unsafe { + let ret: u64; + asm!("mrs {}, tpidr_el1", out(reg) ret); + ret + } } pub unsafe fn tpidr_el1_write(val: u64) { - asm!("msr tpidr_el1, {}", in(reg) val); + unsafe { + asm!("msr tpidr_el1, {}", in(reg) val); + } } pub unsafe fn tpidrro_el0() -> u64 { - let ret: u64; - asm!("mrs {}, tpidrro_el0", out(reg) ret); - ret + unsafe { + let ret: u64; + asm!("mrs {}, tpidrro_el0", out(reg) ret); + ret + } } pub unsafe fn tpidrro_el0_write(val: u64) { - asm!("msr tpidrro_el0, {}", in(reg) val); + unsafe { + asm!("msr tpidrro_el0, {}", in(reg) val); + } } pub unsafe fn esr_el1() -> u32 { - let ret: u32; - asm!("mrs {0:w}, esr_el1", out(reg) ret); - ret + unsafe { + let ret: u32; + asm!("mrs {0:w}, esr_el1", out(reg) ret); + ret + } } pub unsafe fn vhe_present() -> bool { - let mut mmfr1: u64; - asm!("mrs {}, id_aa64mmfr1_el1", out(reg) mmfr1); + unsafe { + let mut mmfr1: u64; + asm!("mrs {}, id_aa64mmfr1_el1", out(reg) mmfr1); - // The VHE (Virtualization Host Extensions) field is in bits [7:4]. - let vhe_field = (mmfr1 >> 4) & 0b1111; + // The VHE (Virtualization Host Extensions) field is in bits [7:4]. + let vhe_field = (mmfr1 >> 4) & 0b1111; - vhe_field != 0 + vhe_field != 0 + } } pub unsafe fn cntfrq_el0() -> u32 { - let ret: usize; - asm!("mrs {}, cntfrq_el0", out(reg) ret); - ret as u32 + unsafe { + let ret: usize; + asm!("mrs {}, cntfrq_el0", out(reg) ret); + ret as u32 + } } pub unsafe fn ptmr_ctrl() -> u32 { - let ret: usize; - asm!("mrs {}, cntp_ctl_el0", out(reg) ret); - ret as u32 + unsafe { + let ret: usize; + asm!("mrs {}, cntp_ctl_el0", out(reg) ret); + ret as u32 + } } pub unsafe fn ptmr_ctrl_write(val: u32) { - asm!("msr cntp_ctl_el0, {}", in(reg) val as usize); + unsafe { + asm!("msr cntp_ctl_el0, {}", in(reg) val as usize); + } } pub unsafe fn ptmr_tval() -> u32 { - let ret: usize; - asm!("mrs {0}, cntp_tval_el0", out(reg) ret); - ret as u32 + unsafe { + let ret: usize; + asm!("mrs {0}, cntp_tval_el0", out(reg) ret); + ret as u32 + } } pub unsafe fn ptmr_tval_write(val: u32) { - asm!("msr cntp_tval_el0, {}", in(reg) val as usize); + unsafe { + asm!("msr cntp_tval_el0, {}", in(reg) val as usize); + } } pub unsafe fn vtmr_ctrl() -> u32 { - let ret: usize; - asm!("mrs {}, cntv_ctl_el0", out(reg) ret); - ret as u32 + unsafe { + let ret: usize; + asm!("mrs {}, cntv_ctl_el0", out(reg) ret); + ret as u32 + } } pub unsafe fn vtmr_ctrl_write(val: u32) { - asm!("msr cntv_ctl_el0, {}", in(reg) val as usize); + unsafe { + asm!("msr cntv_ctl_el0, {}", in(reg) val as usize); + } } pub unsafe fn vtmr_tval() -> u32 { - let ret: usize; - asm!("mrs {0}, cntv_tval_el0", out(reg) ret); - ret as u32 + unsafe { + let ret: usize; + asm!("mrs {0}, cntv_tval_el0", out(reg) ret); + ret as u32 + } } pub unsafe fn vtmr_tval_write(val: u32) { - asm!("msr cntv_tval_el0, {}", in(reg) val as usize); + unsafe { + asm!("msr cntv_tval_el0, {}", in(reg) val as usize); + } } pub unsafe fn midr() -> u32 { - let ret: usize; - asm!("mrs {}, midr_el1", out(reg) ret); - ret as u32 + unsafe { + let ret: usize; + asm!("mrs {}, midr_el1", out(reg) ret); + ret as u32 + } } diff --git a/src/arch/aarch64/device/generic_timer.rs b/src/arch/aarch64/device/generic_timer.rs index 822677df73..eb145f1d7b 100644 --- a/src/arch/aarch64/device/generic_timer.rs +++ b/src/arch/aarch64/device/generic_timer.rs @@ -24,22 +24,24 @@ bitflags! { } pub unsafe fn init(fdt: &Fdt) { - let mut timer = GenericTimer::new(); - timer.init(); - if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) { - let irq = get_interrupt(fdt, &node, 1).unwrap(); - debug!("irq = {:?}", irq); - if let Some(ic_idx) = ic_for_chip(&fdt, &node) { - //PHYS_NONSECURE_PPI only - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_xlate(irq) - .unwrap(); - info!("generic_timer virq = {}", virq); - register_irq(virq as u32, Box::new(timer)); - IRQ_CHIP.irq_enable(virq as u32); - } else { - error!("Failed to find irq parent for generic timer"); + unsafe { + let mut timer = GenericTimer::new(); + timer.init(); + if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) { + let irq = get_interrupt(fdt, &node, 1).unwrap(); + debug!("irq = {:?}", irq); + if let Some(ic_idx) = ic_for_chip(&fdt, &node) { + //PHYS_NONSECURE_PPI only + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(irq) + .unwrap(); + info!("generic_timer virq = {}", virq); + register_irq(virq as u32, Box::new(timer)); + IRQ_CHIP.irq_enable(virq as u32); + } else { + error!("Failed to find irq parent for generic timer"); + } } } } diff --git a/src/arch/aarch64/device/irqchip/gic.rs b/src/arch/aarch64/device/irqchip/gic.rs index 1507360d42..22f4d24686 100644 --- a/src/arch/aarch64/device/irqchip/gic.rs +++ b/src/arch/aarch64/device/irqchip/gic.rs @@ -158,73 +158,83 @@ pub struct GicDistIf { impl GicDistIf { pub unsafe fn init(&mut self, addr: usize) { - self.address = addr; + unsafe { + self.address = addr; - // Disable IRQ Distribution - self.write(GICD_CTLR, 0); + // Disable IRQ Distribution + self.write(GICD_CTLR, 0); - let typer = self.read(GICD_TYPER); - self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1; - self.nirqs = ((typer & 0x1f) + 1) * 32; - info!( - "gic: Distributor supports {:?} CPUs and {:?} IRQs", - self.ncpus, self.nirqs - ); + let typer = self.read(GICD_TYPER); + self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1; + self.nirqs = ((typer & 0x1f) + 1) * 32; + info!( + "gic: Distributor supports {:?} CPUs and {:?} IRQs", + self.ncpus, self.nirqs + ); - // Set all SPIs to level triggered - for irq in (32..self.nirqs).step_by(16) { - self.write(GICD_ICFGR + ((irq / 16) * 4), 0); - } + // Set all SPIs to level triggered + for irq in (32..self.nirqs).step_by(16) { + self.write(GICD_ICFGR + ((irq / 16) * 4), 0); + } - // Disable all SPIs - for irq in (32..self.nirqs).step_by(32) { - self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff); - } + // Disable all SPIs + for irq in (32..self.nirqs).step_by(32) { + self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff); + } - // Affine all SPIs to CPU0 and set priorities for all IRQs - for irq in 0..self.nirqs { - if irq > 31 { - let ext_offset = GICD_ITARGETSR + (4 * (irq / 4)); + // Affine all SPIs to CPU0 and set priorities for all IRQs + for irq in 0..self.nirqs { + if irq > 31 { + let ext_offset = GICD_ITARGETSR + (4 * (irq / 4)); + let int_offset = irq % 4; + let mut val = self.read(ext_offset); + val |= 0b0000_0001 << (8 * int_offset); + self.write(ext_offset, val); + } + + let ext_offset = GICD_IPRIORITY + (4 * (irq / 4)); let int_offset = irq % 4; let mut val = self.read(ext_offset); - val |= 0b0000_0001 << (8 * int_offset); + val |= 0b0000_0000 << (8 * int_offset); self.write(ext_offset, val); } - let ext_offset = GICD_IPRIORITY + (4 * (irq / 4)); - let int_offset = irq % 4; - let mut val = self.read(ext_offset); - val |= 0b0000_0000 << (8 * int_offset); - self.write(ext_offset, val); + // Enable IRQ group 0 and group 1 non-secure distribution + self.write(GICD_CTLR, 0x3); } - - // Enable IRQ group 0 and group 1 non-secure distribution - self.write(GICD_CTLR, 0x3); } pub unsafe fn irq_enable(&mut self, irq: u32) { - let offset = GICD_ISENABLER + (4 * (irq / 32)); - let shift = 1 << (irq % 32); - let mut val = self.read(offset); - val |= shift; - self.write(offset, val); + unsafe { + let offset = GICD_ISENABLER + (4 * (irq / 32)); + let shift = 1 << (irq % 32); + let mut val = self.read(offset); + val |= shift; + self.write(offset, val); + } } pub unsafe fn irq_disable(&mut self, irq: u32) { - let offset = GICD_ICENABLER + (4 * (irq / 32)); - let shift = 1 << (irq % 32); - let mut val = self.read(offset); - val |= shift; - self.write(offset, val); + unsafe { + let offset = GICD_ICENABLER + (4 * (irq / 32)); + let shift = 1 << (irq % 32); + let mut val = self.read(offset); + val |= shift; + self.write(offset, val); + } } unsafe fn read(&self, reg: u32) -> u32 { - let val = read_volatile((self.address + reg as usize) as *const u32); - val + unsafe { + let val = read_volatile((self.address + reg as usize) as *const u32); + val + } } unsafe fn write(&mut self, reg: u32, value: u32) { - write_volatile((self.address + reg as usize) as *mut u32, value); + unsafe { + write_volatile((self.address + reg as usize) as *mut u32, value); + } } } @@ -235,32 +245,42 @@ pub struct GicCpuIf { impl GicCpuIf { pub unsafe fn init(&mut self, addr: usize) { - self.address = addr; + unsafe { + self.address = addr; - // Enable CPU0's GIC interface - self.write(GICC_CTLR, 1); - // Set CPU0's Interrupt Priority Mask - self.write(GICC_PMR, 0xff); + // Enable CPU0's GIC interface + self.write(GICC_CTLR, 1); + // Set CPU0's Interrupt Priority Mask + self.write(GICC_PMR, 0xff); + } } unsafe fn irq_ack(&mut self) -> u32 { - let irq = self.read(GICC_IAR) & 0x1ff; - if irq == 1023 { - panic!("irq_ack: got ID 1023!!!"); + unsafe { + let irq = self.read(GICC_IAR) & 0x1ff; + if irq == 1023 { + panic!("irq_ack: got ID 1023!!!"); + } + irq } - irq } unsafe fn irq_eoi(&mut self, irq: u32) { - self.write(GICC_EOIR, irq); + unsafe { + self.write(GICC_EOIR, irq); + } } unsafe fn read(&self, reg: u32) -> u32 { - let val = read_volatile((self.address + reg as usize) as *const u32); - val + unsafe { + let val = read_volatile((self.address + reg as usize) as *const u32); + val + } } unsafe fn write(&mut self, reg: u32, value: u32) { - write_volatile((self.address + reg as usize) as *mut u32, value); + unsafe { + write_volatile((self.address + reg as usize) as *mut u32, value); + } } } diff --git a/src/arch/aarch64/device/irqchip/gicv3.rs b/src/arch/aarch64/device/irqchip/gicv3.rs index 076965d62e..a4ec2ad62b 100644 --- a/src/arch/aarch64/device/irqchip/gicv3.rs +++ b/src/arch/aarch64/device/irqchip/gicv3.rs @@ -149,39 +149,45 @@ pub struct GicV3CpuIf; impl GicV3CpuIf { pub unsafe fn init(&mut self) { - // Enable system register access - { - let value = 1_usize; - asm!("msr icc_sre_el1, {}", in(reg) value); - } - // Set control register - { - let value = 0_usize; - asm!("msr icc_ctlr_el1, {}", in(reg) value); - } - // Enable non-secure group 1 - { - let value = 1_usize; - asm!("msr icc_igrpen1_el1, {}", in(reg) value); - } - // Set CPU0's Interrupt Priority Mask - { - let value = 0xFF_usize; - asm!("msr icc_pmr_el1, {}", in(reg) value); + unsafe { + // Enable system register access + { + let value = 1_usize; + asm!("msr icc_sre_el1, {}", in(reg) value); + } + // Set control register + { + let value = 0_usize; + asm!("msr icc_ctlr_el1, {}", in(reg) value); + } + // Enable non-secure group 1 + { + let value = 1_usize; + asm!("msr icc_igrpen1_el1, {}", in(reg) value); + } + // Set CPU0's Interrupt Priority Mask + { + let value = 0xFF_usize; + asm!("msr icc_pmr_el1, {}", in(reg) value); + } } } unsafe fn irq_ack(&mut self) -> u32 { - let mut irq: usize; - asm!("mrs {}, icc_iar1_el1", out(reg) irq); - irq &= 0x1ff; - if irq == 1023 { - panic!("irq_ack: got ID 1023!!!"); + unsafe { + let mut irq: usize; + asm!("mrs {}, icc_iar1_el1", out(reg) irq); + irq &= 0x1ff; + if irq == 1023 { + panic!("irq_ack: got ID 1023!!!"); + } + irq as u32 } - irq as u32 } unsafe fn irq_eoi(&mut self, irq: u32) { - asm!("msr icc_eoir1_el1, {}", in(reg) irq as usize); + unsafe { + asm!("msr icc_eoir1_el1, {}", in(reg) irq as usize); + } } } diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs index 80e0b8747e..067b897d8a 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs @@ -72,47 +72,55 @@ impl Bcm2835ArmInterruptController { } } unsafe fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize, Option)> { - //assert address_cells == 0x1, size_cells == 0x1 - let mem = node.reg().unwrap().nth(0).unwrap(); - let base = get_mmio_address(fdt, node, &mem).unwrap(); - let size = mem.size.unwrap() as u32; - let mut ret_virq = None; + unsafe { + //assert address_cells == 0x1, size_cells == 0x1 + let mem = node.reg().unwrap().nth(0).unwrap(); + let base = get_mmio_address(fdt, node, &mem).unwrap(); + let size = mem.size.unwrap() as u32; + let mut ret_virq = None; - if let Some(interrupt_parent) = node.property("interrupt-parent") { - let phandle = interrupt_parent.as_usize().unwrap() as u32; - let irq = get_interrupt(fdt, node, 0).unwrap(); - let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap(); - //PHYS_NONSECURE_PPI only - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_xlate(irq) - .unwrap(); - info!( - "register bcm2835arm_ctrl as ic_idx {}'s child virq = {}", - ic_idx, virq - ); - ret_virq = Some(virq); + if let Some(interrupt_parent) = node.property("interrupt-parent") { + let phandle = interrupt_parent.as_usize().unwrap() as u32; + let irq = get_interrupt(fdt, node, 0).unwrap(); + let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap(); + //PHYS_NONSECURE_PPI only + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(irq) + .unwrap(); + info!( + "register bcm2835arm_ctrl as ic_idx {}'s child virq = {}", + ic_idx, virq + ); + ret_virq = Some(virq); + } + Ok((base as usize, size as usize, ret_virq)) } - Ok((base as usize, size as usize, ret_virq)) } unsafe fn init(&mut self) { - debug!("IRQ BCM2835 INIT"); - //disable all interrupt - self.write(DISABLE_0, 0xffff_ffff); - self.write(DISABLE_1, 0xffff_ffff); - self.write(DISABLE_2, 0xffff_ffff); + unsafe { + debug!("IRQ BCM2835 INIT"); + //disable all interrupt + self.write(DISABLE_0, 0xffff_ffff); + self.write(DISABLE_1, 0xffff_ffff); + self.write(DISABLE_2, 0xffff_ffff); - debug!("IRQ BCM2835 END"); + debug!("IRQ BCM2835 END"); + } } unsafe fn read(&self, reg: u32) -> u32 { - let val = read_volatile((self.address + reg as usize) as *const u32); - val + unsafe { + let val = read_volatile((self.address + reg as usize) as *const u32); + val + } } unsafe fn write(&mut self, reg: u32, value: u32) { - write_volatile((self.address + reg as usize) as *mut u32, value); + unsafe { + write_volatile((self.address + reg as usize) as *mut u32, value); + } } } diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs index fc3bd45510..a3e9357fca 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs @@ -89,24 +89,30 @@ impl Bcm2836ArmInterruptController { } unsafe fn init(&mut self) { - debug!("IRQ BCM2836 INIT"); - //init local timer freq - self.write(LOCAL_CONTROL, 0x0); - self.write(LOCAL_PRESCALER, 0x8000_0000); + unsafe { + debug!("IRQ BCM2836 INIT"); + //init local timer freq + self.write(LOCAL_CONTROL, 0x0); + self.write(LOCAL_PRESCALER, 0x8000_0000); - //routing all irq to core - self.write(LOCAL_GPU_ROUTING, self.active_cpu); - debug!("routing all irq to core {}", self.active_cpu); - debug!("IRQ BCM2836 END"); + //routing all irq to core + self.write(LOCAL_GPU_ROUTING, self.active_cpu); + debug!("routing all irq to core {}", self.active_cpu); + debug!("IRQ BCM2836 END"); + } } unsafe fn read(&self, reg: u32) -> u32 { - let val = read_volatile((self.address + reg as usize) as *const u32); - val + unsafe { + let val = read_volatile((self.address + reg as usize) as *const u32); + val + } } unsafe fn write(&mut self, reg: u32, value: u32) { - write_volatile((self.address + reg as usize) as *mut u32, value); + unsafe { + write_volatile((self.address + reg as usize) as *mut u32, value); + } } } diff --git a/src/arch/aarch64/device/mod.rs b/src/arch/aarch64/device/mod.rs index 0f729c1697..5c977b7f56 100644 --- a/src/arch/aarch64/device/mod.rs +++ b/src/arch/aarch64/device/mod.rs @@ -16,36 +16,40 @@ pub static ROOT_IC_IDX: AtomicUsize = AtomicUsize::new(0); pub static ROOT_IC_IDX_IS_SET: AtomicUsize = AtomicUsize::new(0); unsafe fn init_root_ic(fdt: &Fdt) { - let is_set = ROOT_IC_IDX_IS_SET.load(Ordering::Relaxed); - if is_set != 0 { - let ic_idx = ROOT_IC_IDX.load(Ordering::Relaxed); - info!("Already selected {} as root ic", ic_idx); - return; - } + unsafe { + let is_set = ROOT_IC_IDX_IS_SET.load(Ordering::Relaxed); + if is_set != 0 { + let ic_idx = ROOT_IC_IDX.load(Ordering::Relaxed); + info!("Already selected {} as root ic", ic_idx); + return; + } - let root_irqc_phandle = fdt - .root() - .property("interrupt-parent") - .unwrap() - .as_usize() - .unwrap(); - let ic_idx = IRQ_CHIP - .phandle_to_ic_idx(root_irqc_phandle as u32) - .unwrap(); - info!("select {} as root ic", ic_idx); - ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed); + let root_irqc_phandle = fdt + .root() + .property("interrupt-parent") + .unwrap() + .as_usize() + .unwrap(); + let ic_idx = IRQ_CHIP + .phandle_to_ic_idx(root_irqc_phandle as u32) + .unwrap(); + info!("select {} as root ic", ic_idx); + ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed); + } } pub unsafe fn init_devicetree(fdt: &Fdt) { - info!("IRQCHIP INIT"); - crate::dtb::irqchip::init(&fdt); - init_root_ic(&fdt); - info!("GIT INIT"); - generic_timer::init(fdt); - info!("SERIAL INIT"); - serial::init(fdt); - info!("RTC INIT"); - rtc::init(fdt); + unsafe { + info!("IRQCHIP INIT"); + crate::dtb::irqchip::init(&fdt); + init_root_ic(&fdt); + info!("GIT INIT"); + generic_timer::init(fdt); + info!("SERIAL INIT"); + serial::init(fdt); + info!("RTC INIT"); + rtc::init(fdt); + } } #[derive(Default)] diff --git a/src/arch/aarch64/device/rtc.rs b/src/arch/aarch64/device/rtc.rs index 6553ace359..f4cb849891 100644 --- a/src/arch/aarch64/device/rtc.rs +++ b/src/arch/aarch64/device/rtc.rs @@ -30,7 +30,7 @@ struct Pl031rtc { impl Pl031rtc { unsafe fn read(&self, reg: usize) -> u32 { - read_volatile((crate::PHYS_OFFSET + self.phys + reg) as *const u32) + unsafe { read_volatile((crate::PHYS_OFFSET + self.phys + reg) as *const u32) } } pub fn time(&mut self) -> u64 { diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs index f172cd8e5c..81df8716f2 100644 --- a/src/arch/aarch64/device/serial.rs +++ b/src/arch/aarch64/device/serial.rs @@ -76,63 +76,67 @@ impl InterruptHandler for Com1Irq { } pub unsafe fn init_early(dtb: &Fdt) { - if COM1.lock().is_some() { - // Hardcoded UART - return; - } + unsafe { + if COM1.lock().is_some() { + // Hardcoded UART + return; + } - if let Some((phys, size, skip_init, cts, compatible)) = diag_uart_range(dtb) { - let virt = crate::PHYS_OFFSET + phys; - let serial_opt = if compatible.contains("arm,pl011") { - let mut serial_port = uart_pl011::SerialPort::new(virt, cts); - if !skip_init { - serial_port.init(false); - } - Some(SerialKind::Pl011(serial_port)) - } else if compatible.contains("ns16550a") || compatible.contains("snps,dw-apb-uart") { - //TODO: get actual register size from device tree - let serial_port = uart_16550::SerialPort::>::new(virt); - if !skip_init { - serial_port.init(); - } - Some(SerialKind::Ns16550u32(serial_port)) - } else { - None - }; - match serial_opt { - Some(serial) => { - *COM1.lock() = Some(serial); - info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size); - } - None => { - log::warn!( - "UART {:?} at {:#X} size {:#X}: no driver found", - compatible, - virt, - size - ); + if let Some((phys, size, skip_init, cts, compatible)) = diag_uart_range(dtb) { + let virt = crate::PHYS_OFFSET + phys; + let serial_opt = if compatible.contains("arm,pl011") { + let mut serial_port = uart_pl011::SerialPort::new(virt, cts); + if !skip_init { + serial_port.init(false); + } + Some(SerialKind::Pl011(serial_port)) + } else if compatible.contains("ns16550a") || compatible.contains("snps,dw-apb-uart") { + //TODO: get actual register size from device tree + let serial_port = uart_16550::SerialPort::>::new(virt); + if !skip_init { + serial_port.init(); + } + Some(SerialKind::Ns16550u32(serial_port)) + } else { + None + }; + match serial_opt { + Some(serial) => { + *COM1.lock() = Some(serial); + info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size); + } + None => { + log::warn!( + "UART {:?} at {:#X} size {:#X}: no driver found", + compatible, + virt, + size + ); + } } } } } pub unsafe fn init(fdt: &Fdt) { - //TODO: find actual serial device, not just any PL011 - if let Some(node) = fdt.find_compatible(&["arm,pl011"]) { - let irq = get_interrupt(fdt, &node, 0).unwrap(); - if let Some(ic_idx) = ic_for_chip(&fdt, &node) { - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_xlate(irq) - .unwrap(); - info!("serial_port virq = {}", virq); - register_irq(virq as u32, Box::new(Com1Irq {})); - IRQ_CHIP.irq_enable(virq as u32); - } else { - error!("serial port irq parent not found"); + unsafe { + //TODO: find actual serial device, not just any PL011 + if let Some(node) = fdt.find_compatible(&["arm,pl011"]) { + let irq = get_interrupt(fdt, &node, 0).unwrap(); + if let Some(ic_idx) = ic_for_chip(&fdt, &node) { + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(irq) + .unwrap(); + info!("serial_port virq = {}", virq); + register_irq(virq as u32, Box::new(Com1Irq {})); + IRQ_CHIP.irq_enable(virq as u32); + } else { + error!("serial port irq parent not found"); + } + } + if let Some(ref mut serial_port) = *COM1.lock() { + serial_port.enable_irq(); } } - if let Some(ref mut serial_port) = *COM1.lock() { - serial_port.enable_irq(); - } } diff --git a/src/arch/aarch64/interrupt/exception.rs b/src/arch/aarch64/interrupt/exception.rs index bd996f790f..04accdbfec 100644 --- a/src/arch/aarch64/interrupt/exception.rs +++ b/src/arch/aarch64/interrupt/exception.rs @@ -26,9 +26,11 @@ fn iss(esr: usize) -> u32 { } unsafe fn far_el1() -> usize { - let ret: usize; - core::arch::asm!("mrs {}, far_el1", out(reg) ret); - ret + unsafe { + let ret: usize; + core::arch::asm!("mrs {}, far_el1", out(reg) ret); + ret + } } unsafe fn instr_data_abort_inner( @@ -37,48 +39,56 @@ unsafe fn instr_data_abort_inner( instr_not_data: bool, _from: &str, ) -> bool { - let iss = iss(stack.iret.esr_el1); - let fsc = iss & 0x3F; - //dbg!(fsc); + unsafe { + let iss = iss(stack.iret.esr_el1); + let fsc = iss & 0x3F; + //dbg!(fsc); - let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111; - //let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111; - let write_not_read_if_data = iss & (1 << 6) != 0; + let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111; + //let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111; + let write_not_read_if_data = iss & (1 << 6) != 0; - let mut flags = GenericPfFlags::empty(); - flags.set(GenericPfFlags::PRESENT, !was_translation_fault); + let mut flags = GenericPfFlags::empty(); + flags.set(GenericPfFlags::PRESENT, !was_translation_fault); - // TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64 - // doesn't appear to require that flag to be set if the read alone would trigger a fault. - flags.set( - GenericPfFlags::INVOLVED_WRITE, - write_not_read_if_data && !instr_not_data, - ); - flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data); - flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user); + // TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64 + // doesn't appear to require that flag to be set if the read alone would trigger a fault. + flags.set( + GenericPfFlags::INVOLVED_WRITE, + write_not_read_if_data && !instr_not_data, + ); + flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data); + flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user); - let faulting_addr = VirtualAddress::new(far_el1()); - //dbg!(faulting_addr, flags, from); + let faulting_addr = VirtualAddress::new(far_el1()); + //dbg!(faulting_addr, flags, from); - crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok() + crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok() + } } unsafe fn cntfrq_el0() -> usize { - let ret: usize; - core::arch::asm!("mrs {}, cntfrq_el0", out(reg) ret); - ret + unsafe { + let ret: usize; + core::arch::asm!("mrs {}, cntfrq_el0", out(reg) ret); + ret + } } unsafe fn cntpct_el0() -> usize { - let ret: usize; - core::arch::asm!("mrs {}, cntpct_el0", out(reg) ret); - ret + unsafe { + let ret: usize; + core::arch::asm!("mrs {}, cntpct_el0", out(reg) ret); + ret + } } unsafe fn cntvct_el0() -> usize { - let ret: usize; - core::arch::asm!("mrs {}, cntvct_el0", out(reg) ret); - ret + unsafe { + let ret: usize; + core::arch::asm!("mrs {}, cntvct_el0", out(reg) ret); + ret + } } unsafe fn instr_trapped_msr_mrs_inner( @@ -87,52 +97,54 @@ unsafe fn instr_trapped_msr_mrs_inner( _instr_not_data: bool, _from: &str, ) -> bool { - let iss = iss(stack.iret.esr_el1); - // let res0 = (iss & 0x1C0_0000) >> 22; - let op0 = (iss & 0x030_0000) >> 20; - let op2 = (iss & 0x00e_0000) >> 17; - let op1 = (iss & 0x001_c000) >> 14; - let crn = (iss & 0x000_3c00) >> 10; - let rt = (iss & 0x000_03e0) >> 5; - let crm = (iss & 0x000_001e) >> 1; - let dir = iss & 0x000_0001; + unsafe { + let iss = iss(stack.iret.esr_el1); + // let res0 = (iss & 0x1C0_0000) >> 22; + let op0 = (iss & 0x030_0000) >> 20; + let op2 = (iss & 0x00e_0000) >> 17; + let op1 = (iss & 0x001_c000) >> 14; + let crn = (iss & 0x000_3c00) >> 10; + let rt = (iss & 0x000_03e0) >> 5; + let crm = (iss & 0x000_001e) >> 1; + let dir = iss & 0x000_0001; - /* - print!("iss=0x{:x}, res0=0b{:03b}, op0=0b{:02b}\n - op2=0b{:03b}, op1=0b{:03b}, crn=0b{:04b}\n - rt=0b{:05b}, crm=0b{:04b}, dir=0b{:b}\n", - iss, res0, op0, op2, op1, crn, rt, crm, dir); - */ + /* + print!("iss=0x{:x}, res0=0b{:03b}, op0=0b{:02b}\n + op2=0b{:03b}, op1=0b{:03b}, crn=0b{:04b}\n + rt=0b{:05b}, crm=0b{:04b}, dir=0b{:b}\n", + iss, res0, op0, op2, op1, crn, rt, crm, dir); + */ - match (op0, op1, crn, crm, op2, dir) { - //MRS , CNTFRQ_EL0 - (0b11, 0b011, 0b1110, 0b0000, 0b000, 0b1) => { - let reg_val = cntfrq_el0(); - stack.store_reg(rt as usize, reg_val); - //skip faulting instruction, A64 instructions are always 32-bits - stack.iret.elr_el1 += 4; - return true; + match (op0, op1, crn, crm, op2, dir) { + //MRS , CNTFRQ_EL0 + (0b11, 0b011, 0b1110, 0b0000, 0b000, 0b1) => { + let reg_val = cntfrq_el0(); + stack.store_reg(rt as usize, reg_val); + //skip faulting instruction, A64 instructions are always 32-bits + stack.iret.elr_el1 += 4; + return true; + } + //MRS , CNTPCT_EL0 + (0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => { + let reg_val = cntpct_el0(); + stack.store_reg(rt as usize, reg_val); + //skip faulting instruction, A64 instructions are always 32-bits + stack.iret.elr_el1 += 4; + return true; + } + //MRS , CNTVCT_EL0 + (0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => { + let reg_val = cntvct_el0(); + stack.store_reg(rt as usize, reg_val); + //skip faulting instruction, A64 instructions are always 32-bits + stack.iret.elr_el1 += 4; + return true; + } + _ => {} } - //MRS , CNTPCT_EL0 - (0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => { - let reg_val = cntpct_el0(); - stack.store_reg(rt as usize, reg_val); - //skip faulting instruction, A64 instructions are always 32-bits - stack.iret.elr_el1 += 4; - return true; - } - //MRS , CNTVCT_EL0 - (0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => { - let reg_val = cntvct_el0(); - stack.store_reg(rt as usize, reg_val); - //skip faulting instruction, A64 instructions are always 32-bits - stack.iret.elr_el1 += 4; - return true; - } - _ => {} + + false } - - false } exception_stack!(synchronous_exception_at_el1_with_spx, |stack| { @@ -155,19 +167,21 @@ exception_stack!(synchronous_exception_at_el1_with_spx, |stack| { } }); unsafe fn pf_inner(stack: &mut InterruptStack, ty: u8, from: &str) -> bool { - match ty { - // "Data Abort taken from a lower Exception level" - 0b100100 => instr_data_abort_inner(stack, true, false, from), - // "Data Abort taken without a change in Exception level" - 0b100101 => instr_data_abort_inner(stack, false, false, from), - // "Instruction Abort taken from a lower Exception level" - 0b100000 => instr_data_abort_inner(stack, true, true, from), - // "Instruction Abort taken without a change in Exception level" - 0b100001 => instr_data_abort_inner(stack, false, true, from), - // "Trapped MSR, MRS or System instruction execution in AArch64 state" - 0b011000 => instr_trapped_msr_mrs_inner(stack, true, true, from), + unsafe { + match ty { + // "Data Abort taken from a lower Exception level" + 0b100100 => instr_data_abort_inner(stack, true, false, from), + // "Data Abort taken without a change in Exception level" + 0b100101 => instr_data_abort_inner(stack, false, false, from), + // "Instruction Abort taken from a lower Exception level" + 0b100000 => instr_data_abort_inner(stack, true, true, from), + // "Instruction Abort taken without a change in Exception level" + 0b100001 => instr_data_abort_inner(stack, false, true, from), + // "Trapped MSR, MRS or System instruction execution in AArch64 state" + 0b011000 => instr_trapped_msr_mrs_inner(stack, true, true, from), - _ => return false, + _ => return false, + } } } diff --git a/src/arch/aarch64/interrupt/handler.rs b/src/arch/aarch64/interrupt/handler.rs index c21d0e9fd3..f0358fe0f1 100644 --- a/src/arch/aarch64/interrupt/handler.rs +++ b/src/arch/aarch64/interrupt/handler.rs @@ -267,7 +267,7 @@ impl InterruptStack { #[macro_export] macro_rules! aarch64_asm { - ($($strings:expr,)+) => { + ($($strings:expr_2021,)+) => { core::arch::global_asm!(concat!( $($strings),+, )); @@ -376,11 +376,11 @@ macro_rules! pop_special { macro_rules! exception_stack { ($name:ident, |$stack:ident| $code:block) => { #[naked] - #[no_mangle] - pub unsafe extern "C" fn $name(stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { - unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name(stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { unsafe { + unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { unsafe { $code - } + }} core::arch::naked_asm!(concat!( // Backup all userspace registers to stack push_preserved!(), @@ -399,17 +399,19 @@ macro_rules! exception_stack { "eret\n", ), sym inner); - } + }} }; } #[naked] pub unsafe extern "C" fn enter_usermode() -> ! { - core::arch::naked_asm!(concat!( - "blr x28\n", - // Restore all userspace registers - pop_special!(), - pop_scratch!(), - pop_preserved!(), - "eret\n", - )); + unsafe { + core::arch::naked_asm!(concat!( + "blr x28\n", + // Restore all userspace registers + pop_special!(), + pop_scratch!(), + pop_preserved!(), + "eret\n", + )); + } } diff --git a/src/arch/aarch64/interrupt/irq.rs b/src/arch/aarch64/interrupt/irq.rs index 53f5f5a00d..4ee05c02e8 100644 --- a/src/arch/aarch64/interrupt/irq.rs +++ b/src/arch/aarch64/interrupt/irq.rs @@ -4,9 +4,11 @@ use core::sync::atomic::Ordering; // use crate::percpu::PercpuBlock; unsafe fn irq_ack() -> (u32, Option) { - let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic; - let irq = ic.irq_ack(); - (irq, ic.irq_to_virq(irq)) + unsafe { + let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic; + let irq = ic.irq_ack(); + (irq, ic.irq_to_virq(irq)) + } } exception_stack!(irq_at_el0, |_stack| { @@ -33,11 +35,13 @@ exception_stack!(irq_at_el1, |_stack| { //TODO pub unsafe fn trigger(irq: u32) { - // FIXME add_irq accepts a u8 as irq number - // PercpuBlock::current().stats.add_irq(irq); + unsafe { + // FIXME add_irq accepts a u8 as irq number + // PercpuBlock::current().stats.add_irq(irq); - irq_trigger(irq.try_into().unwrap()); - IRQ_CHIP.irq_eoi(irq); + irq_trigger(irq.try_into().unwrap()); + IRQ_CHIP.irq_eoi(irq); + } } /* diff --git a/src/arch/aarch64/interrupt/mod.rs b/src/arch/aarch64/interrupt/mod.rs index 42b7da824d..ad9d12e8f0 100644 --- a/src/arch/aarch64/interrupt/mod.rs +++ b/src/arch/aarch64/interrupt/mod.rs @@ -15,7 +15,9 @@ pub use self::handler::InterruptStack; /// Clear interrupts #[inline(always)] pub unsafe fn disable() { - asm!("msr daifset, #2"); + unsafe { + asm!("msr daifset, #2"); + } } /// Set interrupts and halt @@ -23,8 +25,10 @@ pub unsafe fn disable() { /// Performing enable followed by halt is not guaranteed to be atomic, use this instead! #[inline(always)] pub unsafe fn enable_and_halt() { - asm!("msr daifclr, #2"); - asm!("wfi"); + unsafe { + asm!("msr daifclr, #2"); + asm!("wfi"); + } } /// Set interrupts and nop @@ -32,14 +36,18 @@ pub unsafe fn enable_and_halt() { /// Simply enabling interrupts does not gurantee that they will trigger, use this instead! #[inline(always)] pub unsafe fn enable_and_nop() { - asm!("msr daifclr, #2"); - asm!("nop"); + unsafe { + asm!("msr daifclr, #2"); + asm!("nop"); + } } /// Halt instruction #[inline(always)] pub unsafe fn halt() { - asm!("wfi"); + unsafe { + asm!("wfi"); + } } /// Pause instruction @@ -51,12 +59,14 @@ pub fn pause() { #[inline(always)] pub unsafe fn init() { - // Setup interrupt handlers - asm!( - " + unsafe { + // Setup interrupt handlers + asm!( + " ldr {tmp}, =exception_vector_base msr vbar_el1, {tmp} ", - tmp = out(reg) _, - ); + tmp = out(reg) _, + ); + } } diff --git a/src/arch/aarch64/interrupt/syscall.rs b/src/arch/aarch64/interrupt/syscall.rs index 40bb7e1624..2b17a71182 100644 --- a/src/arch/aarch64/interrupt/syscall.rs +++ b/src/arch/aarch64/interrupt/syscall.rs @@ -1,7 +1,7 @@ -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn do_exception_unhandled() {} -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn do_exception_synchronous() {} #[allow(dead_code)] diff --git a/src/arch/aarch64/interrupt/trace.rs b/src/arch/aarch64/interrupt/trace.rs index b5ddb71a82..72ec17d758 100644 --- a/src/arch/aarch64/interrupt/trace.rs +++ b/src/arch/aarch64/interrupt/trace.rs @@ -8,21 +8,25 @@ pub struct StackTrace { impl StackTrace { #[inline(always)] pub unsafe fn start() -> Option { - let fp: usize; - asm!("mov {}, fp", out(reg) fp); - let pc_ptr = fp.checked_add(mem::size_of::())?; - Some(StackTrace { - fp, - pc_ptr: pc_ptr as *const usize, - }) + unsafe { + let fp: usize; + asm!("mov {}, fp", out(reg) fp); + let pc_ptr = fp.checked_add(mem::size_of::())?; + Some(StackTrace { + fp, + pc_ptr: pc_ptr as *const usize, + }) + } } pub unsafe fn next(self) -> Option { - let fp = *(self.fp as *const usize); - let pc_ptr = fp.checked_add(mem::size_of::())?; - Some(StackTrace { - fp: fp, - pc_ptr: pc_ptr as *const usize, - }) + unsafe { + let fp = *(self.fp as *const usize); + let pc_ptr = fp.checked_add(mem::size_of::())?; + Some(StackTrace { + fp: fp, + pc_ptr: pc_ptr as *const usize, + }) + } } } diff --git a/src/arch/aarch64/macros.rs b/src/arch/aarch64/macros.rs index 4e3566fc55..a9c83081f0 100644 --- a/src/arch/aarch64/macros.rs +++ b/src/arch/aarch64/macros.rs @@ -11,6 +11,6 @@ macro_rules! print { #[macro_export] macro_rules! println { () => (print!("\n")); - ($fmt:expr) => (print!(concat!($fmt, "\n"))); - ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); + ($fmt:expr_2021) => (print!(concat!($fmt, "\n"))); + ($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } diff --git a/src/arch/aarch64/misc.rs b/src/arch/aarch64/misc.rs index 8620f7e9bc..88cfaefac4 100644 --- a/src/arch/aarch64/misc.rs +++ b/src/arch/aarch64/misc.rs @@ -12,10 +12,12 @@ impl PercpuBlock { #[cold] pub unsafe fn init(cpu_id: LogicalCpuId) { - let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory"); - let virt = RmmA::phys_to_virt(frame.base()).data() as *mut PercpuBlock; + unsafe { + let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory"); + let virt = RmmA::phys_to_virt(frame.base()).data() as *mut PercpuBlock; - virt.write(PercpuBlock::init(cpu_id)); + virt.write(PercpuBlock::init(cpu_id)); - crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64); + crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64); + } } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 16714c189d..eb10d09ab2 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -40,11 +40,12 @@ pub use ::rmm::AArch64Arch as CurrentRmmArch; pub use arch_copy_to_user as arch_copy_from_user; #[naked] -#[link_section = ".usercopy-fns"] +#[unsafe(link_section = ".usercopy-fns")] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - // x0, x1, x2 - core::arch::naked_asm!( - " + unsafe { + // x0, x1, x2 + core::arch::naked_asm!( + " mov x4, x0 mov x0, 0 2: @@ -62,7 +63,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - 3: ret " - ); + ); + } } pub const KFX_SIZE: usize = 1024; diff --git a/src/arch/aarch64/paging/mod.rs b/src/arch/aarch64/paging/mod.rs index 09906ae678..019628429b 100644 --- a/src/arch/aarch64/paging/mod.rs +++ b/src/arch/aarch64/paging/mod.rs @@ -18,19 +18,23 @@ pub const PAGE_MASK: usize = RmmA::PAGE_OFFSET_MASK; /// Setup Memory Access Indirection Register #[cold] unsafe fn init_mair() { - let mut val: control_regs::MairEl1 = control_regs::mair_el1(); + unsafe { + let mut val: control_regs::MairEl1 = control_regs::mair_el1(); - val.insert(control_regs::MairEl1::DEVICE_MEMORY); - val.insert(control_regs::MairEl1::NORMAL_UNCACHED_MEMORY); - val.insert(control_regs::MairEl1::NORMAL_WRITEBACK_MEMORY); + val.insert(control_regs::MairEl1::DEVICE_MEMORY); + val.insert(control_regs::MairEl1::NORMAL_UNCACHED_MEMORY); + val.insert(control_regs::MairEl1::NORMAL_WRITEBACK_MEMORY); - control_regs::mair_el1_write(val); + control_regs::mair_el1_write(val); + } } /// Initialize MAIR #[cold] pub unsafe fn init() { - init_mair(); + unsafe { + init_mair(); + } } /// Page diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index 0d9b4fb426..854b41ea4f 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -52,189 +52,191 @@ pub struct KernelArgs { } /// The entry to Rust, all things must be initialized -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { - let bootstrap = { - let args = args_ptr.read(); + unsafe { + let bootstrap = { + let args = args_ptr.read(); - // BSS should already be zero - { - assert_eq!(BSS_TEST_ZERO, 0); - assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF); - } + // BSS should already be zero + { + assert_eq!(BSS_TEST_ZERO, 0); + assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF); + } - KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst); - KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst); + KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst); + KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst); - // Convert env to slice - let env = slice::from_raw_parts( - (crate::PHYS_OFFSET + args.env_base) as *const u8, - args.env_size, - ); + // Convert env to slice + let env = slice::from_raw_parts( + (crate::PHYS_OFFSET + args.env_base) as *const u8, + args.env_size, + ); - // Set up graphical debug - #[cfg(feature = "graphical_debug")] - graphical_debug::init(env); + // Set up graphical debug + #[cfg(feature = "graphical_debug")] + graphical_debug::init(env); - // Get hardware descriptor data - //TODO: use env {DTB,RSDT}_{BASE,SIZE}? - let hwdesc_data = if args.hwdesc_base != 0 { - Some(unsafe { - slice::from_raw_parts( - (crate::PHYS_OFFSET + args.hwdesc_base) as *const u8, - args.hwdesc_size, - ) - }) - } else { - None - }; - - let dtb_res = hwdesc_data - .ok_or(fdt::FdtError::BadPtr) - .and_then(|data| Fdt::new(data)); - - let rsdp_opt = hwdesc_data.and_then(|data| { - if data.starts_with(b"RSD PTR ") { - Some(data.as_ptr()) + // Get hardware descriptor data + //TODO: use env {DTB,RSDT}_{BASE,SIZE}? + let hwdesc_data = if args.hwdesc_base != 0 { + Some(unsafe { + slice::from_raw_parts( + (crate::PHYS_OFFSET + args.hwdesc_base) as *const u8, + args.hwdesc_size, + ) + }) } else { None + }; + + let dtb_res = hwdesc_data + .ok_or(fdt::FdtError::BadPtr) + .and_then(|data| Fdt::new(data)); + + let rsdp_opt = hwdesc_data.and_then(|data| { + if data.starts_with(b"RSD PTR ") { + Some(data.as_ptr()) + } else { + None + } + }); + + // Try to find serial port prior to logging + if let Ok(dtb) = &dtb_res { + device::serial::init_early(dtb); } - }); - // Try to find serial port prior to logging - if let Ok(dtb) = &dtb_res { - device::serial::init_early(dtb); - } + // Initialize logger + crate::log::init_logger(|r| { + use core::fmt::Write; + let _ = write!( + crate::debug::Writer::new(), + "{}:{} -- {}\n", + r.target(), + r.level(), + r.args() + ); + }); + log::set_max_level(::log::LevelFilter::Debug); - // Initialize logger - crate::log::init_logger(|r| { - use core::fmt::Write; - let _ = write!( - crate::debug::Writer::new(), - "{}:{} -- {}\n", - r.target(), - r.level(), - r.args() + info!("Redox OS starting..."); + info!( + "Kernel: {:X}:{:X}", + { args.kernel_base }, + args.kernel_base + args.kernel_size + ); + info!( + "Stack: {:X}:{:X}", + { args.stack_base }, + args.stack_base + args.stack_size + ); + info!( + "Env: {:X}:{:X}", + { args.env_base }, + args.env_base + args.env_size + ); + info!( + "HWDESC: {:X}:{:X}", + { args.hwdesc_base }, + args.hwdesc_base + args.hwdesc_size + ); + info!( + "Areas: {:X}:{:X}", + { args.areas_base }, + args.areas_base + args.areas_size + ); + info!( + "Bootstrap: {:X}:{:X}", + { args.bootstrap_base }, + args.bootstrap_base + args.bootstrap_size ); - }); - log::set_max_level(::log::LevelFilter::Debug); - info!("Redox OS starting..."); - info!( - "Kernel: {:X}:{:X}", - { args.kernel_base }, - args.kernel_base + args.kernel_size - ); - info!( - "Stack: {:X}:{:X}", - { args.stack_base }, - args.stack_base + args.stack_size - ); - info!( - "Env: {:X}:{:X}", - { args.env_base }, - args.env_base + args.env_size - ); - info!( - "HWDESC: {:X}:{:X}", - { args.hwdesc_base }, - args.hwdesc_base + args.hwdesc_size - ); - info!( - "Areas: {:X}:{:X}", - { args.areas_base }, - args.areas_base + args.areas_size - ); - info!( - "Bootstrap: {:X}:{:X}", - { args.bootstrap_base }, - args.bootstrap_base + args.bootstrap_size - ); + interrupt::init(); - interrupt::init(); - - // Initialize RMM - register_bootloader_areas(args.areas_base, args.areas_size); - if let Ok(dtb) = &dtb_res { - register_dev_memory_ranges(dtb); - } - - register_memory_region( - args.kernel_base, - args.kernel_size, - BootloaderMemoryKind::Kernel, - ); - register_memory_region( - args.stack_base, - args.stack_size, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.env_base, - args.env_size, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.hwdesc_base, - args.hwdesc_size, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.bootstrap_base, - args.bootstrap_size, - BootloaderMemoryKind::IdentityMap, - ); - crate::startup::memory::init(None, None); - - // Initialize paging - paging::init(); - - crate::misc::init(crate::cpu_set::LogicalCpuId::new(0)); - - // Reset AP variables - CPU_COUNT.store(1, Ordering::SeqCst); - AP_READY.store(false, Ordering::SeqCst); - BSP_READY.store(false, Ordering::SeqCst); - - // Setup kernel heap - allocator::init(); - - // Set up double buffer for graphical debug now that heap is available - #[cfg(feature = "graphical_debug")] - graphical_debug::init_heap(); - - // Activate memory logging - crate::log::init(); - - // Initialize devices - match dtb_res { - Ok(dtb) => { - dtb::init(hwdesc_data.map(|slice| (slice.as_ptr() as usize, slice.len()))); - device::init_devicetree(&dtb); + // Initialize RMM + register_bootloader_areas(args.areas_base, args.areas_size); + if let Ok(dtb) = &dtb_res { + register_dev_memory_ranges(dtb); } - Err(err) => { - dtb::init(None); - log::warn!("failed to parse DTB: {}", err); - #[cfg(feature = "acpi")] - { - crate::acpi::init(rsdp_opt); + register_memory_region( + args.kernel_base, + args.kernel_size, + BootloaderMemoryKind::Kernel, + ); + register_memory_region( + args.stack_base, + args.stack_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.env_base, + args.env_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.hwdesc_base, + args.hwdesc_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.bootstrap_base, + args.bootstrap_size, + BootloaderMemoryKind::IdentityMap, + ); + crate::startup::memory::init(None, None); + + // Initialize paging + paging::init(); + + crate::misc::init(crate::cpu_set::LogicalCpuId::new(0)); + + // Reset AP variables + CPU_COUNT.store(1, Ordering::SeqCst); + AP_READY.store(false, Ordering::SeqCst); + BSP_READY.store(false, Ordering::SeqCst); + + // Setup kernel heap + allocator::init(); + + // Set up double buffer for graphical debug now that heap is available + #[cfg(feature = "graphical_debug")] + graphical_debug::init_heap(); + + // Activate memory logging + crate::log::init(); + + // Initialize devices + match dtb_res { + Ok(dtb) => { + dtb::init(hwdesc_data.map(|slice| (slice.as_ptr() as usize, slice.len()))); + device::init_devicetree(&dtb); + } + Err(err) => { + dtb::init(None); + log::warn!("failed to parse DTB: {}", err); + + #[cfg(feature = "acpi")] + { + crate::acpi::init(rsdp_opt); + } } } - } - BSP_READY.store(true, Ordering::SeqCst); + BSP_READY.store(true, Ordering::SeqCst); - crate::Bootstrap { - base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new( - args.bootstrap_base, - )), - page_count: args.bootstrap_size / crate::memory::PAGE_SIZE, - env, - } - }; + crate::Bootstrap { + base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new( + args.bootstrap_base, + )), + page_count: args.bootstrap_size / crate::memory::PAGE_SIZE, + env, + } + }; - crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap); + crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap); + } } #[repr(C, packed)] diff --git a/src/arch/aarch64/stop.rs b/src/arch/aarch64/stop.rs index 1734e3d3de..d477a0d4cb 100644 --- a/src/arch/aarch64/stop.rs +++ b/src/arch/aarch64/stop.rs @@ -1,26 +1,32 @@ use core::arch::asm; pub unsafe fn kreset() -> ! { - println!("kreset"); + unsafe { + println!("kreset"); - asm!("hvc #0", - in("x0") 0x8400_0009_usize, - options(noreturn), - ) + asm!("hvc #0", + in("x0") 0x8400_0009_usize, + options(noreturn), + ) + } } pub unsafe fn emergency_reset() -> ! { - asm!("hvc #0", - in("x0") 0x8400_0009_usize, - options(noreturn), - ) + unsafe { + asm!("hvc #0", + in("x0") 0x8400_0009_usize, + options(noreturn), + ) + } } pub unsafe fn kstop() -> ! { - println!("kstop"); + unsafe { + println!("kstop"); - asm!("hvc #0", - in("x0") 0x8400_0008_usize, - options(noreturn), - ) + asm!("hvc #0", + in("x0") 0x8400_0008_usize, + options(noreturn), + ) + } } diff --git a/src/arch/riscv64/device/irqchip/hlic.rs b/src/arch/riscv64/device/irqchip/hlic.rs index 68021065c9..dde60f982c 100644 --- a/src/arch/riscv64/device/irqchip/hlic.rs +++ b/src/arch/riscv64/device/irqchip/hlic.rs @@ -25,31 +25,39 @@ fn acknowledge(interrupt: usize) { } pub unsafe fn interrupt(hart: usize, interrupt: usize) { - assert!( - hart < CPU_INTERRUPT_HANDLERS.len(), - "Unexpected hart in interrupt routine" - ); - acknowledge(interrupt); - let ic_idx = CPU_INTERRUPT_HANDLERS[hart].unwrap_or_else(|| { - panic!( - "No hlic connected to hart {} yet interrupt {} occurred", - hart, interrupt - ) - }); - let virq = IRQ_CHIP - .irq_to_virq(ic_idx, interrupt as u32) - .unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt)); - if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler { - handler.irq_handler(virq as u32); - } else if let Some(ic_idx) = IRQ_CHIP.irq_desc[virq].basic.child_ic_idx { - IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_handler(virq as u32); - } else { - panic!( - "Unconnected interrupt {} occurred on hlic connected to hart {}", - interrupt, hart + unsafe { + assert!( + hart < CPU_INTERRUPT_HANDLERS.len(), + "Unexpected hart in interrupt routine" ); + acknowledge(interrupt); + let ic_idx = CPU_INTERRUPT_HANDLERS[hart].unwrap_or_else(|| { + panic!( + "No hlic connected to hart {} yet interrupt {} occurred", + hart, interrupt + ) + }); + let virq = IRQ_CHIP + .irq_to_virq(ic_idx, interrupt as u32) + .unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt)); + match &mut IRQ_CHIP.irq_desc[virq].handler { + Some(handler) => { + handler.irq_handler(virq as u32); + } + _ => match IRQ_CHIP.irq_desc[virq].basic.child_ic_idx { + Some(ic_idx) => { + IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_handler(virq as u32); + } + _ => { + panic!( + "Unconnected interrupt {} occurred on hlic connected to hart {}", + interrupt, hart + ); + } + }, + } } } diff --git a/src/arch/riscv64/device/mod.rs b/src/arch/riscv64/device/mod.rs index de2fa976d8..bceacae144 100644 --- a/src/arch/riscv64/device/mod.rs +++ b/src/arch/riscv64/device/mod.rs @@ -69,14 +69,16 @@ unsafe fn init_intc(cpu: &FdtNode) { } pub unsafe fn init() { - let data = DTB_BINARY.get().unwrap(); - let fdt = Fdt::new(data).unwrap(); + unsafe { + let data = DTB_BINARY.get().unwrap(); + let fdt = Fdt::new(data).unwrap(); - crate::dtb::irqchip::init(&fdt); + crate::dtb::irqchip::init(&fdt); - let cpu = fdt.find_node(format!("/cpus/cpu@{}", 0).as_str()).unwrap(); - init_intc(&cpu); - init_time(&fdt); + let cpu = fdt.find_node(format!("/cpus/cpu@{}", 0).as_str()).unwrap(); + init_intc(&cpu); + init_time(&fdt); + } } fn init_time(fdt: &Fdt) { @@ -90,11 +92,13 @@ fn init_time(fdt: &Fdt) { } pub unsafe fn init_noncore() { - let data = DTB_BINARY.get().unwrap(); - let fdt = Fdt::new(data).unwrap(); + unsafe { + let data = DTB_BINARY.get().unwrap(); + let fdt = Fdt::new(data).unwrap(); - init_clint(&fdt); - serial::init(&fdt); + init_clint(&fdt); + serial::init(&fdt); + } } #[derive(Default)] diff --git a/src/arch/riscv64/device/serial.rs b/src/arch/riscv64/device/serial.rs index 5fc366d12e..53aef975af 100644 --- a/src/arch/riscv64/device/serial.rs +++ b/src/arch/riscv64/device/serial.rs @@ -65,65 +65,68 @@ impl InterruptHandler for Com1Irq { } pub unsafe fn init_early(dtb: &Fdt) { - if COM1.lock().is_some() { - // Hardcoded UART - return; - } + unsafe { + if COM1.lock().is_some() { + // Hardcoded UART + return; + } - if let Some((phys, size, skip_init, _cts, compatible)) = diag_uart_range(dtb) { - let virt = crate::PHYS_OFFSET + phys; - let serial_opt = - if compatible.contains("ns16550a") { - //TODO: get actual register size from device tree - let serial_port = uart_16550::SerialPort::>::new(virt); - if !skip_init { - serial_port.init(); - } - Some(SerialPort::Ns16550u8(serial_port)) - } else if compatible.contains("snps,dw-apb-uart") { - //TODO: get actual register size from device tree - let serial_port = uart_16550::SerialPort::>::new(virt); - if !skip_init { - serial_port.init(); - } - Some(SerialPort::Ns16550u32(serial_port)) - } else { - None - }; - match serial_opt { - Some(serial) => { - *COM1.lock() = Some(serial); - info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size); - } - None => { - log::warn!( - "UART {:?} at {:#X} size {:#X}: no driver found", - compatible, - virt, - size - ); + if let Some((phys, size, skip_init, _cts, compatible)) = diag_uart_range(dtb) { + let virt = crate::PHYS_OFFSET + phys; + let serial_opt = if compatible.contains("ns16550a") { + //TODO: get actual register size from device tree + let serial_port = uart_16550::SerialPort::>::new(virt); + if !skip_init { + serial_port.init(); + } + Some(SerialPort::Ns16550u8(serial_port)) + } else if compatible.contains("snps,dw-apb-uart") { + //TODO: get actual register size from device tree + let serial_port = uart_16550::SerialPort::>::new(virt); + if !skip_init { + serial_port.init(); + } + Some(SerialPort::Ns16550u32(serial_port)) + } else { + None + }; + match serial_opt { + Some(serial) => { + *COM1.lock() = Some(serial); + info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size); + } + None => { + log::warn!( + "UART {:?} at {:#X} size {:#X}: no driver found", + compatible, + virt, + size + ); + } } } } } pub unsafe fn init(fdt: &Fdt) -> Option<()> { - if let Some(node) = fdt.find_compatible(&["ns16550a", "snps,dw-apb-uart"]) { - let intr = get_interrupt(fdt, &node, 0).unwrap(); - let interrupt_parent = interrupt_parent(fdt, &node)?; - let phandle = interrupt_parent.property("phandle")?.as_usize()? as u32; - let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle)?; + unsafe { + if let Some(node) = fdt.find_compatible(&["ns16550a", "snps,dw-apb-uart"]) { + let intr = get_interrupt(fdt, &node, 0).unwrap(); + let interrupt_parent = interrupt_parent(fdt, &node)?; + let phandle = interrupt_parent.property("phandle")?.as_usize()? as u32; + let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle)?; - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_xlate(intr) - .unwrap(); - info!("serial_port virq = {}", virq); - register_irq(virq as u32, Box::new(Com1Irq {})); - IRQ_CHIP.irq_enable(virq as u32); + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(intr) + .unwrap(); + info!("serial_port virq = {}", virq); + register_irq(virq as u32, Box::new(Com1Irq {})); + IRQ_CHIP.irq_enable(virq as u32); + } + if let Some(ref mut _serial_port) = *COM1.lock() { + // serial_port.enable_irq(); // FIXME receive int is enabled by default in 16550. Disable by default? + } + Some(()) } - if let Some(ref mut _serial_port) = *COM1.lock() { - // serial_port.enable_irq(); // FIXME receive int is enabled by default in 16550. Disable by default? - } - Some(()) } diff --git a/src/arch/riscv64/interrupt/exception.rs b/src/arch/riscv64/interrupt/exception.rs index 56495197ad..3d8fe1836a 100644 --- a/src/arch/riscv64/interrupt/exception.rs +++ b/src/arch/riscv64/interrupt/exception.rs @@ -86,129 +86,142 @@ global_asm!(concat!( ); unsafe fn exception_handler_inner(regs: &mut InterruptStack) { - let scause: usize; - let sstatus: usize; - core::arch::asm!( - "csrr t0, scause", - "csrr t1, sstatus", - lateout("t0") scause, - lateout("t1") sstatus, - options(nostack) - ); + unsafe { + let scause: usize; + let sstatus: usize; + core::arch::asm!( + "csrr t0, scause", + "csrr t1, sstatus", + lateout("t0") scause, + lateout("t1") sstatus, + options(nostack) + ); - //log::info!("Exception handler incoming: sepc={:x} scause={:x} sstatus={:x}", regs.iret.sepc, scause, sstatus); + //log::info!("Exception handler incoming: sepc={:x} scause={:x} sstatus={:x}", regs.iret.sepc, scause, sstatus); - let user_mode = sstatus & (1 << 8) == 0; + let user_mode = sstatus & (1 << 8) == 0; - if (scause as isize) < 0 { - handle_interrupt(scause & 0xF); - } else if page_fault(scause, regs, user_mode) { - } else if user_mode { - handle_user_exception(scause, regs); - } else { - handle_system_exception(scause, regs); + if (scause as isize) < 0 { + handle_interrupt(scause & 0xF); + } else if page_fault(scause, regs, user_mode) { + } else if user_mode { + handle_user_exception(scause, regs); + } else { + handle_system_exception(scause, regs); + } + //log::info!("Exception handler outgoing"); } - //log::info!("Exception handler outgoing"); } unsafe fn handle_system_exception(scause: usize, regs: &InterruptStack) { - let stval: usize; - let tp: usize; - core::arch::asm!( - "csrr t0, stval", - "mv t1, tp", - lateout("t0") stval, - lateout("t1") tp, - options(nostack) - ); + unsafe { + let stval: usize; + let tp: usize; + core::arch::asm!( + "csrr t0, stval", + "mv t1, tp", + lateout("t0") stval, + lateout("t1") tp, + options(nostack) + ); - error!( - "S-mode exception! scause={:#016x}, stval={:#016x}", - scause, stval - ); - regs.dump(); + error!( + "S-mode exception! scause={:#016x}, stval={:#016x}", + scause, stval + ); + regs.dump(); - if tp == 0 { - // Early failure - before misc::init and potentially before RMM init - // Do not attempt to trace stack because it would probably trap again + if tp == 0 { + // Early failure - before misc::init and potentially before RMM init + // Do not attempt to trace stack because it would probably trap again + loop {} + } + + stack_trace(); loop {} } - - stack_trace(); - loop {} } unsafe fn handle_interrupt(interrupt: usize) { - // FIXME retrieve from percpu area - // For now all the interrupts go to boot hart so this suffices... - let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed); - irqchip::hlic::interrupt(hart, interrupt); + unsafe { + // FIXME retrieve from percpu area + // For now all the interrupts go to boot hart so this suffices... + let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed); + irqchip::hlic::interrupt(hart, interrupt); + } } unsafe fn handle_user_exception(scause: usize, regs: &mut InterruptStack) { - if scause == USERMODE_ECALL { - let r = &mut regs.registers; - regs.iret.sepc += 4; // skip ecall - let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14); - r.x10 = ret; - return; - } - - if scause == BREAKPOINT { - if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_some() { + unsafe { + if scause == USERMODE_ECALL { + let r = &mut regs.registers; + regs.iret.sepc += 4; // skip ecall + let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14); + r.x10 = ret; return; } - } - let stval: usize; - core::arch::asm!( - "csrr t0, stval", - lateout("t0") stval, - options(nostack) - ); + if scause == BREAKPOINT { + if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_some() { + return; + } + } - info!( - "U-mode exception! scause={:#016x}, stval={:#016x}", - scause, stval - ); - regs.dump(); - - // TODO - /* - let signal = match scause { - 0 | 4 | 6 | 18 | 19 => SIGBUS, // misaligned / machine check - 2 | 8 | 9 => SIGILL, // Illegal instruction / breakpoint / ecall - BREAKPOINT => SIGTRAP, - _ => SIGSEGV, - }; - */ - excp_handler(Exception { kind: scause }); -} - -unsafe fn page_fault(scause: usize, regs: &mut InterruptStack, user_mode: bool) -> bool { - if scause != INSTRUCTION_PAGE_FAULT && scause != LOAD_PAGE_FAULT && scause != STORE_PAGE_FAULT { - return false; - } - - let stval: usize; - core::arch::asm!( + let stval: usize; + core::arch::asm!( "csrr t0, stval", lateout("t0") stval, options(nostack) - ); + ); - let address = VirtualAddress::new(stval); - let mut generic_flags = GenericPfFlags::empty(); + info!( + "U-mode exception! scause={:#016x}, stval={:#016x}", + scause, stval + ); + regs.dump(); - generic_flags.set(GenericPfFlags::INVOLVED_WRITE, scause == STORE_PAGE_FAULT); - generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, user_mode); - generic_flags.set( - GenericPfFlags::INSTR_NOT_DATA, - scause == INSTRUCTION_PAGE_FAULT, - ); - // FIXME can these conditions be distinguished? Should they be? - generic_flags.set(GenericPfFlags::INVL, false); - generic_flags.set(GenericPfFlags::PRESENT, false); - - crate::memory::page_fault_handler(regs, generic_flags, address).is_ok() + // TODO + /* + let signal = match scause { + 0 | 4 | 6 | 18 | 19 => SIGBUS, // misaligned / machine check + 2 | 8 | 9 => SIGILL, // Illegal instruction / breakpoint / ecall + BREAKPOINT => SIGTRAP, + _ => SIGSEGV, + }; + */ + excp_handler(Exception { kind: scause }); + } +} + +unsafe fn page_fault(scause: usize, regs: &mut InterruptStack, user_mode: bool) -> bool { + unsafe { + if scause != INSTRUCTION_PAGE_FAULT + && scause != LOAD_PAGE_FAULT + && scause != STORE_PAGE_FAULT + { + return false; + } + + let stval: usize; + core::arch::asm!( + "csrr t0, stval", + lateout("t0") stval, + options(nostack) + ); + + let address = VirtualAddress::new(stval); + let mut generic_flags = GenericPfFlags::empty(); + + generic_flags.set(GenericPfFlags::INVOLVED_WRITE, scause == STORE_PAGE_FAULT); + generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, user_mode); + generic_flags.set( + GenericPfFlags::INSTR_NOT_DATA, + scause == INSTRUCTION_PAGE_FAULT, + ); + // FIXME can these conditions be distinguished? Should they be? + generic_flags.set(GenericPfFlags::INVL, false); + generic_flags.set(GenericPfFlags::PRESENT, false); + + crate::memory::page_fault_handler(regs, generic_flags, address).is_ok() + } } diff --git a/src/arch/riscv64/interrupt/handler.rs b/src/arch/riscv64/interrupt/handler.rs index e7c2914325..4d3b25f70e 100644 --- a/src/arch/riscv64/interrupt/handler.rs +++ b/src/arch/riscv64/interrupt/handler.rs @@ -302,15 +302,17 @@ macro_rules! pop_registers { #[naked] pub unsafe extern "C" fn enter_usermode() -> ! { - core::arch::naked_asm!(concat!( - "jalr s11\n", - "li t0, 1 << 8\n", // force U mode on sret - "csrc sstatus, t0\n", - "li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode) - "csrs sstatus, t0\n", - "addi t0, sp, 32 * 8\n", // save S mode stack to percpu - "sd t0, 8(tp)\n", - pop_registers!(), - "sret\n", - )) + unsafe { + core::arch::naked_asm!(concat!( + "jalr s11\n", + "li t0, 1 << 8\n", // force U mode on sret + "csrc sstatus, t0\n", + "li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode) + "csrs sstatus, t0\n", + "addi t0, sp, 32 * 8\n", // save S mode stack to percpu + "sd t0, 8(tp)\n", + pop_registers!(), + "sret\n", + )) + } } diff --git a/src/arch/riscv64/interrupt/mod.rs b/src/arch/riscv64/interrupt/mod.rs index a838690900..9b191cbbab 100644 --- a/src/arch/riscv64/interrupt/mod.rs +++ b/src/arch/riscv64/interrupt/mod.rs @@ -12,13 +12,13 @@ pub use handler::InterruptStack; /// Clear interrupts #[inline(always)] pub unsafe fn disable() { - asm!("csrci sstatus, 1 << 1") + unsafe { asm!("csrci sstatus, 1 << 1") } } /// Set interrupts #[inline(always)] pub unsafe fn enable() { - asm!("csrsi sstatus, 1 << 1") + unsafe { asm!("csrsi sstatus, 1 << 1") } } /// Set interrupts and halt @@ -26,7 +26,7 @@ pub unsafe fn enable() { /// Performing enable followed by halt is not guaranteed to be atomic, use this instead! #[inline(always)] pub unsafe fn enable_and_halt() { - asm!("csrsi sstatus, 1 << 1", "wfi") + unsafe { asm!("csrsi sstatus, 1 << 1", "wfi") } } /// Set interrupts and nop @@ -34,13 +34,13 @@ pub unsafe fn enable_and_halt() { /// Simply enabling interrupts does not gurantee that they will trigger, use this instead! #[inline(always)] pub unsafe fn enable_and_nop() { - asm!("csrsi sstatus, 1 << 1", "nop") + unsafe { asm!("csrsi sstatus, 1 << 1", "nop") } } /// Halt instruction #[inline(always)] pub unsafe fn halt() { - asm!("wfi", options(nomem, nostack)) + unsafe { asm!("wfi", options(nomem, nostack)) } } /// Pause instruction @@ -55,9 +55,11 @@ pub fn pause() { #[inline(always)] pub unsafe fn init() { - // Setup interrupt handlers - asm!( - "la t0, exception_handler", // WARL=0 - direct mode combined handler - "csrw stvec, t0" - ); + unsafe { + // Setup interrupt handlers + asm!( + "la t0, exception_handler", // WARL=0 - direct mode combined handler + "csrw stvec, t0" + ); + } } diff --git a/src/arch/riscv64/interrupt/trace.rs b/src/arch/riscv64/interrupt/trace.rs index 88d754a2b9..bf428c1ca0 100644 --- a/src/arch/riscv64/interrupt/trace.rs +++ b/src/arch/riscv64/interrupt/trace.rs @@ -8,24 +8,28 @@ pub struct StackTrace { impl StackTrace { #[inline(always)] pub unsafe fn start() -> Option { - let fp: usize; - asm!("mv {}, fp", out(reg) fp); + unsafe { + let fp: usize; + asm!("mv {}, fp", out(reg) fp); - let pc_ptr = fp.checked_sub(mem::size_of::())?; - let fp = pc_ptr.checked_sub(mem::size_of::())?; - Some(StackTrace { - fp, - pc_ptr: pc_ptr as *const usize, - }) + let pc_ptr = fp.checked_sub(mem::size_of::())?; + let fp = pc_ptr.checked_sub(mem::size_of::())?; + Some(StackTrace { + fp, + pc_ptr: pc_ptr as *const usize, + }) + } } pub unsafe fn next(self) -> Option { - let fp = *(self.fp as *const usize); - let pc_ptr = fp.checked_sub(mem::size_of::())?; - let fp = pc_ptr.checked_sub(mem::size_of::())?; - Some(StackTrace { - fp: fp, - pc_ptr: pc_ptr as *const usize, - }) + unsafe { + let fp = *(self.fp as *const usize); + let pc_ptr = fp.checked_sub(mem::size_of::())?; + let fp = pc_ptr.checked_sub(mem::size_of::())?; + Some(StackTrace { + fp: fp, + pc_ptr: pc_ptr as *const usize, + }) + } } } diff --git a/src/arch/riscv64/macros.rs b/src/arch/riscv64/macros.rs index 4e3566fc55..a9c83081f0 100644 --- a/src/arch/riscv64/macros.rs +++ b/src/arch/riscv64/macros.rs @@ -11,6 +11,6 @@ macro_rules! print { #[macro_export] macro_rules! println { () => (print!("\n")); - ($fmt:expr) => (print!(concat!($fmt, "\n"))); - ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); + ($fmt:expr_2021) => (print!(concat!($fmt, "\n"))); + ($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } diff --git a/src/arch/riscv64/misc.rs b/src/arch/riscv64/misc.rs index 60bde118ac..cee4c23c58 100644 --- a/src/arch/riscv64/misc.rs +++ b/src/arch/riscv64/misc.rs @@ -28,18 +28,20 @@ impl PercpuBlock { #[cold] pub unsafe fn init(cpu_id: LogicalCpuId) { - let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory"); - let virt = RmmA::phys_to_virt(frame.base()).data() as *mut ArchPercpu; + unsafe { + let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory"); + let virt = RmmA::phys_to_virt(frame.base()).data() as *mut ArchPercpu; - virt.write(ArchPercpu { - tmp: 0, - s_sp: 0, - percpu: PercpuBlock::init(cpu_id), - }); + virt.write(ArchPercpu { + tmp: 0, + s_sp: 0, + percpu: PercpuBlock::init(cpu_id), + }); - asm!( - "mv tp, {}", - "csrw sscratch, tp", - in(reg) virt as usize - ); + asm!( + "mv tp, {}", + "csrw sscratch, tp", + in(reg) virt as usize + ); + } } diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs index 62d1afbd3b..3ad2bcb2ce 100644 --- a/src/arch/riscv64/mod.rs +++ b/src/arch/riscv64/mod.rs @@ -18,11 +18,12 @@ use core::arch::naked_asm; pub use arch_copy_to_user as arch_copy_from_user; -#[link_section = ".usercopy-fns"] +#[unsafe(link_section = ".usercopy-fns")] #[naked] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - naked_asm!( - " + unsafe { + naked_asm!( + " addi sp, sp, -16 sd fp, 0(sp) sd ra, 8(sp) @@ -59,7 +60,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - 5: mv a0, x0 ret " - ) + ) + } } pub const KFX_SIZE: usize = 1024; diff --git a/src/arch/riscv64/start.rs b/src/arch/riscv64/start.rs index e29ac32cfc..bc02002de8 100644 --- a/src/arch/riscv64/start.rs +++ b/src/arch/riscv64/start.rs @@ -65,168 +65,171 @@ fn get_boot_hart_id(env: &[u8]) -> Option { } /// The entry to Rust, all things must be initialized -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { - asm!( - "mv tp, x0", // reset percpu until it is initialized - "csrw sscratch, tp", - "sd x0, -16(fp)", // and stop frame walker here - "sd x0, -8(fp)", - ); - - let bootstrap = { - let args = &*args_ptr; - - // BSS should already be zero - { - assert_eq!(BSS_TEST_ZERO, 0); - assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF); - } - - KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst); - KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst); - - let env = slice::from_raw_parts( - (crate::PHYS_OFFSET + args.env_base) as *const u8, - args.env_size, + unsafe { + asm!( + "mv tp, x0", // reset percpu until it is initialized + "csrw sscratch, tp", + "sd x0, -16(fp)", // and stop frame walker here + "sd x0, -8(fp)", ); - let dtb_data = if args.acpi_base != 0 { - Some((crate::PHYS_OFFSET + args.acpi_base, args.acpi_size)) - } else { - None - }; - let dtb = dtb_data - .map(|(base, size)| unsafe { slice::from_raw_parts(base as *const u8, size) }) - .and_then(|data| Fdt::new(data).ok()); + let bootstrap = { + let args = &*args_ptr; - #[cfg(feature = "graphical_debug")] - graphical_debug::init(env); + // BSS should already be zero + { + assert_eq!(BSS_TEST_ZERO, 0); + assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF); + } - #[cfg(feature = "serial_debug")] - if let Some(dtb) = &dtb { - init_early(dtb); - } + KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst); + KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst); - // Initialize logger - crate::log::init_logger(|r| { - use core::fmt::Write; - let _ = write!( - crate::debug::Writer::new(), - "{}:{} -- {}\n", - r.target(), - r.level(), - r.args() + let env = slice::from_raw_parts( + (crate::PHYS_OFFSET + args.env_base) as *const u8, + args.env_size, ); - }); - ::log::set_max_level(::log::LevelFilter::Debug); - info!("Redox OS starting..."); - info!( - "Kernel: {:X}:{:X}", - { args.kernel_base }, - args.kernel_base + args.kernel_size - ); - info!( - "Stack: {:X}:{:X}", - { args.stack_base }, - args.stack_base + args.stack_size - ); - info!( - "Env: {:X}:{:X}", - { args.env_base }, - args.env_base + args.env_size - ); - info!( - "RSDPs: {:X}:{:X}", - { args.acpi_size }, - args.acpi_size + args.acpi_size - ); - info!( - "Areas: {:X}:{:X}", - { args.areas_base }, - args.areas_base + args.areas_size - ); - info!( - "Bootstrap: {:X}:{:X}", - { args.bootstrap_base }, - args.bootstrap_base + args.bootstrap_size - ); + let dtb_data = if args.acpi_base != 0 { + Some((crate::PHYS_OFFSET + args.acpi_base, args.acpi_size)) + } else { + None + }; + let dtb = dtb_data + .map(|(base, size)| unsafe { slice::from_raw_parts(base as *const u8, size) }) + .and_then(|data| Fdt::new(data).ok()); - if let Some(dtb) = &dtb { - device::dump_fdt(&dtb); - } + #[cfg(feature = "graphical_debug")] + graphical_debug::init(env); - interrupt::init(); + #[cfg(feature = "serial_debug")] + if let Some(dtb) = &dtb { + init_early(dtb); + } - let bootstrap = crate::Bootstrap { - base: Frame::containing(PhysicalAddress::new(args.bootstrap_base)), - page_count: args.bootstrap_size / PAGE_SIZE, - env, + // Initialize logger + crate::log::init_logger(|r| { + use core::fmt::Write; + let _ = write!( + crate::debug::Writer::new(), + "{}:{} -- {}\n", + r.target(), + r.level(), + r.args() + ); + }); + ::log::set_max_level(::log::LevelFilter::Debug); + + info!("Redox OS starting..."); + info!( + "Kernel: {:X}:{:X}", + { args.kernel_base }, + args.kernel_base + args.kernel_size + ); + info!( + "Stack: {:X}:{:X}", + { args.stack_base }, + args.stack_base + args.stack_size + ); + info!( + "Env: {:X}:{:X}", + { args.env_base }, + args.env_base + args.env_size + ); + info!( + "RSDPs: {:X}:{:X}", + { args.acpi_size }, + args.acpi_size + args.acpi_size + ); + info!( + "Areas: {:X}:{:X}", + { args.areas_base }, + args.areas_base + args.areas_size + ); + info!( + "Bootstrap: {:X}:{:X}", + { args.bootstrap_base }, + args.bootstrap_base + args.bootstrap_size + ); + + if let Some(dtb) = &dtb { + device::dump_fdt(&dtb); + } + + interrupt::init(); + + let bootstrap = crate::Bootstrap { + base: Frame::containing(PhysicalAddress::new(args.bootstrap_base)), + page_count: args.bootstrap_size / PAGE_SIZE, + env, + }; + + // Initialize RMM + register_bootloader_areas(args.areas_base, args.areas_size); + if let Some(dt) = &dtb { + register_dev_memory_ranges(dt); + } + + register_memory_region( + args.kernel_base, + args.kernel_size, + BootloaderMemoryKind::Kernel, + ); + register_memory_region( + args.stack_base, + args.stack_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.env_base, + args.env_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.acpi_base, + args.acpi_size, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.bootstrap_base, + args.bootstrap_size, + BootloaderMemoryKind::IdentityMap, + ); + + crate::startup::memory::init(None, None); + + let boot_hart_id = + get_boot_hart_id(env).expect("Didn't get boot HART id from bootloader"); + info!("Booting on HART {}", boot_hart_id); + BOOT_HART_ID.store(boot_hart_id, Ordering::Relaxed); + + paging::init(); + + crate::misc::init(crate::cpu_set::LogicalCpuId::new(0)); + + CPU_COUNT.store(1, Ordering::SeqCst); + + // Setup kernel heap + allocator::init(); + + // Activate memory logging + crate::log::init(); + + crate::dtb::init(dtb_data); + + // Initialize devices + device::init(); + + // Initialize all of the non-core devices not otherwise needed to complete initialization + device::init_noncore(); + + // FIXME bringup AP HARTs + + bootstrap }; - // Initialize RMM - register_bootloader_areas(args.areas_base, args.areas_size); - if let Some(dt) = &dtb { - register_dev_memory_ranges(dt); - } - - register_memory_region( - args.kernel_base, - args.kernel_size, - BootloaderMemoryKind::Kernel, - ); - register_memory_region( - args.stack_base, - args.stack_size, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.env_base, - args.env_size, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.acpi_base, - args.acpi_size, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.bootstrap_base, - args.bootstrap_size, - BootloaderMemoryKind::IdentityMap, - ); - - crate::startup::memory::init(None, None); - - let boot_hart_id = get_boot_hart_id(env).expect("Didn't get boot HART id from bootloader"); - info!("Booting on HART {}", boot_hart_id); - BOOT_HART_ID.store(boot_hart_id, Ordering::Relaxed); - - paging::init(); - - crate::misc::init(crate::cpu_set::LogicalCpuId::new(0)); - - CPU_COUNT.store(1, Ordering::SeqCst); - - // Setup kernel heap - allocator::init(); - - // Activate memory logging - crate::log::init(); - - crate::dtb::init(dtb_data); - - // Initialize devices - device::init(); - - // Initialize all of the non-core devices not otherwise needed to complete initialization - device::init_noncore(); - - // FIXME bringup AP HARTs - - bootstrap - }; - - crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap); + crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap); + } } diff --git a/src/arch/x86/gdt.rs b/src/arch/x86/gdt.rs index 8aa9722d25..7f614cc16e 100644 --- a/src/arch/x86/gdt.rs +++ b/src/arch/x86/gdt.rs @@ -135,9 +135,11 @@ pub struct ProcessorControlRegion { pub struct TssWrapper(pub TaskStateSegment); pub unsafe fn pcr() -> *mut ProcessorControlRegion { - let mut ret: *mut ProcessorControlRegion; - core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref))); - ret + unsafe { + let mut ret: *mut ProcessorControlRegion; + core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref))); + ret + } } #[cfg(feature = "pti")] @@ -151,84 +153,93 @@ pub unsafe fn set_tss_stack(stack: usize) { #[cfg(not(feature = "pti"))] pub unsafe fn set_tss_stack(stack: usize) { - addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16); - addr_of_mut!((*pcr()).tss.0.esp0).write(stack as u32); + unsafe { + addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16); + addr_of_mut!((*pcr()).tss.0.esp0).write(stack as u32); + } } pub unsafe fn set_userspace_io_allowed(allowed: bool) { - addr_of_mut!((*pcr()).tss.0.iobp_offset).write(if allowed { - mem::size_of::() as u16 - } else { - 0xFFFF - }); + unsafe { + addr_of_mut!((*pcr()).tss.0.iobp_offset).write(if allowed { + mem::size_of::() as u16 + } else { + 0xFFFF + }); + } } /// Initialize a minimal GDT without configuring percpu. pub unsafe fn init() { - // Load the initial GDT, before the kernel remaps itself. - dtables::lgdt(&DescriptorTablePointer { - limit: (INIT_GDT.len() * mem::size_of::() - 1) as u16, - base: INIT_GDT.as_ptr() as *const SegmentDescriptor, - }); + unsafe { + // Load the initial GDT, before the kernel remaps itself. + dtables::lgdt(&DescriptorTablePointer { + limit: (INIT_GDT.len() * mem::size_of::() - 1) as u16, + base: INIT_GDT.as_ptr() as *const SegmentDescriptor, + }); - // Load the segment descriptors - segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); - segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + // Load the segment descriptors + segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); + segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + } } /// Initialize GDT and configure percpu. pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { - let alloc_order = mem::size_of::() - .div_ceil(PAGE_SIZE) - .next_power_of_two() - .trailing_zeros(); - let pcr_frame = - crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR frame"); - let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion); + unsafe { + let alloc_order = mem::size_of::() + .div_ceil(PAGE_SIZE) + .next_power_of_two() + .trailing_zeros(); + let pcr_frame = + crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR frame"); + let pcr = + &mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion); - pcr.self_ref = pcr as *const _ as usize; - pcr.gdt = BASE_GDT; - pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32); + pcr.self_ref = pcr as *const _ as usize; + pcr.gdt = BASE_GDT; + pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32); - let gdtr: DescriptorTablePointer = DescriptorTablePointer { - limit: (pcr.gdt.len() * mem::size_of::() - 1) as u16, - base: pcr.gdt.as_ptr() as *const SegmentDescriptor, - }; + let gdtr: DescriptorTablePointer = DescriptorTablePointer { + limit: (pcr.gdt.len() * mem::size_of::() - 1) as u16, + base: pcr.gdt.as_ptr() as *const SegmentDescriptor, + }; - { - pcr._all_ones = 0xFF; - pcr.tss.0.iobp_offset = 0xFFFF; - let tss = &pcr.tss.0 as *const _ as usize as u32; + { + pcr._all_ones = 0xFF; + pcr.tss.0.iobp_offset = 0xFFFF; + let tss = &pcr.tss.0 as *const _ as usize as u32; - pcr.gdt[GDT_TSS].set_offset(tss); - pcr.gdt[GDT_TSS] - .set_limit(mem::size_of::() as u32 + IOBITMAP_SIZE as u32); + pcr.gdt[GDT_TSS].set_offset(tss); + pcr.gdt[GDT_TSS] + .set_limit(mem::size_of::() as u32 + IOBITMAP_SIZE as u32); + } + + // Load the new GDT, which is correctly located in thread local storage. + dtables::lgdt(&gdtr); + + // Reload the segment descriptors + segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); + segmentation::load_ds(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3)); + segmentation::load_es(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3)); + segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + + // TODO: Use FS for kernel percpu on i686? + segmentation::load_fs(SegmentSelector::new(GDT_USER_FS as u16, Ring::Ring0)); + segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0)); + + // Set the stack pointer to use when coming back from userspace. + set_tss_stack(stack_offset); + + // Load the task register + task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); + + pcr.percpu = crate::percpu::PercpuBlock::init(cpu_id); + crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu); } - - // Load the new GDT, which is correctly located in thread local storage. - dtables::lgdt(&gdtr); - - // Reload the segment descriptors - segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); - segmentation::load_ds(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3)); - segmentation::load_es(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3)); - segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - - // TODO: Use FS for kernel percpu on i686? - segmentation::load_fs(SegmentSelector::new(GDT_USER_FS as u16, Ring::Ring0)); - segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0)); - - // Set the stack pointer to use when coming back from userspace. - set_tss_stack(stack_offset); - - // Load the task register - task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); - - pcr.percpu = crate::percpu::PercpuBlock::init(cpu_id); - crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu); } #[derive(Copy, Clone, Debug)] diff --git a/src/arch/x86/interrupt/handler.rs b/src/arch/x86/interrupt/handler.rs index 973298e7cf..6083dd7767 100644 --- a/src/arch/x86/interrupt/handler.rs +++ b/src/arch/x86/interrupt/handler.rs @@ -267,7 +267,7 @@ macro_rules! interrupt_stack { // use idents directly instead. ($name:ident, |$stack:ident| $code:block) => { #[naked] - pub unsafe extern "C" fn $name() { + pub unsafe extern "C" fn $name() { unsafe { unsafe extern "fastcall" fn inner($stack: &mut $crate::arch::x86::interrupt::InterruptStack) { // TODO: Force the declarations to specify unsafe? @@ -308,7 +308,7 @@ macro_rules! interrupt_stack { ), inner = sym inner, ); - } + }} }; ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, |$stack| $code); }; ($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, |$stack| $code); } @@ -318,10 +318,10 @@ macro_rules! interrupt_stack { macro_rules! interrupt { ($name:ident, || $code:block) => { #[naked] - pub unsafe extern "C" fn $name() { - unsafe extern "C" fn inner() { + pub unsafe extern "C" fn $name() { unsafe { + unsafe extern "C" fn inner() { unsafe { $code - } + }} core::arch::naked_asm!(concat!( // Backup all userspace registers to stack @@ -350,7 +350,7 @@ macro_rules! interrupt { ), inner = sym inner, ); - } + }} }; } @@ -358,7 +358,7 @@ macro_rules! interrupt { macro_rules! interrupt_error { ($name:ident, |$stack:ident| $code:block) => { #[naked] - pub unsafe extern "C" fn $name() { + pub unsafe extern "C" fn $name() { unsafe { unsafe extern "C" fn inner($stack: &mut $crate::arch::x86::interrupt::handler::InterruptErrorStack) { #[allow(unused_unsafe)] unsafe { @@ -408,20 +408,22 @@ macro_rules! interrupt_error { "iretd\n", ), inner = sym inner); - } + }} }; } #[naked] unsafe extern "C" fn usercopy_trampoline() { - core::arch::naked_asm!( - " + unsafe { + core::arch::naked_asm!( + " pop esi pop edi mov eax, 1 ret " - ); + ); + } } impl ArchIntCtx for InterruptStack { @@ -443,15 +445,17 @@ impl ArchIntCtx for InterruptStack { #[naked] pub unsafe extern "C" fn enter_usermode() { - core::arch::naked_asm!(concat!( - // TODO: Unmap PTI - // $crate::arch::x86::pti::unmap(); + unsafe { + core::arch::naked_asm!(concat!( + // TODO: Unmap PTI + // $crate::arch::x86::pti::unmap(); - // Exit kernel TLS segment - exit_gs!(), - // Restore all userspace registers - pop_preserved!(), - pop_scratch!(), - "iretd\n", - )) + // Exit kernel TLS segment + exit_gs!(), + // Restore all userspace registers + pop_preserved!(), + pop_scratch!(), + "iretd\n", + )) + } } diff --git a/src/arch/x86/interrupt/irq.rs b/src/arch/x86/interrupt/irq.rs index c6520814a8..0b6da58c8c 100644 --- a/src/arch/x86/interrupt/irq.rs +++ b/src/arch/x86/interrupt/irq.rs @@ -71,85 +71,101 @@ fn irq_method() -> IrqMethod { /// Notify the IRQ scheme that an IRQ has been registered. This should mask the IRQ until the /// scheme user unmasks it ("acknowledges" it). unsafe fn trigger(irq: u8) { - match irq_method() { - IrqMethod::Pic => { - if irq < 16 { - pic_mask(irq) + unsafe { + match irq_method() { + IrqMethod::Pic => { + if irq < 16 { + pic_mask(irq) + } } + IrqMethod::Apic => ioapic_mask(irq), } - IrqMethod::Apic => ioapic_mask(irq), + irq_trigger(irq); } - irq_trigger(irq); } /// Unmask the IRQ. This is called from the IRQ scheme, which does this when a user process has /// processed the IRQ. pub unsafe fn acknowledge(irq: usize) { - match irq_method() { - IrqMethod::Pic => { - if irq < 16 { - pic_unmask(irq) + unsafe { + match irq_method() { + IrqMethod::Pic => { + if irq < 16 { + pic_unmask(irq) + } } + IrqMethod::Apic => ioapic_unmask(irq), } - IrqMethod::Apic => ioapic_unmask(irq), } } /// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one. pub unsafe fn eoi(irq: u8) { - PercpuBlock::current().stats.add_irq(irq); + unsafe { + PercpuBlock::current().stats.add_irq(irq); - match irq_method() { - IrqMethod::Pic => { - if irq < 16 { - pic_eoi(irq) + match irq_method() { + IrqMethod::Pic => { + if irq < 16 { + pic_eoi(irq) + } } + IrqMethod::Apic => lapic_eoi(), } - IrqMethod::Apic => lapic_eoi(), } } unsafe fn pic_mask(irq: u8) { - debug_assert!(irq < 16); + unsafe { + debug_assert!(irq < 16); - if irq >= 8 { - pic::slave().mask_set(irq - 8); - } else { - pic::master().mask_set(irq); + if irq >= 8 { + pic::slave().mask_set(irq - 8); + } else { + pic::master().mask_set(irq); + } } } unsafe fn ioapic_mask(irq: u8) { - ioapic::mask(irq); + unsafe { + ioapic::mask(irq); + } } unsafe fn pic_eoi(irq: u8) { - debug_assert!(irq < 16); + unsafe { + debug_assert!(irq < 16); - if irq >= 8 { - pic::master().ack(); - pic::slave().ack(); - } else { - pic::master().ack(); + if irq >= 8 { + pic::master().ack(); + pic::slave().ack(); + } else { + pic::master().ack(); + } } } unsafe fn lapic_eoi() { - local_apic::the_local_apic().eoi() + unsafe { local_apic::the_local_apic().eoi() } } unsafe fn pic_unmask(irq: usize) { - debug_assert!(irq < 16); + unsafe { + debug_assert!(irq < 16); - if irq >= 8 { - pic::slave().mask_clear(irq as u8 - 8); - } else { - pic::master().mask_clear(irq as u8); + if irq >= 8 { + pic::slave().mask_clear(irq as u8 - 8); + } else { + pic::master().mask_clear(irq as u8); + } } } unsafe fn ioapic_unmask(irq: usize) { - ioapic::unmask(irq as u8); + unsafe { + ioapic::unmask(irq as u8); + } } interrupt_stack!(pit_stack, |_stack| { @@ -286,7 +302,7 @@ interrupt!(lapic_error, || { // XXX: This would look way prettier using const generics. macro_rules! allocatable_irq( - ( $idt:expr, $number:literal, $name:ident ) => { + ( $idt:expr_2021, $number:literal, $name:ident ) => { interrupt!($name, || { allocatable_irq_generic($number); }); @@ -294,8 +310,10 @@ macro_rules! allocatable_irq( ); pub unsafe fn allocatable_irq_generic(number: u8) { - irq_trigger(number - 32); - lapic_eoi(); + unsafe { + irq_trigger(number - 32); + lapic_eoi(); + } } define_default_irqs!(); diff --git a/src/arch/x86/macros.rs b/src/arch/x86/macros.rs index 5f888cb082..ef4d8bd22f 100644 --- a/src/arch/x86/macros.rs +++ b/src/arch/x86/macros.rs @@ -11,13 +11,13 @@ macro_rules! print { #[macro_export] macro_rules! println { () => (print!("\n")); - ($fmt:expr) => (print!(concat!($fmt, "\n"))); - ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); + ($fmt:expr_2021) => (print!(concat!($fmt, "\n"))); + ($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } #[macro_export] macro_rules! irqs( - ( [ $( ($idt:expr, $number:literal, $name:ident) ,)* ], $submac:ident ) => { + ( [ $( ($idt:expr_2021, $number:literal, $name:ident) ,)* ], $submac:ident ) => { $( $submac!($idt, $number, $name); )* @@ -28,7 +28,7 @@ macro_rules! irqs( // allocatable_irq_NUM. #[macro_export] macro_rules! default_irqs( - ($idt:expr, $submac:ident) => { + ($idt:expr_2021, $submac:ident) => { irqs!([ // interrupt vectors below 32 are exceptions // vectors 32..=47 are used for standard 8259 pic irqs. diff --git a/src/arch/x86/mod.rs b/src/arch/x86/mod.rs index b43528d434..6cbbf7654a 100644 --- a/src/arch/x86/mod.rs +++ b/src/arch/x86/mod.rs @@ -20,10 +20,11 @@ pub mod flags { } #[naked] -#[link_section = ".usercopy-fns"] +#[unsafe(link_section = ".usercopy-fns")] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - core::arch::naked_asm!( - " + unsafe { + core::arch::naked_asm!( + " push edi push esi @@ -38,7 +39,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - xor eax, eax ret " - ); + ); + } } pub use arch_copy_to_user as arch_copy_from_user; diff --git a/src/arch/x86_64/alternative.rs b/src/arch/x86_64/alternative.rs index 639540e5b9..e552c7585c 100644 --- a/src/arch/x86_64/alternative.rs +++ b/src/arch/x86_64/alternative.rs @@ -38,212 +38,216 @@ pub struct AltReloc { #[cold] pub unsafe fn early_init(bsp: bool) { - let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start(); - let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset; + unsafe { + let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start(); + let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset; - assert_eq!(relocs_size % size_of::(), 0); - let relocs = core::slice::from_raw_parts( - relocs_offset as *const AltReloc, - relocs_size / size_of::(), - ); - - let mut enable = KcpuFeatures::empty(); - - if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) { - // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any - // userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This - // limits user-memory accesses to the UserSlice wrapper, so that no data outside of - // usercopy functions can be accidentally accessed by the kernel. - x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP); - // Clear CLAC in (the probably unlikely) case the bootloader set it earlier. - x86::bits64::rflags::clac(); - - enable |= KcpuFeatures::SMAP; - } else { - assert!(cfg!(not(cpu_feature_always = "smap"))); - } - - if cfg!(not(cpu_feature_never = "fsgsbase")) - && let Some(f) = cpuid().get_extended_feature_info() - && f.has_fsgsbase() - { - x86::controlregs::cr4_write( - x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE, + assert_eq!(relocs_size % size_of::(), 0); + let relocs = core::slice::from_raw_parts( + relocs_offset as *const AltReloc, + relocs_size / size_of::(), ); - enable |= KcpuFeatures::FSGSBASE; - } else { - assert!(cfg!(not(cpu_feature_always = "fsgsbase"))); + let mut enable = KcpuFeatures::empty(); + + if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) { + // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any + // userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This + // limits user-memory accesses to the UserSlice wrapper, so that no data outside of + // usercopy functions can be accidentally accessed by the kernel. + x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP); + // Clear CLAC in (the probably unlikely) case the bootloader set it earlier. + x86::bits64::rflags::clac(); + + enable |= KcpuFeatures::SMAP; + } else { + assert!(cfg!(not(cpu_feature_always = "smap"))); + } + + if cfg!(not(cpu_feature_never = "fsgsbase")) + && let Some(f) = cpuid().get_extended_feature_info() + && f.has_fsgsbase() + { + x86::controlregs::cr4_write( + x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE, + ); + + enable |= KcpuFeatures::FSGSBASE; + } else { + assert!(cfg!(not(cpu_feature_always = "fsgsbase"))); + } + + #[cfg(not(cpu_feature_never = "xsave"))] + if feature_info().has_xsave() { + use raw_cpuid::{ExtendedRegisterStateLocation, ExtendedRegisterType}; + + x86::controlregs::cr4_write( + x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE, + ); + + let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE; + x86::controlregs::xcr0_write(xcr0); + let ext_state_info = cpuid() + .get_extended_state_info() + .expect("must be present if XSAVE is supported"); + + enable |= KcpuFeatures::XSAVE; + enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt()); + + let info = xsave::XsaveInfo { + ymm_upper_offset: feature_info().has_avx().then(|| { + xcr0 |= Xcr0::XCR0_AVX_STATE; + x86::controlregs::xcr0_write(xcr0); + + let state = ext_state_info + .iter() + .find(|state| { + state.register() == ExtendedRegisterType::Avx + && state.location() == ExtendedRegisterStateLocation::Xcr0 + }) + .expect("CPUID said AVX was supported but there's no state info"); + + if state.size() as usize != 16 * core::mem::size_of::() { + log::warn!("Unusual AVX state size {}", state.size()); + } + + state.offset() + }), + xsave_size: ext_state_info.xsave_area_size_enabled_features(), + }; + log::debug!("XSAVE: {:?}", info); + + xsave::XSAVE_INFO.call_once(|| info); + } else { + assert!(cfg!(not(cpu_feature_always = "xsave"))); + } + + if !bsp { + return; + } + + overwrite(&relocs, enable); + + if cfg!(not(feature = "self_modifying")) { + assert!( + cfg!(not(cpu_feature_auto = "smap")) + && cfg!(not(cpu_feature_auto = "fsgsbase")) + && cfg!(not(cpu_feature_auto = "xsave")) + && cfg!(not(cpu_feature_auto = "xsaveopt")) + ); + } + + FEATURES.call_once(|| enable); } - - #[cfg(not(cpu_feature_never = "xsave"))] - if feature_info().has_xsave() { - use raw_cpuid::{ExtendedRegisterStateLocation, ExtendedRegisterType}; - - x86::controlregs::cr4_write( - x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE, - ); - - let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE; - x86::controlregs::xcr0_write(xcr0); - let ext_state_info = cpuid() - .get_extended_state_info() - .expect("must be present if XSAVE is supported"); - - enable |= KcpuFeatures::XSAVE; - enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt()); - - let info = xsave::XsaveInfo { - ymm_upper_offset: feature_info().has_avx().then(|| { - xcr0 |= Xcr0::XCR0_AVX_STATE; - x86::controlregs::xcr0_write(xcr0); - - let state = ext_state_info - .iter() - .find(|state| { - state.register() == ExtendedRegisterType::Avx - && state.location() == ExtendedRegisterStateLocation::Xcr0 - }) - .expect("CPUID said AVX was supported but there's no state info"); - - if state.size() as usize != 16 * core::mem::size_of::() { - log::warn!("Unusual AVX state size {}", state.size()); - } - - state.offset() - }), - xsave_size: ext_state_info.xsave_area_size_enabled_features(), - }; - log::debug!("XSAVE: {:?}", info); - - xsave::XSAVE_INFO.call_once(|| info); - } else { - assert!(cfg!(not(cpu_feature_always = "xsave"))); - } - - if !bsp { - return; - } - - overwrite(&relocs, enable); - - if cfg!(not(feature = "self_modifying")) { - assert!( - cfg!(not(cpu_feature_auto = "smap")) - && cfg!(not(cpu_feature_auto = "fsgsbase")) - && cfg!(not(cpu_feature_auto = "xsave")) - && cfg!(not(cpu_feature_auto = "xsaveopt")) - ); - } - - FEATURES.call_once(|| enable); } unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) { - if cfg!(not(feature = "self_modifying")) { - return; - } - - log::info!("self-modifying features: {:?}", enable); - - let mut mapper = KernelMapper::lock(); - for reloc in relocs.iter().copied() { - let name = core::str::from_utf8(core::slice::from_raw_parts( - reloc.name_start, - reloc.name_len, - )) - .expect("invalid feature name"); - let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len); - - let dst_pages = PageSpan::between( - Page::containing_address(VirtualAddress::new(reloc.code_start as usize)), - Page::containing_address(VirtualAddress::new( - (reloc.code_start as usize + reloc.padded_len).next_multiple_of(PAGE_SIZE), - )), - ); - for page in dst_pages.pages() { - mapper - .get_mut() - .unwrap() - .remap( - page.start_address(), - PageFlags::new().write(true).execute(true).global(true), - ) - .unwrap() - .flush(); + unsafe { + if cfg!(not(feature = "self_modifying")) { + return; } - let code = core::slice::from_raw_parts_mut(reloc.code_start, reloc.padded_len); + log::info!("self-modifying features: {:?}", enable); - log::trace!( - "feature {} current {:x?} altcode {:x?}", - name, - code, - altcode - ); + let mut mapper = KernelMapper::lock(); + for reloc in relocs.iter().copied() { + let name = core::str::from_utf8(core::slice::from_raw_parts( + reloc.name_start, + reloc.name_len, + )) + .expect("invalid feature name"); + let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len); - let feature_is_enabled = match name { - "smap" => enable.contains(KcpuFeatures::SMAP), - "fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE), - "xsave" => enable.contains(KcpuFeatures::XSAVE), - "xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT), - //_ => panic!("unknown altcode relocation: {}", name), - _ => true, - }; - - // XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X` - // only outputs the (slower) single-byte 0x90 NOP. - - // This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November - // 2020). - const NOPS_TABLE: [&[u8]; 11] = [ - &[0x90], - &[0x66, 0x90], - &[0x0f, 0x1f, 0x00], - &[0x0f, 0x1f, 0x40, 0x00], - &[0x0f, 0x1f, 0x44, 0x00, 0x00], - &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00], - &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00], - &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], - &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], - &[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], - &[ - 0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, - ], - ]; - - if feature_is_enabled { - log::trace!("feature {} origcode {:x?}", name, code); - let (dst, dst_nops) = code.split_at_mut(altcode.len()); - dst.copy_from_slice(altcode); - - for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) { - chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]); - } - log::trace!("feature {} new {:x?} altcode {:x?}", name, code, altcode); - } else { - log::trace!("feature !{} origcode {:x?}", name, code); - let (_, padded) = code.split_at_mut(reloc.origcode_len); - - // Not strictly necessary, but reduces the number of instructions using longer nop - // instructions. - for chunk in padded.chunks_mut(NOPS_TABLE.len()) { - chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]); + let dst_pages = PageSpan::between( + Page::containing_address(VirtualAddress::new(reloc.code_start as usize)), + Page::containing_address(VirtualAddress::new( + (reloc.code_start as usize + reloc.padded_len).next_multiple_of(PAGE_SIZE), + )), + ); + for page in dst_pages.pages() { + mapper + .get_mut() + .unwrap() + .remap( + page.start_address(), + PageFlags::new().write(true).execute(true).global(true), + ) + .unwrap() + .flush(); } - log::trace!("feature !{} new {:x?}", name, code); - } + let code = core::slice::from_raw_parts_mut(reloc.code_start, reloc.padded_len); - for page in dst_pages.pages() { - mapper - .get_mut() - .unwrap() - .remap( - page.start_address(), - PageFlags::new().write(false).execute(true).global(true), - ) - .unwrap() - .flush(); + log::trace!( + "feature {} current {:x?} altcode {:x?}", + name, + code, + altcode + ); + + let feature_is_enabled = match name { + "smap" => enable.contains(KcpuFeatures::SMAP), + "fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE), + "xsave" => enable.contains(KcpuFeatures::XSAVE), + "xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT), + //_ => panic!("unknown altcode relocation: {}", name), + _ => true, + }; + + // XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X` + // only outputs the (slower) single-byte 0x90 NOP. + + // This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November + // 2020). + const NOPS_TABLE: [&[u8]; 11] = [ + &[0x90], + &[0x66, 0x90], + &[0x0f, 0x1f, 0x00], + &[0x0f, 0x1f, 0x40, 0x00], + &[0x0f, 0x1f, 0x44, 0x00, 0x00], + &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00], + &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00], + &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[ + 0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, + ], + ]; + + if feature_is_enabled { + log::trace!("feature {} origcode {:x?}", name, code); + let (dst, dst_nops) = code.split_at_mut(altcode.len()); + dst.copy_from_slice(altcode); + + for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) { + chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]); + } + log::trace!("feature {} new {:x?} altcode {:x?}", name, code, altcode); + } else { + log::trace!("feature !{} origcode {:x?}", name, code); + let (_, padded) = code.split_at_mut(reloc.origcode_len); + + // Not strictly necessary, but reduces the number of instructions using longer nop + // instructions. + for chunk in padded.chunks_mut(NOPS_TABLE.len()) { + chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]); + } + + log::trace!("feature !{} new {:x?}", name, code); + } + + for page in dst_pages.pages() { + mapper + .get_mut() + .unwrap() + .remap( + page.start_address(), + PageFlags::new().write(false).execute(true).global(true), + ) + .unwrap() + .flush(); + } } } } diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 83c1e6dee5..88a6153a06 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -142,12 +142,14 @@ const _: () = { }; pub unsafe fn pcr() -> *mut ProcessorControlRegion { - // Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that - // obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably - // microcoded) instructions. - let mut ret: *mut ProcessorControlRegion; - core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref))); - ret + unsafe { + // Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that + // obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably + // microcoded) instructions. + let mut ret: *mut ProcessorControlRegion; + core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref))); + ret + } } #[cfg(feature = "pti")] @@ -160,109 +162,122 @@ pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) { #[cfg(not(feature = "pti"))] pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) { - // TODO: If this increases performance, read gs:[offset] directly - core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64); + unsafe { + // TODO: If this increases performance, read gs:[offset] directly + core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64); + } } pub unsafe fn set_userspace_io_allowed(pcr: *mut ProcessorControlRegion, allowed: bool) { - let offset = if allowed { - u16::try_from(size_of::()).unwrap() - } else { - 0xFFFF - }; - core::ptr::addr_of_mut!((*pcr).tss.iomap_base).write(offset); + unsafe { + let offset = if allowed { + u16::try_from(size_of::()).unwrap() + } else { + 0xFFFF + }; + core::ptr::addr_of_mut!((*pcr).tss.iomap_base).write(offset); + } } // Initialize startup GDT #[cold] pub unsafe fn init() { - // Before the kernel can remap itself, it needs to switch to a GDT it controls. Start with a - // minimal kernel-only GDT. - dtables::lgdt(&DescriptorTablePointer { - limit: (INIT_GDT.len() * size_of::() - 1) as u16, - base: INIT_GDT.as_ptr() as *const SegmentDescriptor, - }); + unsafe { + // Before the kernel can remap itself, it needs to switch to a GDT it controls. Start with a + // minimal kernel-only GDT. + dtables::lgdt(&DescriptorTablePointer { + limit: (INIT_GDT.len() * size_of::() - 1) as u16, + base: INIT_GDT.as_ptr() as *const SegmentDescriptor, + }); - load_segments(); + load_segments(); + } } #[cold] unsafe fn load_segments() { - segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); - segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + unsafe { + segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); + segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_ds(SegmentSelector::from_raw(0)); - segmentation::load_es(SegmentSelector::from_raw(0)); - segmentation::load_fs(SegmentSelector::from_raw(0)); + segmentation::load_ds(SegmentSelector::from_raw(0)); + segmentation::load_es(SegmentSelector::from_raw(0)); + segmentation::load_fs(SegmentSelector::from_raw(0)); - // What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However, - // GSBASE is set later, and percpu is not used until gdt::init_paging(). - segmentation::load_gs(SegmentSelector::from_raw(0)); + // What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However, + // GSBASE is set later, and percpu is not used until gdt::init_paging(). + segmentation::load_gs(SegmentSelector::from_raw(0)); + } } /// Initialize GDT and PCR. #[cold] pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { - let alloc_order = size_of::() - .div_ceil(PAGE_SIZE) - .next_power_of_two() - .trailing_zeros(); - let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR"); - let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion); + unsafe { + let alloc_order = size_of::() + .div_ceil(PAGE_SIZE) + .next_power_of_two() + .trailing_zeros(); + let pcr_frame = + crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR"); + let pcr = + &mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion); - pcr.self_ref = pcr as *mut ProcessorControlRegion as usize; + pcr.self_ref = pcr as *mut ProcessorControlRegion as usize; - // Setup the GDT. - pcr.gdt = BASE_GDT; + // Setup the GDT. + pcr.gdt = BASE_GDT; - let limit = (pcr.gdt.len() * size_of::() - 1) - .try_into() - .expect("main GDT way too large"); - let base = pcr.gdt.as_ptr() as *const SegmentDescriptor; + let limit = (pcr.gdt.len() * size_of::() - 1) + .try_into() + .expect("main GDT way too large"); + let base = pcr.gdt.as_ptr() as *const SegmentDescriptor; - let gdtr: DescriptorTablePointer = DescriptorTablePointer { limit, base }; + let gdtr: DescriptorTablePointer = + DescriptorTablePointer { limit, base }; - { - pcr.tss.iomap_base = 0xFFFF; - pcr._all_ones = 0xFF; + { + pcr.tss.iomap_base = 0xFFFF; + pcr._all_ones = 0xFF; - let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64; - let tss_lo = (tss & 0xFFFF_FFFF) as u32; - let tss_hi = (tss >> 32) as u32; + let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64; + let tss_lo = (tss & 0xFFFF_FFFF) as u32; + let tss_hi = (tss >> 32) as u32; - pcr.gdt[GDT_TSS].set_offset(tss_lo); - pcr.gdt[GDT_TSS].set_limit(size_of::() as u32 + IOBITMAP_SIZE); + pcr.gdt[GDT_TSS].set_offset(tss_lo); + pcr.gdt[GDT_TSS].set_limit(size_of::() as u32 + IOBITMAP_SIZE); - (&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry) - .cast::() - .write(tss_hi); + (&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry) + .cast::() + .write(tss_hi); + } + + // Load the new GDT, which is correctly located in thread local storage. + dtables::lgdt(&gdtr); + + // Load segments again, possibly resetting FSBASE and GSBASE. + load_segments(); + + // Ensure that GSBASE always points to the PCR in kernel space. + x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64); + + // While GSBASE points to the PCR in kernel space, userspace is free to set it to other values. + // Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is + // because entering usermode will entail executing the SWAPGS instruction. + x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0); + + // Set the userspace FSBASE to zero. + x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0); + + // Set the stack pointer to use when coming back from userspace. + set_tss_stack(pcr, stack_offset); + + // Load the task register + task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); + + pcr.percpu = PercpuBlock::init(cpu_id); + + crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu); } - - // Load the new GDT, which is correctly located in thread local storage. - dtables::lgdt(&gdtr); - - // Load segments again, possibly resetting FSBASE and GSBASE. - load_segments(); - - // Ensure that GSBASE always points to the PCR in kernel space. - x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64); - - // While GSBASE points to the PCR in kernel space, userspace is free to set it to other values. - // Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is - // because entering usermode will entail executing the SWAPGS instruction. - x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0); - - // Set the userspace FSBASE to zero. - x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0); - - // Set the stack pointer to use when coming back from userspace. - set_tss_stack(pcr, stack_offset); - - // Load the task register - task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); - - pcr.percpu = PercpuBlock::init(cpu_id); - - crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu); } #[derive(Copy, Clone, Debug)] #[repr(C, packed)] diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index 75d9f94dd3..7597de3261 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -360,12 +360,12 @@ macro_rules! nop { macro_rules! interrupt_stack { // XXX: Apparently we cannot use $expr and check for bool exhaustiveness, so we will have to // use idents directly instead. - ($name:ident, $save1:ident!, $save2:ident!, $rstor2:ident!, $rstor1:ident!, is_paranoid: $is_paranoid:expr, |$stack:ident| $code:block) => { + ($name:ident, $save1:ident!, $save2:ident!, $rstor2:ident!, $rstor1:ident!, is_paranoid: $is_paranoid:expr_2021, |$stack:ident| $code:block) => { #[naked] - pub unsafe extern "C" fn $name() { - unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) { + pub unsafe extern "C" fn $name() { unsafe { + unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) { unsafe { $code - } + }} core::arch::naked_asm!(concat!( // Clear direction flag, required by ABI when running any Rust code in the kernel. "cld;", @@ -405,7 +405,7 @@ macro_rules! interrupt_stack { PCR_GDT_OFFSET = const(core::mem::offset_of!(crate::gdt::ProcessorControlRegion, gdt)), ); - } + }} }; ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, swapgs_iff_ring3_fast!, nop!, nop!, swapgs_iff_ring3_fast!, is_paranoid: false, |$stack| $code); }; ($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, conditional_swapgs_paranoid!, conditional_swapgs_back_paranoid!, nop!, is_paranoid: true, |$stack| $code); } @@ -415,10 +415,10 @@ macro_rules! interrupt_stack { macro_rules! interrupt { ($name:ident, || $code:block) => { #[naked] - pub unsafe extern "C" fn $name() { - unsafe extern "C" fn inner() { + pub unsafe extern "C" fn $name() { unsafe { + unsafe extern "C" fn inner() { unsafe { $code - } + }} core::arch::naked_asm!(concat!( // Clear direction flag, required by ABI when running any Rust code in the kernel. @@ -447,7 +447,7 @@ macro_rules! interrupt { inner = sym inner, ); - } + }} }; } @@ -455,10 +455,10 @@ macro_rules! interrupt { macro_rules! interrupt_error { ($name:ident, |$stack:ident, $error_code:ident| $code:block) => { #[naked] - pub unsafe extern "C" fn $name() { - unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::handler::InterruptStack, $error_code: usize) { + pub unsafe extern "C" fn $name() { unsafe { + unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::handler::InterruptStack, $error_code: usize) { unsafe { $code - } + }} core::arch::naked_asm!(concat!( // Clear direction flag, required by ABI when running any Rust code in the kernel. @@ -500,7 +500,7 @@ macro_rules! interrupt_error { inner = sym inner, rax_offset = const(::core::mem::size_of::<$crate::interrupt::handler::PreservedRegisters>() + ::core::mem::size_of::<$crate::interrupt::handler::ScratchRegisters>() - 8), ); - } + }} }; } diff --git a/src/arch/x86_64/interrupt/irq.rs b/src/arch/x86_64/interrupt/irq.rs index 113556c6bb..4ceeacd991 100644 --- a/src/arch/x86_64/interrupt/irq.rs +++ b/src/arch/x86_64/interrupt/irq.rs @@ -70,85 +70,101 @@ fn irq_method() -> IrqMethod { /// Notify the IRQ scheme that an IRQ has been registered. This should mask the IRQ until the /// scheme user unmasks it ("acknowledges" it). unsafe fn trigger(irq: u8) { - match irq_method() { - IrqMethod::Pic => { - if irq < 16 { - pic_mask(irq) + unsafe { + match irq_method() { + IrqMethod::Pic => { + if irq < 16 { + pic_mask(irq) + } } + IrqMethod::Apic => ioapic_mask(irq), } - IrqMethod::Apic => ioapic_mask(irq), + irq_trigger(irq); } - irq_trigger(irq); } /// Unmask the IRQ. This is called from the IRQ scheme, which does this when a user process has /// processed the IRQ. pub unsafe fn acknowledge(irq: usize) { - match irq_method() { - IrqMethod::Pic => { - if irq < 16 { - pic_unmask(irq) + unsafe { + match irq_method() { + IrqMethod::Pic => { + if irq < 16 { + pic_unmask(irq) + } } + IrqMethod::Apic => ioapic_unmask(irq), } - IrqMethod::Apic => ioapic_unmask(irq), } } /// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one. pub unsafe fn eoi(irq: u8) { - PercpuBlock::current().stats.add_irq(irq); + unsafe { + PercpuBlock::current().stats.add_irq(irq); - match irq_method() { - IrqMethod::Pic => { - if irq < 16 { - pic_eoi(irq) + match irq_method() { + IrqMethod::Pic => { + if irq < 16 { + pic_eoi(irq) + } } + IrqMethod::Apic => lapic_eoi(), } - IrqMethod::Apic => lapic_eoi(), } } unsafe fn pic_mask(irq: u8) { - debug_assert!(irq < 16); + unsafe { + debug_assert!(irq < 16); - if irq >= 8 { - pic::slave().mask_set(irq - 8); - } else { - pic::master().mask_set(irq); + if irq >= 8 { + pic::slave().mask_set(irq - 8); + } else { + pic::master().mask_set(irq); + } } } unsafe fn ioapic_mask(irq: u8) { - ioapic::mask(irq); + unsafe { + ioapic::mask(irq); + } } unsafe fn pic_eoi(irq: u8) { - debug_assert!(irq < 16); + unsafe { + debug_assert!(irq < 16); - if irq >= 8 { - pic::master().ack(); - pic::slave().ack(); - } else { - pic::master().ack(); + if irq >= 8 { + pic::master().ack(); + pic::slave().ack(); + } else { + pic::master().ack(); + } } } unsafe fn lapic_eoi() { - local_apic::the_local_apic().eoi() + unsafe { local_apic::the_local_apic().eoi() } } unsafe fn pic_unmask(irq: usize) { - debug_assert!(irq < 16); + unsafe { + debug_assert!(irq < 16); - if irq >= 8 { - pic::slave().mask_clear(irq as u8 - 8); - } else { - pic::master().mask_clear(irq as u8); + if irq >= 8 { + pic::slave().mask_clear(irq as u8 - 8); + } else { + pic::master().mask_clear(irq as u8); + } } } unsafe fn ioapic_unmask(irq: usize) { - ioapic::unmask(irq as u8); + unsafe { + ioapic::unmask(irq as u8); + } } interrupt_stack!(pit_stack, |_stack| { @@ -311,7 +327,7 @@ __generic_interrupts_start: __generic_interrupts_end: ", sym generic_irq); -extern "C" { +unsafe extern "C" { pub fn __generic_interrupts_start(); pub fn __generic_interrupts_end(); } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 828908539a..39a7bc50d5 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -11,173 +11,179 @@ use x86::{ }; pub unsafe fn init() { - // IA32_STAR[31:0] are reserved. + unsafe { + // IA32_STAR[31:0] are reserved. - // The base selector of the two consecutive segments for kernel code and the immediately - // suceeding stack (data). - let syscall_cs_ss_base = (gdt::GDT_KERNEL_CODE as u16) << 3; - // The base selector of the three consecutive segments (of which two are used) for user code - // and user data. It points to a 32-bit code segment, which must be followed by a data segment - // (stack), and a 64-bit code segment. - let sysret_cs_ss_base = ((gdt::GDT_USER_CODE32_UNUSED as u16) << 3) | 3; - let star_high = u32::from(syscall_cs_ss_base) | (u32::from(sysret_cs_ss_base) << 16); + // The base selector of the two consecutive segments for kernel code and the immediately + // suceeding stack (data). + let syscall_cs_ss_base = (gdt::GDT_KERNEL_CODE as u16) << 3; + // The base selector of the three consecutive segments (of which two are used) for user code + // and user data. It points to a 32-bit code segment, which must be followed by a data segment + // (stack), and a 64-bit code segment. + let sysret_cs_ss_base = ((gdt::GDT_USER_CODE32_UNUSED as u16) << 3) | 3; + let star_high = u32::from(syscall_cs_ss_base) | (u32::from(sysret_cs_ss_base) << 16); - msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32); - msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64); + msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32); + msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64); - // DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK, - // userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and - // cause all sorts of memory corruption. - // - // IF needs to be cleared, as the kernel currently assumes interrupts are disabled except in - // usermode and in kmain. - // - // TF needs to be cleared, as enabling userspace-rflags-controlled singlestep in the kernel - // would be a bad idea. - // - // AC it should always be cleared when entering the kernel (and never be set except in usercopy - // functions), if for some reason AC was set before entering userspace (AC can only be modified - // by kernel code). - // - // The other flags could indeed be preserved and excluded from FMASK, but since they are not - // used to pass data to the kernel, they might as well be masked with *marginal* security - // benefits. - // - // Flags not included here are IOPL (not relevant to the kernel at all), "CPUID flag" (not used - // at all in 64-bit mode), RF (not used yet, but DR breakpoints would remain enabled both in - // user and kernel mode), VM8086 (not used at all), and VIF/VIP (system-level status flags?). + // DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK, + // userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and + // cause all sorts of memory corruption. + // + // IF needs to be cleared, as the kernel currently assumes interrupts are disabled except in + // usermode and in kmain. + // + // TF needs to be cleared, as enabling userspace-rflags-controlled singlestep in the kernel + // would be a bad idea. + // + // AC it should always be cleared when entering the kernel (and never be set except in usercopy + // functions), if for some reason AC was set before entering userspace (AC can only be modified + // by kernel code). + // + // The other flags could indeed be preserved and excluded from FMASK, but since they are not + // used to pass data to the kernel, they might as well be masked with *marginal* security + // benefits. + // + // Flags not included here are IOPL (not relevant to the kernel at all), "CPUID flag" (not used + // at all in 64-bit mode), RF (not used yet, but DR breakpoints would remain enabled both in + // user and kernel mode), VM8086 (not used at all), and VIF/VIP (system-level status flags?). - let mask_critical = RFlags::FLAGS_DF | RFlags::FLAGS_IF | RFlags::FLAGS_TF | RFlags::FLAGS_AC; - let mask_other = RFlags::FLAGS_CF - | RFlags::FLAGS_PF - | RFlags::FLAGS_AF - | RFlags::FLAGS_ZF - | RFlags::FLAGS_SF - | RFlags::FLAGS_OF; - msr::wrmsr(msr::IA32_FMASK, (mask_critical | mask_other).bits()); + let mask_critical = + RFlags::FLAGS_DF | RFlags::FLAGS_IF | RFlags::FLAGS_TF | RFlags::FLAGS_AC; + let mask_other = RFlags::FLAGS_CF + | RFlags::FLAGS_PF + | RFlags::FLAGS_AF + | RFlags::FLAGS_ZF + | RFlags::FLAGS_SF + | RFlags::FLAGS_OF; + msr::wrmsr(msr::IA32_FMASK, (mask_critical | mask_other).bits()); - let efer = msr::rdmsr(msr::IA32_EFER); - msr::wrmsr(msr::IA32_EFER, efer | 1); + let efer = msr::rdmsr(msr::IA32_EFER); + msr::wrmsr(msr::IA32_EFER, efer | 1); + } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) { - let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None) - .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE))); + unsafe { + let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None) + .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE))); - if allowed.unwrap_or(true) { - let scratch = &(*stack).scratch; + if allowed.unwrap_or(true) { + let scratch = &(*stack).scratch; - let ret = syscall::syscall( - scratch.rax, - scratch.rdi, - scratch.rsi, - scratch.rdx, - scratch.r10, - scratch.r8, - ); - (*stack).scratch.rax = ret; + let ret = syscall::syscall( + scratch.rax, + scratch.rdi, + scratch.rsi, + scratch.rdx, + scratch.r10, + scratch.r8, + ); + (*stack).scratch.rax = ret; + } + + ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); } - - ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); } #[naked] #[allow(named_asm_labels)] pub unsafe extern "C" fn syscall_instruction() { - core::arch::naked_asm!(concat!( - // Yes, this is magic. No, you don't need to understand - "swapgs;", // Swap KGSBASE with GSBASE, allowing fast TSS access. - "mov gs:[{sp}], rsp;", // Save userspace stack pointer - "mov rsp, gs:[{ksp}];", // Load kernel stack pointer - "push QWORD PTR {ss_sel};", // Push fake userspace SS (resembling iret frame) - "push QWORD PTR gs:[{sp}];", // Push userspace rsp - "push r11;", // Push rflags - "push QWORD PTR {cs_sel};", // Push fake CS (resembling iret stack frame) - "push rcx;", // Push userspace return pointer + unsafe { + core::arch::naked_asm!(concat!( + // Yes, this is magic. No, you don't need to understand + "swapgs;", // Swap KGSBASE with GSBASE, allowing fast TSS access. + "mov gs:[{sp}], rsp;", // Save userspace stack pointer + "mov rsp, gs:[{ksp}];", // Load kernel stack pointer + "push QWORD PTR {ss_sel};", // Push fake userspace SS (resembling iret frame) + "push QWORD PTR gs:[{sp}];", // Push userspace rsp + "push r11;", // Push rflags + "push QWORD PTR {cs_sel};", // Push fake CS (resembling iret stack frame) + "push rcx;", // Push userspace return pointer - // Push context registers - "push rax;", - push_scratch!(), - push_preserved!(), + // Push context registers + "push rax;", + push_scratch!(), + push_preserved!(), - // TODO: Map PTI - // $crate::arch::x86_64::pti::map(); + // TODO: Map PTI + // $crate::arch::x86_64::pti::map(); - // Call inner funtion - "mov rdi, rsp;", - "call __inner_syscall_instruction;", + // Call inner funtion + "mov rdi, rsp;", + "call __inner_syscall_instruction;", - // TODO: Unmap PTI - // $crate::arch::x86_64::pti::unmap(); + // TODO: Unmap PTI + // $crate::arch::x86_64::pti::unmap(); - " + " .globl enter_usermode enter_usermode: ", - // Pop context registers - pop_preserved!(), - pop_scratch!(), + // Pop context registers + pop_preserved!(), + pop_scratch!(), - // Restore user GSBASE by swapping GSBASE and KGSBASE. - "swapgs;", + // Restore user GSBASE by swapping GSBASE and KGSBASE. + "swapgs;", - // TODO: Should we unconditionally jump or avoid jumping, to hint to the branch predictor that - // singlestep is NOT set? - // - // It appears Intel CPUs assume (previously unknown) forward conditional branches to not be - // taken, and AMD appears to assume all previously unknown conditional branches will not be - // taken. + // TODO: Should we unconditionally jump or avoid jumping, to hint to the branch predictor that + // singlestep is NOT set? + // + // It appears Intel CPUs assume (previously unknown) forward conditional branches to not be + // taken, and AMD appears to assume all previously unknown conditional branches will not be + // taken. - // Check if the Trap Flag (singlestep flag) is set. If so, sysretq will return to before the - // instruction, whereas debuggers expect the iretq behavior of returning to after the - // instruction. + // Check if the Trap Flag (singlestep flag) is set. If so, sysretq will return to before the + // instruction, whereas debuggers expect the iretq behavior of returning to after the + // instruction. - // TODO: Which one is faster? - // bt DWORD PTR [rsp + 16], 8 - // or, - // bt BYTE PTR [rsp + 17], 0 - // or, - // test BYTE PTR [rsp + 17], 1 - // or, - // test WORD PTR [rsp + 16], 0x100 - // or, - // test DWORD PTR [rsp + 16], 0x100 - // ? + // TODO: Which one is faster? + // bt DWORD PTR [rsp + 16], 8 + // or, + // bt BYTE PTR [rsp + 17], 0 + // or, + // test BYTE PTR [rsp + 17], 1 + // or, + // test WORD PTR [rsp + 16], 0x100 + // or, + // test DWORD PTR [rsp + 16], 0x100 + // ? - "test BYTE PTR [rsp + 17], 1;", - // If set, return using IRETQ instead. - "jnz 2f;", + "test BYTE PTR [rsp + 17], 1;", + // If set, return using IRETQ instead. + "jnz 2f;", - // Otherwise, continue with the fast sysretq. + // Otherwise, continue with the fast sysretq. - // Pop userspace return pointer - "pop rcx;", + // Pop userspace return pointer + "pop rcx;", - // We must ensure RCX is canonical; if it is not when running sysretq, the consequences can be - // fatal from a security perspective. - // - // See https://xenproject.org/2012/06/13/the-intel-sysret-privilege-escalation/. - // - // This is not just theoretical; ptrace allows userspace to change RCX (via RIP) of target - // processes. - // - // While we could also conditionally IRETQ here, an easier method is to simply sign-extend RCX: + // We must ensure RCX is canonical; if it is not when running sysretq, the consequences can be + // fatal from a security perspective. + // + // See https://xenproject.org/2012/06/13/the-intel-sysret-privilege-escalation/. + // + // This is not just theoretical; ptrace allows userspace to change RCX (via RIP) of target + // processes. + // + // While we could also conditionally IRETQ here, an easier method is to simply sign-extend RCX: - // Shift away the upper 16 bits (0xBAAD_8000_DEAD_BEEF => 0x8000_DEAD_BEEF_XXXX). - "shl rcx, 16;", - // Shift arithmetically right by 16 bits, effectively extending the 47th sign bit to bits - // 63:48 (0x8000_DEAD_BEEF_XXXX => 0xFFFF_8000_DEAD_BEEF). - "sar rcx, 16;", + // Shift away the upper 16 bits (0xBAAD_8000_DEAD_BEEF => 0x8000_DEAD_BEEF_XXXX). + "shl rcx, 16;", + // Shift arithmetically right by 16 bits, effectively extending the 47th sign bit to bits + // 63:48 (0x8000_DEAD_BEEF_XXXX => 0xFFFF_8000_DEAD_BEEF). + "sar rcx, 16;", - "add rsp, 8;", // Pop fake userspace CS - "pop r11;", // Pop rflags - "pop rsp;", // Restore userspace stack pointer - "sysretq;", // Return into userspace; RCX=>RIP,R11=>RFLAGS + "add rsp, 8;", // Pop fake userspace CS + "pop r11;", // Pop rflags + "pop rsp;", // Restore userspace stack pointer + "sysretq;", // Return into userspace; RCX=>RIP,R11=>RFLAGS - // IRETQ fallback: - " + // IRETQ fallback: + " .p2align 4 2: xor rcx, rcx @@ -185,13 +191,14 @@ pub unsafe extern "C" fn syscall_instruction() { iretq "), - sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)), - ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)), - ss_sel = const(SegmentSelector::new(gdt::GDT_USER_DATA as u16, x86::Ring::Ring3).bits()), - cs_sel = const(SegmentSelector::new(gdt::GDT_USER_CODE as u16, x86::Ring::Ring3).bits()), - ); + sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)), + ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)), + ss_sel = const(SegmentSelector::new(gdt::GDT_USER_DATA as u16, x86::Ring::Ring3).bits()), + cs_sel = const(SegmentSelector::new(gdt::GDT_USER_CODE as u16, x86::Ring::Ring3).bits()), + ); + } } -extern "C" { +unsafe extern "C" { // TODO: macro? pub fn enter_usermode(); } diff --git a/src/arch/x86_64/macros.rs b/src/arch/x86_64/macros.rs index 5b165f9001..efdfa22295 100644 --- a/src/arch/x86_64/macros.rs +++ b/src/arch/x86_64/macros.rs @@ -11,18 +11,18 @@ macro_rules! print { #[macro_export] macro_rules! println { () => (print!("\n")); - ($fmt:expr) => (print!(concat!($fmt, "\n"))); - ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); + ($fmt:expr_2021) => (print!(concat!($fmt, "\n"))); + ($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } macro_rules! expand_bool( - ($value:expr) => { + ($value:expr_2021) => { concat!($value) } ); macro_rules! alternative( - (feature: $feature:literal, then: [$($then:expr),*], default: [$($default:expr),*]) => { + (feature: $feature:literal, then: [$($then:expr_2021),*], default: [$($default:expr_2021),*]) => { alternative2!(feature1: $feature, then1: [$($then),*], feature2: "", then2: [""], default: [$($default),*]) } ); @@ -39,7 +39,7 @@ macro_rules! saturating_sub( // // An empty string as feature is equivalent with "never". macro_rules! alternative2( - (feature1: $feature1:literal, then1: [$($then1:expr),*], feature2: $feature2:literal, then2: [$($then2:expr),*], default: [$($default:expr),*]) => { + (feature1: $feature1:literal, then1: [$($then1:expr_2021),*], feature2: $feature2:literal, then2: [$($then2:expr_2021),*], default: [$($default:expr_2021),*]) => { concat!(" .set true, 1 .set false, 0 @@ -70,7 +70,7 @@ macro_rules! alternative2( }; ); macro_rules! alternative_auto( - ($first_digit:literal, $feature:literal, [$($then:expr),*]) => { concat!( + ($first_digit:literal, $feature:literal, [$($then:expr_2021),*]) => { concat!( ".if ", expand_bool!(cfg!(cpu_feature_auto = $feature)), " .pushsection .altcode.", $feature, ",\"a\" ", $first_digit, "0: diff --git a/src/arch/x86_64/misc.rs b/src/arch/x86_64/misc.rs index 8659101c43..62e23e6fca 100644 --- a/src/arch/x86_64/misc.rs +++ b/src/arch/x86_64/misc.rs @@ -6,22 +6,24 @@ use crate::{ }; pub unsafe fn init(cpu_id: LogicalCpuId) { - if has_ext_feat(|feat| feat.has_umip()) { - // UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT, - // SMSW and STR. KASLR is currently not implemented, but this protects against leaking - // addresses. - x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_UMIP); - } - if has_ext_feat(|feat| feat.has_smep()) { - // SMEP (Supervisor-Mode Execution Prevention) forbids the kernel from executing - // instruction on any page marked "userspace-accessible". This improves security for - // obvious reasons. - x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP); - } + unsafe { + if has_ext_feat(|feat| feat.has_umip()) { + // UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT, + // SMSW and STR. KASLR is currently not implemented, but this protects against leaking + // addresses. + x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_UMIP); + } + if has_ext_feat(|feat| feat.has_smep()) { + // SMEP (Supervisor-Mode Execution Prevention) forbids the kernel from executing + // instruction on any page marked "userspace-accessible". This improves security for + // obvious reasons. + x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP); + } - if let Some(feats) = cpuid().get_extended_processor_and_feature_identifiers() - && feats.has_rdtscp() - { - x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into()); + if let Some(feats) = cpuid().get_extended_processor_and_feature_identifiers() + && feats.has_rdtscp() + { + x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into()); + } } } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 490d55b13e..f0cb6590ea 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -30,13 +30,14 @@ pub mod flags { // TODO: Maybe support rewriting relocations (using LD's --emit-relocs) when working with entire // functions? #[naked] -#[link_section = ".usercopy-fns"] +#[unsafe(link_section = ".usercopy-fns")] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - // TODO: spectre_v1 + unsafe { + // TODO: spectre_v1 - core::arch::naked_asm!(alternative!( - feature: "smap", - then: [" + core::arch::naked_asm!(alternative!( + feature: "smap", + then: [" xor eax, eax mov rcx, rdx stac @@ -44,13 +45,14 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - clac ret "], - default: [" + default: [" xor eax, eax mov rcx, rdx rep movsb ret "] - )); + )); + } } pub use arch_copy_to_user as arch_copy_from_user; diff --git a/src/arch/x86_shared/device/hpet.rs b/src/arch/x86_shared/device/hpet.rs index fd8708ae44..1e36f5b4e4 100644 --- a/src/arch/x86_shared/device/hpet.rs +++ b/src/arch/x86_shared/device/hpet.rs @@ -23,97 +23,103 @@ pub(crate) const T0_COMPARATOR_OFFSET: usize = 0x108; const PER_INT_CAP: u64 = 0x10; pub unsafe fn init(hpet: &mut Hpet) -> bool { - println!("HPET @ {:#x}", { hpet.base_address.address }); - debug_caps(hpet); + unsafe { + println!("HPET @ {:#x}", { hpet.base_address.address }); + debug_caps(hpet); - println!("HPET Before Init"); - debug_config(hpet); + println!("HPET Before Init"); + debug_config(hpet); - // Disable HPET - { - let mut config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET); - config_word &= !(LEG_RT_CNF | ENABLE_CNF); - hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word); + // Disable HPET + { + let mut config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET); + config_word &= !(LEG_RT_CNF | ENABLE_CNF); + hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word); + } + + let capability = hpet.read_u64(CAPABILITY_OFFSET); + if capability & LEG_RT_CAP == 0 { + log::warn!("HPET missing capability LEG_RT_CAP"); + return false; + } + + let period_fs = capability >> 32; + let divisor = (pit::RATE as u64 * 1_000_000) / period_fs; + + let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET); + if t0_capabilities & PER_INT_CAP == 0 { + log::warn!("HPET T0 missing capability PER_INT_CAP"); + return false; + } + + let counter = hpet.read_u64(MAIN_COUNTER_OFFSET); + + let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF; + hpet.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word); + // set accumulator value + hpet.write_u64(T0_COMPARATOR_OFFSET, counter + divisor); + // set interval + hpet.write_u64(T0_COMPARATOR_OFFSET, divisor); + + // Enable interrupts from the HPET + { + let mut config_word: u64 = hpet.read_u64(GENERAL_CONFIG_OFFSET); + config_word |= LEG_RT_CNF | ENABLE_CNF; + hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word); + } + + println!("HPET After Init"); + debug_config(hpet); + + true } - - let capability = hpet.read_u64(CAPABILITY_OFFSET); - if capability & LEG_RT_CAP == 0 { - log::warn!("HPET missing capability LEG_RT_CAP"); - return false; - } - - let period_fs = capability >> 32; - let divisor = (pit::RATE as u64 * 1_000_000) / period_fs; - - let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET); - if t0_capabilities & PER_INT_CAP == 0 { - log::warn!("HPET T0 missing capability PER_INT_CAP"); - return false; - } - - let counter = hpet.read_u64(MAIN_COUNTER_OFFSET); - - let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF; - hpet.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word); - // set accumulator value - hpet.write_u64(T0_COMPARATOR_OFFSET, counter + divisor); - // set interval - hpet.write_u64(T0_COMPARATOR_OFFSET, divisor); - - // Enable interrupts from the HPET - { - let mut config_word: u64 = hpet.read_u64(GENERAL_CONFIG_OFFSET); - config_word |= LEG_RT_CNF | ENABLE_CNF; - hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word); - } - - println!("HPET After Init"); - debug_config(hpet); - - true } unsafe fn debug_caps(hpet: &mut Hpet) { - let capability = hpet.read_u64(CAPABILITY_OFFSET); - println!(" caps: {:#x}", capability); - println!( - " clock period: {:?}", - Duration::from_nanos((capability >> 32) / 1_000_000) - ); - println!( - " ID: {:#x} revision: {}", - (capability >> 16) as u16, - capability as u8 - ); - println!( - " LEG_RT_CAP: {} COUNT_SIZE_CAP: {}", - capability & (1 << 15) == (1 << 15), - capability & (1 << 13) == (1 << 13) - ); - // The NUM_TIM_CAP field contains the index of the last timer. - // Add 1 to get the amount of timers. - println!(" timers: {}", (capability >> 8) as u8 & 0x1F + 1); + unsafe { + let capability = hpet.read_u64(CAPABILITY_OFFSET); + println!(" caps: {:#x}", capability); + println!( + " clock period: {:?}", + Duration::from_nanos((capability >> 32) / 1_000_000) + ); + println!( + " ID: {:#x} revision: {}", + (capability >> 16) as u16, + capability as u8 + ); + println!( + " LEG_RT_CAP: {} COUNT_SIZE_CAP: {}", + capability & (1 << 15) == (1 << 15), + capability & (1 << 13) == (1 << 13) + ); + // The NUM_TIM_CAP field contains the index of the last timer. + // Add 1 to get the amount of timers. + println!(" timers: {}", (capability >> 8) as u8 & 0x1F + 1); - let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET); - println!( - " T0 interrupt routing: {:#x}", - (t0_capabilities >> 32) as u32 - ); + let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET); + println!( + " T0 interrupt routing: {:#x}", + (t0_capabilities >> 32) as u32 + ); + } } unsafe fn debug_config(hpet: &mut Hpet) { - let config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET); - println!(" config: {:#x}", config_word); + unsafe { + let config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET); + println!(" config: {:#x}", config_word); - let interrupt_status = hpet.read_u64(GENERAL_INTERRUPT_OFFSET); - println!(" interrupt status: {:#x}", interrupt_status); + let interrupt_status = hpet.read_u64(GENERAL_INTERRUPT_OFFSET); + println!(" interrupt status: {:#x}", interrupt_status); - let counter = hpet.read_u64(MAIN_COUNTER_OFFSET); - println!(" counter: {:#x}", counter); + let counter = hpet.read_u64(MAIN_COUNTER_OFFSET); + println!(" counter: {:#x}", counter); - let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET); - println!(" T0 flags: {:#x}", t0_capabilities as u32); + let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET); + println!(" T0 flags: {:#x}", t0_capabilities as u32); - let t0_comparator = hpet.read_u64(T0_COMPARATOR_OFFSET); - println!(" T0 comparator: {:#x}", t0_comparator); + let t0_comparator = hpet.read_u64(T0_COMPARATOR_OFFSET); + println!(" T0 comparator: {:#x}", t0_comparator); + } } diff --git a/src/arch/x86_shared/device/ioapic.rs b/src/arch/x86_shared/device/ioapic.rs index 9fcce6d7e9..87013f48e2 100644 --- a/src/arch/x86_shared/device/ioapic.rs +++ b/src/arch/x86_shared/device/ioapic.rs @@ -238,188 +238,196 @@ pub fn src_overrides() -> &'static [Override] { #[cfg(feature = "acpi")] pub unsafe fn handle_ioapic(mapper: &mut KernelMapper, madt_ioapic: &'static MadtIoApic) { - // map the I/O APIC registers + unsafe { + // map the I/O APIC registers - let frame = Frame::containing(PhysicalAddress::new(madt_ioapic.address as usize)); - #[cfg(target_arch = "x86")] - let page = Page::containing_address(rmm::VirtualAddress::new(crate::IOAPIC_OFFSET)); - #[cfg(target_arch = "x86_64")] - let page = Page::containing_address(RmmA::phys_to_virt(frame.base())); + let frame = Frame::containing(PhysicalAddress::new(madt_ioapic.address as usize)); + #[cfg(target_arch = "x86")] + let page = Page::containing_address(rmm::VirtualAddress::new(crate::IOAPIC_OFFSET)); + #[cfg(target_arch = "x86_64")] + let page = Page::containing_address(RmmA::phys_to_virt(frame.base())); - assert!(mapper.translate(page.start_address()).is_none()); + assert!(mapper.translate(page.start_address()).is_none()); - mapper - .get_mut() - .expect("expected KernelMapper not to be locked re-entrant while mapping I/O APIC memory") - .map_phys( - page.start_address(), - frame.base(), - PageFlags::new() - .write(true) - .custom_flag(EntryFlags::NO_CACHE.bits(), true), - ) - .expect("failed to map I/O APIC") - .flush(); + mapper + .get_mut() + .expect( + "expected KernelMapper not to be locked re-entrant while mapping I/O APIC memory", + ) + .map_phys( + page.start_address(), + frame.base(), + PageFlags::new() + .write(true) + .custom_flag(EntryFlags::NO_CACHE.bits(), true), + ) + .expect("failed to map I/O APIC") + .flush(); - let ioapic_registers = page.start_address().data() as *const u32; - let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base); + let ioapic_registers = page.start_address().data() as *const u32; + let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base); - assert_eq!( - ioapic.regs.lock().id(), - madt_ioapic.id, - "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC" - ); + assert_eq!( + ioapic.regs.lock().id(), + madt_ioapic.id, + "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC" + ); - (*IOAPICS.get()).get_or_insert_with(Vec::new).push(ioapic); + (*IOAPICS.get()).get_or_insert_with(Vec::new).push(ioapic); + } } #[cfg(feature = "acpi")] pub unsafe fn handle_src_override(src_override: &'static MadtIntSrcOverride) { - let flags = src_override.flags; + unsafe { + let flags = src_override.flags; - let polarity_raw = (flags & 0x0003) as u8; - let trigger_mode_raw = ((flags & 0x000C) >> 2) as u8; + let polarity_raw = (flags & 0x0003) as u8; + let trigger_mode_raw = ((flags & 0x000C) >> 2) as u8; - let polarity = match polarity_raw { - 0b00 => Polarity::ConformsToSpecs, - 0b01 => Polarity::ActiveHigh, - 0b10 => return, // reserved - 0b11 => Polarity::ActiveLow, + let polarity = match polarity_raw { + 0b00 => Polarity::ConformsToSpecs, + 0b01 => Polarity::ActiveHigh, + 0b10 => return, // reserved + 0b11 => Polarity::ActiveLow, - _ => unreachable!(), - }; + _ => unreachable!(), + }; - let trigger_mode = match trigger_mode_raw { - 0b00 => TriggerMode::ConformsToSpecs, - 0b01 => TriggerMode::Edge, - 0b10 => return, // reserved - 0b11 => TriggerMode::Level, - _ => unreachable!(), - }; + let trigger_mode = match trigger_mode_raw { + 0b00 => TriggerMode::ConformsToSpecs, + 0b01 => TriggerMode::Edge, + 0b10 => return, // reserved + 0b11 => TriggerMode::Level, + _ => unreachable!(), + }; - let over = Override { - bus_irq: src_override.irq_source, - gsi: src_override.gsi_base, - polarity, - trigger_mode, - }; - (*SRC_OVERRIDES.get()) - .get_or_insert_with(Vec::new) - .push(over); + let over = Override { + bus_irq: src_override.irq_source, + gsi: src_override.gsi_base, + polarity, + trigger_mode, + }; + (*SRC_OVERRIDES.get()) + .get_or_insert_with(Vec::new) + .push(over); + } } #[allow(dead_code)] pub unsafe fn init(active_table: &mut KernelMapper) { - let bsp_apic_id = ApicId::new(u32::from( - cpuid().get_feature_info().unwrap().initial_local_apic_id(), - )); // TODO: remove unwraps + unsafe { + let bsp_apic_id = ApicId::new(u32::from( + cpuid().get_feature_info().unwrap().initial_local_apic_id(), + )); // TODO: remove unwraps - // search the madt for all IOAPICs. - #[cfg(feature = "acpi")] - { - let madt: &'static Madt = match madt::madt() { - Some(m) => m, - // TODO: Parse MP tables too. - None => return, - }; - if madt.flags & madt::FLAG_PCAT != 0 { - pic::disable(); - } + // search the madt for all IOAPICs. + #[cfg(feature = "acpi")] + { + let madt: &'static Madt = match madt::madt() { + Some(m) => m, + // TODO: Parse MP tables too. + None => return, + }; + if madt.flags & madt::FLAG_PCAT != 0 { + pic::disable(); + } - // find all I/O APICs (usually one). + // find all I/O APICs (usually one). - for entry in madt.iter() { - match entry { - MadtEntry::IoApic(ioapic) => handle_ioapic(active_table, ioapic), - MadtEntry::IntSrcOverride(src_override) => handle_src_override(src_override), - _ => (), + for entry in madt.iter() { + match entry { + MadtEntry::IoApic(ioapic) => handle_ioapic(active_table, ioapic), + MadtEntry::IntSrcOverride(src_override) => handle_src_override(src_override), + _ => (), + } } } - } - println!( - "I/O APICs: {:?}, overrides: {:?}", - ioapics(), - src_overrides() - ); + println!( + "I/O APICs: {:?}, overrides: {:?}", + ioapics(), + src_overrides() + ); - // map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it - // wouldn't have been disabled due to this I/O APIC) - for legacy_irq in 0..=15 { - let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) { - Some(over) => (over.gsi, over.trigger_mode, over.polarity), - None => { - if src_overrides() - .iter() - .any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq) - && !src_overrides() + // map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it + // wouldn't have been disabled due to this I/O APIC) + for legacy_irq in 0..=15 { + let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) { + Some(over) => (over.gsi, over.trigger_mode, over.polarity), + None => { + if src_overrides() .iter() - .any(|over| over.bus_irq == legacy_irq) - { - // there's an IRQ conflict, making this legacy IRQ inaccessible. + .any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq) + && !src_overrides() + .iter() + .any(|over| over.bus_irq == legacy_irq) + { + // there's an IRQ conflict, making this legacy IRQ inaccessible. + continue; + } + ( + legacy_irq.into(), + TriggerMode::ConformsToSpecs, + Polarity::ConformsToSpecs, + ) + } + }; + let apic = match find_ioapic(gsi) { + Some(ioapic) => ioapic, + None => { + println!("Unable to find a suitable APIC for legacy IRQ {} (GSI {}). It will not be mapped.", legacy_irq, gsi); continue; } - ( - legacy_irq.into(), - TriggerMode::ConformsToSpecs, - Polarity::ConformsToSpecs, - ) - } - }; - let apic = match find_ioapic(gsi) { - Some(ioapic) => ioapic, - None => { - println!("Unable to find a suitable APIC for legacy IRQ {} (GSI {}). It will not be mapped.", legacy_irq, gsi); - continue; - } - }; - let redir_tbl_index = (gsi - apic.gsi_start) as u8; + }; + let redir_tbl_index = (gsi - apic.gsi_start) as u8; - let map_info = MapInfo { - // only send to the BSP - dest: bsp_apic_id, - dest_mode: DestinationMode::Physical, - delivery_mode: DeliveryMode::Fixed, - mask: false, - polarity: match polarity { - Polarity::ActiveHigh => ApicPolarity::ActiveHigh, - Polarity::ActiveLow => ApicPolarity::ActiveLow, - Polarity::ConformsToSpecs => ApicPolarity::ActiveHigh, - }, - trigger_mode: match trigger_mode { - TriggerMode::Edge => ApicTriggerMode::Edge, - TriggerMode::Level => ApicTriggerMode::Level, - TriggerMode::ConformsToSpecs => ApicTriggerMode::Edge, - }, - vector: 32 + legacy_irq, - }; - apic.map(redir_tbl_index, map_info); - } - println!( - "I/O APICs: {:?}, overrides: {:?}", - ioapics(), - src_overrides() - ); - irq::set_irq_method(irq::IrqMethod::Apic); - - // tell the firmware that we're using APIC rather than the default 8259 PIC. - - // FIXME: With ACPI moved to userspace, we should instead allow userspace to check whether the - // IOAPIC has been initialized, and then subsequently let some ACPI driver call the AML from - // userspace. - - /*#[cfg(feature = "acpi")] - { - let method = { - let namespace_guard = crate::acpi::ACPI_TABLE.namespace.read(); - if let Some(value) = namespace_guard.as_ref().unwrap().get("\\_PIC") { - value.get_as_method().ok() - } else { - None - } - }; - if let Some(m) = method { - m.execute("\\_PIC".into(), vec!(crate::acpi::aml::AmlValue::Integer(1))); + let map_info = MapInfo { + // only send to the BSP + dest: bsp_apic_id, + dest_mode: DestinationMode::Physical, + delivery_mode: DeliveryMode::Fixed, + mask: false, + polarity: match polarity { + Polarity::ActiveHigh => ApicPolarity::ActiveHigh, + Polarity::ActiveLow => ApicPolarity::ActiveLow, + Polarity::ConformsToSpecs => ApicPolarity::ActiveHigh, + }, + trigger_mode: match trigger_mode { + TriggerMode::Edge => ApicTriggerMode::Edge, + TriggerMode::Level => ApicTriggerMode::Level, + TriggerMode::ConformsToSpecs => ApicTriggerMode::Edge, + }, + vector: 32 + legacy_irq, + }; + apic.map(redir_tbl_index, map_info); } - }*/ + println!( + "I/O APICs: {:?}, overrides: {:?}", + ioapics(), + src_overrides() + ); + irq::set_irq_method(irq::IrqMethod::Apic); + + // tell the firmware that we're using APIC rather than the default 8259 PIC. + + // FIXME: With ACPI moved to userspace, we should instead allow userspace to check whether the + // IOAPIC has been initialized, and then subsequently let some ACPI driver call the AML from + // userspace. + + /*#[cfg(feature = "acpi")] + { + let method = { + let namespace_guard = crate::acpi::ACPI_TABLE.namespace.read(); + if let Some(value) = namespace_guard.as_ref().unwrap().get("\\_PIC") { + value.get_as_method().ok() + } else { + None + } + }; + if let Some(m) = method { + m.execute("\\_PIC".into(), vec!(crate::acpi::aml::AmlValue::Integer(1))); + } + }*/ + } } fn get_override(irq: u8) -> Option<&'static Override> { src_overrides().iter().find(|over| over.bus_irq == irq) diff --git a/src/arch/x86_shared/device/local_apic.rs b/src/arch/x86_shared/device/local_apic.rs index 14b10bd98c..0f3dd17338 100644 --- a/src/arch/x86_shared/device/local_apic.rs +++ b/src/arch/x86_shared/device/local_apic.rs @@ -30,15 +30,19 @@ static LOCAL_APIC: SyncUnsafeCell = SyncUnsafeCell::new(LocalApic { x2: false, }); pub unsafe fn the_local_apic() -> &'static mut LocalApic { - &mut *LOCAL_APIC.get() + unsafe { &mut *LOCAL_APIC.get() } } pub unsafe fn init(active_table: &mut KernelMapper) { - the_local_apic().init(active_table); + unsafe { + the_local_apic().init(active_table); + } } pub unsafe fn init_ap() { - the_local_apic().init_ap(); + unsafe { + the_local_apic().init_ap(); + } } /// Local APIC @@ -49,63 +53,69 @@ pub struct LocalApic { impl LocalApic { unsafe fn init(&mut self, mapper: &mut KernelMapper) { - let mapper = mapper - .get_mut() - .expect("expected KernelMapper not to be locked re-entrant while initializing LAPIC"); + unsafe { + let mapper = mapper.get_mut().expect( + "expected KernelMapper not to be locked re-entrant while initializing LAPIC", + ); - let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000); - #[cfg(target_arch = "x86")] - let virtaddr = rmm::VirtualAddress::new(crate::LAPIC_OFFSET); - #[cfg(target_arch = "x86_64")] - let virtaddr = { - use rmm::Arch; - crate::memory::RmmA::phys_to_virt(physaddr) - }; + let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000); + #[cfg(target_arch = "x86")] + let virtaddr = rmm::VirtualAddress::new(crate::LAPIC_OFFSET); + #[cfg(target_arch = "x86_64")] + let virtaddr = { + use rmm::Arch; + crate::memory::RmmA::phys_to_virt(physaddr) + }; - self.address = virtaddr.data(); - self.x2 = cpuid() - .get_feature_info() - .map_or(false, |feature_info| feature_info.has_x2apic()); + self.address = virtaddr.data(); + self.x2 = cpuid() + .get_feature_info() + .map_or(false, |feature_info| feature_info.has_x2apic()); - if !self.x2 { - log::info!("Detected xAPIC at {:#x}", physaddr.data()); - if let Some((_entry, _, flush)) = mapper.unmap_phys(virtaddr, true) { - // Unmap xAPIC page if already mapped - flush.flush(); + if !self.x2 { + log::info!("Detected xAPIC at {:#x}", physaddr.data()); + if let Some((_entry, _, flush)) = mapper.unmap_phys(virtaddr, true) { + // Unmap xAPIC page if already mapped + flush.flush(); + } + mapper + .map_phys(virtaddr, physaddr, PageFlags::new().write(true)) + .expect("failed to map local APIC memory") + .flush(); + } else { + log::info!("Detected x2APIC"); } - mapper - .map_phys(virtaddr, physaddr, PageFlags::new().write(true)) - .expect("failed to map local APIC memory") - .flush(); - } else { - log::info!("Detected x2APIC"); - } - self.init_ap(); + self.init_ap(); + } } unsafe fn init_ap(&mut self) { - if self.x2 { - wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | 1 << 10); - wrmsr(IA32_X2APIC_SIVR, 0x100); - } else { - self.write(0xF0, 0x100); - } - self.setup_error_int(); - //self.setup_timer(); + unsafe { + if self.x2 { + wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | 1 << 10); + wrmsr(IA32_X2APIC_SIVR, 0x100); + } else { + self.write(0xF0, 0x100); + } + self.setup_error_int(); + //self.setup_timer(); - PercpuBlock::current() - .misc_arch_info - .apic_id_opt - .set(Some(self.id())); + PercpuBlock::current() + .misc_arch_info + .apic_id_opt + .set(Some(self.id())); + } } unsafe fn read(&self, reg: u32) -> u32 { - read_volatile((self.address + reg as usize) as *const u32) + unsafe { read_volatile((self.address + reg as usize) as *const u32) } } unsafe fn write(&mut self, reg: u32, value: u32) { - write_volatile((self.address + reg as usize) as *mut u32, value); + unsafe { + write_volatile((self.address + reg as usize) as *mut u32, value); + } } pub fn id(&self) -> ApicId { @@ -162,90 +172,114 @@ impl LocalApic { } pub unsafe fn eoi(&mut self) { - if self.x2 { - wrmsr(IA32_X2APIC_EOI, 0); - } else { - self.write(0xB0, 0); + unsafe { + if self.x2 { + wrmsr(IA32_X2APIC_EOI, 0); + } else { + self.write(0xB0, 0); + } } } /// Reads the Error Status Register. pub unsafe fn esr(&mut self) -> u32 { - if self.x2 { - // update the ESR to the current state of the local apic. - wrmsr(IA32_X2APIC_ESR, 0); - // read the updated value - rdmsr(IA32_X2APIC_ESR) as u32 - } else { - self.write(0x280, 0); - self.read(0x280) + unsafe { + if self.x2 { + // update the ESR to the current state of the local apic. + wrmsr(IA32_X2APIC_ESR, 0); + // read the updated value + rdmsr(IA32_X2APIC_ESR) as u32 + } else { + self.write(0x280, 0); + self.read(0x280) + } } } pub unsafe fn lvt_timer(&mut self) -> u32 { - if self.x2 { - rdmsr(IA32_X2APIC_LVT_TIMER) as u32 - } else { - self.read(0x320) + unsafe { + if self.x2 { + rdmsr(IA32_X2APIC_LVT_TIMER) as u32 + } else { + self.read(0x320) + } } } pub unsafe fn set_lvt_timer(&mut self, value: u32) { - if self.x2 { - wrmsr(IA32_X2APIC_LVT_TIMER, u64::from(value)); - } else { - self.write(0x320, value); + unsafe { + if self.x2 { + wrmsr(IA32_X2APIC_LVT_TIMER, u64::from(value)); + } else { + self.write(0x320, value); + } } } pub unsafe fn init_count(&mut self) -> u32 { - if self.x2 { - rdmsr(IA32_X2APIC_INIT_COUNT) as u32 - } else { - self.read(0x380) + unsafe { + if self.x2 { + rdmsr(IA32_X2APIC_INIT_COUNT) as u32 + } else { + self.read(0x380) + } } } pub unsafe fn set_init_count(&mut self, initial_count: u32) { - if self.x2 { - wrmsr(IA32_X2APIC_INIT_COUNT, u64::from(initial_count)); - } else { - self.write(0x380, initial_count); + unsafe { + if self.x2 { + wrmsr(IA32_X2APIC_INIT_COUNT, u64::from(initial_count)); + } else { + self.write(0x380, initial_count); + } } } pub unsafe fn cur_count(&mut self) -> u32 { - if self.x2 { - rdmsr(IA32_X2APIC_CUR_COUNT) as u32 - } else { - self.read(0x390) + unsafe { + if self.x2 { + rdmsr(IA32_X2APIC_CUR_COUNT) as u32 + } else { + self.read(0x390) + } } } pub unsafe fn div_conf(&mut self) -> u32 { - if self.x2 { - rdmsr(IA32_X2APIC_DIV_CONF) as u32 - } else { - self.read(0x3E0) + unsafe { + if self.x2 { + rdmsr(IA32_X2APIC_DIV_CONF) as u32 + } else { + self.read(0x3E0) + } } } pub unsafe fn set_div_conf(&mut self, div_conf: u32) { - if self.x2 { - wrmsr(IA32_X2APIC_DIV_CONF, u64::from(div_conf)); - } else { - self.write(0x3E0, div_conf); + unsafe { + if self.x2 { + wrmsr(IA32_X2APIC_DIV_CONF, u64::from(div_conf)); + } else { + self.write(0x3E0, div_conf); + } } } pub unsafe fn lvt_error(&mut self) -> u32 { - if self.x2 { - rdmsr(IA32_X2APIC_LVT_ERROR) as u32 - } else { - self.read(0x370) + unsafe { + if self.x2 { + rdmsr(IA32_X2APIC_LVT_ERROR) as u32 + } else { + self.read(0x370) + } } } pub unsafe fn set_lvt_error(&mut self, lvt_error: u32) { - if self.x2 { - wrmsr(IA32_X2APIC_LVT_ERROR, u64::from(lvt_error)); - } else { - self.write(0x370, lvt_error); + unsafe { + if self.x2 { + wrmsr(IA32_X2APIC_LVT_ERROR, u64::from(lvt_error)); + } else { + self.write(0x370, lvt_error); + } } } unsafe fn setup_error_int(&mut self) { - let vector = 49u32; - self.set_lvt_error(vector); + unsafe { + let vector = 49u32; + self.set_lvt_error(vector); + } } } diff --git a/src/arch/x86_shared/device/mod.rs b/src/arch/x86_shared/device/mod.rs index 0688dbd530..6e8b17db9b 100644 --- a/src/arch/x86_shared/device/mod.rs +++ b/src/arch/x86_shared/device/mod.rs @@ -16,11 +16,13 @@ pub mod system76_ec; pub mod tsc; pub unsafe fn init() { - pic::init(); - local_apic::init(&mut KernelMapper::lock()); + unsafe { + pic::init(); + local_apic::init(&mut KernelMapper::lock()); - // Run here for the side-effect of printing if KVM was used to avoid interleaved logs. - tsc::get_kvm_support(); + // Run here for the side-effect of printing if KVM was used to avoid interleaved logs. + tsc::get_kvm_support(); + } } pub unsafe fn init_after_acpi() { // this will disable the IOAPIC if needed. @@ -29,16 +31,19 @@ pub unsafe fn init_after_acpi() { #[cfg(feature = "acpi")] unsafe fn init_hpet() -> bool { - use crate::acpi::ACPI_TABLE; - if let Some(ref mut hpet) = *ACPI_TABLE.hpet.write() { - if cfg!(target_arch = "x86") { - //TODO: fix HPET on i686 - log::warn!("HPET found but implemented on i686"); - return false; + unsafe { + use crate::acpi::ACPI_TABLE; + match *ACPI_TABLE.hpet.write() { + Some(ref mut hpet) => { + if cfg!(target_arch = "x86") { + //TODO: fix HPET on i686 + log::warn!("HPET found but implemented on i686"); + return false; + } + hpet::init(hpet) + } + _ => false, } - hpet::init(hpet) - } else { - false } } @@ -48,30 +53,34 @@ unsafe fn init_hpet() -> bool { } pub unsafe fn init_noncore() { - log::info!("Initializing system timer"); + unsafe { + log::info!("Initializing system timer"); - #[cfg(feature = "x86_kvm_pv")] - if tsc::init() { - log::info!("TSC used as system clock source"); + #[cfg(feature = "x86_kvm_pv")] + if tsc::init() { + log::info!("TSC used as system clock source"); + } + + if init_hpet() { + log::info!("HPET used as system timer"); + } else { + pit::init(); + log::info!("PIT used as system timer"); + } + + log::info!("Initializing serial"); + serial::init(); + log::info!("Finished initializing devices"); } - - if init_hpet() { - log::info!("HPET used as system timer"); - } else { - pit::init(); - log::info!("PIT used as system timer"); - } - - log::info!("Initializing serial"); - serial::init(); - log::info!("Finished initializing devices"); } pub unsafe fn init_ap() { - local_apic::init_ap(); + unsafe { + local_apic::init_ap(); - #[cfg(feature = "x86_kvm_pv")] - tsc::init(); + #[cfg(feature = "x86_kvm_pv")] + tsc::init(); + } } #[derive(Default)] diff --git a/src/arch/x86_shared/device/pic.rs b/src/arch/x86_shared/device/pic.rs index 79fe74b28a..0cf54ba801 100644 --- a/src/arch/x86_shared/device/pic.rs +++ b/src/arch/x86_shared/device/pic.rs @@ -10,48 +10,52 @@ static SLAVE: SyncUnsafeCell = SyncUnsafeCell::new(Pic::new(0xA0)); // SAFETY: must be main thread pub unsafe fn master<'a>() -> &'a mut Pic { - &mut *MASTER.get() + unsafe { &mut *MASTER.get() } } // SAFETY: must be main thread pub unsafe fn slave<'a>() -> &'a mut Pic { - &mut *SLAVE.get() + unsafe { &mut *SLAVE.get() } } pub unsafe fn init() { - let master = master(); - let slave = slave(); + unsafe { + let master = master(); + let slave = slave(); - // Start initialization - master.cmd.write(0x11); - slave.cmd.write(0x11); + // Start initialization + master.cmd.write(0x11); + slave.cmd.write(0x11); - // Set offsets - master.data.write(0x20); - slave.data.write(0x28); + // Set offsets + master.data.write(0x20); + slave.data.write(0x28); - // Set up cascade - master.data.write(4); - slave.data.write(2); + // Set up cascade + master.data.write(4); + slave.data.write(2); - // Set up interrupt mode (1 is 8086/88 mode, 2 is auto EOI) - master.data.write(1); - slave.data.write(1); + // Set up interrupt mode (1 is 8086/88 mode, 2 is auto EOI) + master.data.write(1); + slave.data.write(1); - // Unmask interrupts - master.data.write(0); - slave.data.write(0); + // Unmask interrupts + master.data.write(0); + slave.data.write(0); - // Ack remaining interrupts - master.ack(); - slave.ack(); + // Ack remaining interrupts + master.ack(); + slave.ack(); - // probably already set to PIC, but double-check - irq::set_irq_method(irq::IrqMethod::Pic); + // probably already set to PIC, but double-check + irq::set_irq_method(irq::IrqMethod::Pic); + } } pub unsafe fn disable() { - master().data.write(0xFF); - slave().data.write(0xFF); + unsafe { + master().data.write(0xFF); + slave().data.write(0xFF); + } } pub struct Pic { diff --git a/src/arch/x86_shared/device/pit.rs b/src/arch/x86_shared/device/pit.rs index e275a972c4..d8013ee58c 100644 --- a/src/arch/x86_shared/device/pit.rs +++ b/src/arch/x86_shared/device/pit.rs @@ -9,11 +9,11 @@ static COMMAND: SyncUnsafeCell> = SyncUnsafeCell::new(Pio::new(0x43)); // SAFETY: must be externally syncd pub unsafe fn chan0<'a>() -> &'a mut Pio { - &mut *CHAN0.get() + unsafe { &mut *CHAN0.get() } } // SAFETY: must be externally syncd pub unsafe fn command<'a>() -> &'a mut Pio { - &mut *COMMAND.get() + unsafe { &mut *COMMAND.get() } } const SELECT_CHAN0: u8 = 0b00 << 6; @@ -31,16 +31,20 @@ pub const CHAN0_DIVISOR: u16 = 4847; pub const RATE: u128 = (CHAN0_DIVISOR as u128 * PERIOD_FS) / 1_000_000; pub unsafe fn init() { - command().write(SELECT_CHAN0 | ACCESS_LOHI | MODE_2); - chan0().write(CHAN0_DIVISOR as u8); - chan0().write((CHAN0_DIVISOR >> 8) as u8); + unsafe { + command().write(SELECT_CHAN0 | ACCESS_LOHI | MODE_2); + chan0().write(CHAN0_DIVISOR as u8); + chan0().write((CHAN0_DIVISOR >> 8) as u8); + } } pub unsafe fn read() -> u16 { - command().write(SELECT_CHAN0 | ACCESS_LATCH); - let low = chan0().read(); - let high = chan0().read(); - let counter = ((high as u16) << 8) | (low as u16); - // Counter is inverted, subtract from CHAN0_DIVISOR - CHAN0_DIVISOR.saturating_sub(counter) + unsafe { + command().write(SELECT_CHAN0 | ACCESS_LATCH); + let low = chan0().read(); + let high = chan0().read(); + let counter = ((high as u16) << 8) | (low as u16); + // Counter is inverted, subtract from CHAN0_DIVISOR + CHAN0_DIVISOR.saturating_sub(counter) + } } diff --git a/src/arch/x86_shared/device/tsc.rs b/src/arch/x86_shared/device/tsc.rs index a8f7c72a43..4b9886e2df 100644 --- a/src/arch/x86_shared/device/tsc.rs +++ b/src/arch/x86_shared/device/tsc.rs @@ -121,39 +121,41 @@ pub fn get_kvm_support() -> &'static Option { } pub unsafe fn init() -> bool { - let cpuid = crate::cpuid::cpuid(); - if !cpuid.get_feature_info().map_or(false, |f| f.has_tsc()) { - return false; - } + unsafe { + let cpuid = crate::cpuid::cpuid(); + if !cpuid.get_feature_info().map_or(false, |f| f.has_tsc()) { + return false; + } - let kvm_support = get_kvm_support(); + let kvm_support = get_kvm_support(); - if let Some(kvm_support) = kvm_support - && kvm_support - .supp_feats - .contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE) - { - let frame = allocate_frame().expect("failed to allocate timer page"); - x86::msr::wrmsr(MSR_KVM_SYSTEM_TIME_NEW, (frame.base().data() as u64) | 1); - let ptr = - crate::paging::RmmA::phys_to_virt(frame.base()).data() as *const PvclockVcpuTimeInfo; - PercpuBlock::current() - .misc_arch_info - .tsc_info - .vcpu_page - .set(ptr); + if let Some(kvm_support) = kvm_support + && kvm_support + .supp_feats + .contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE) + { + let frame = allocate_frame().expect("failed to allocate timer page"); + x86::msr::wrmsr(MSR_KVM_SYSTEM_TIME_NEW, (frame.base().data() as u64) | 1); + let ptr = crate::paging::RmmA::phys_to_virt(frame.base()).data() + as *const PvclockVcpuTimeInfo; + PercpuBlock::current() + .misc_arch_info + .tsc_info + .vcpu_page + .set(ptr); - /*let tsc_ghz = loop { - let val1 = ptr.read_volatile(); - let val2 = ptr.read_volatile(); - if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version { - continue; - } - let val1 - break tsc_hz / 1_000_000_000; - };*/ - true - } else { - false + /*let tsc_ghz = loop { + let val1 = ptr.read_volatile(); + let val2 = ptr.read_volatile(); + if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version { + continue; + } + let val1 + break tsc_hz / 1_000_000_000; + };*/ + true + } else { + false + } } } diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 899c510936..e9d7a7148f 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -92,23 +92,25 @@ pub fn available_irqs_iter(cpu_id: LogicalCpuId) -> impl Iterator + ' #[cfg(target_arch = "x86")] macro_rules! use_irq( - ( $idt: expr, $number:literal, $func:ident ) => {{ + ( $idt: expr_2021, $number:literal, $func:ident ) => {{ $idt[$number].set_func($func); }} ); #[cfg(target_arch = "x86")] macro_rules! use_default_irqs( - ($idt:expr) => {{ + ($idt:expr_2021) => {{ use crate::interrupt::irq::*; default_irqs!($idt, use_irq); }} ); pub unsafe fn init() { - let idt = &mut *INIT_IDT.get(); - set_exceptions(idt); - dtables::lidt(&DescriptorTablePointer::new(&idt)); + unsafe { + let idt = &mut *INIT_IDT.get(); + set_exceptions(idt); + dtables::lidt(&DescriptorTablePointer::new(&idt)); + } } fn set_exceptions(idt: &mut [IdtEntry]) { @@ -155,143 +157,150 @@ const fn new_idt_reservations() -> [AtomicU32; 8] { /// Initialize the IDT for a processor pub unsafe fn init_paging_post_heap(cpu_id: LogicalCpuId) { - let mut idts_guard = IDTS.write(); - let idts_btree = idts_guard.get_or_insert_with(HashMap::new); + unsafe { + let mut idts_guard = IDTS.write(); + let idts_btree = idts_guard.get_or_insert_with(HashMap::new); - if cpu_id == LogicalCpuId::BSP { - idts_btree.insert(cpu_id, &mut *INIT_BSP_IDT.get()); - } else { - let idt = idts_btree - .entry(cpu_id) - .or_insert_with(|| Box::leak(Box::new(Idt::new()))); - init_generic(cpu_id, idt); + if cpu_id == LogicalCpuId::BSP { + idts_btree.insert(cpu_id, &mut *INIT_BSP_IDT.get()); + } else { + let idt = idts_btree + .entry(cpu_id) + .or_insert_with(|| Box::leak(Box::new(Idt::new()))); + init_generic(cpu_id, idt); + } } } /// Initializes a fully functional IDT for use before it be moved into the map. This is ONLY called /// on the BSP, since the kernel heap is ready for the APs. pub unsafe fn init_paging_bsp() { - init_generic(LogicalCpuId::BSP, &mut *INIT_BSP_IDT.get()); + unsafe { + init_generic(LogicalCpuId::BSP, &mut *INIT_BSP_IDT.get()); + } } /// Initializes an IDT for any type of processor. pub unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) { - let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations); + unsafe { + let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations); - let idtr: DescriptorTablePointer = DescriptorTablePointer { - limit: (current_idt.len() * mem::size_of::() - 1) as u16, - base: current_idt.as_ptr() as *const X86IdtEntry, - }; + let idtr: DescriptorTablePointer = DescriptorTablePointer { + limit: (current_idt.len() * mem::size_of::() - 1) as u16, + base: current_idt.as_ptr() as *const X86IdtEntry, + }; - let backup_ist = { - // We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate - // stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are - // considered the most fatal, especially Double Faults which are caused by errors __when - // accessing the system IDT__. If that goes wrong, then kernel memory may be partially - // corrupt, and we want a separate stack. - // - // Note that each CPU has its own "backup interrupt stack". - let index = 1_u8; + let backup_ist = { + // We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate + // stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are + // considered the most fatal, especially Double Faults which are caused by errors __when + // accessing the system IDT__. If that goes wrong, then kernel memory may be partially + // corrupt, and we want a separate stack. + // + // Note that each CPU has its own "backup interrupt stack". + let index = 1_u8; - // Put them in the 1st entry of the IST. - #[cfg(target_arch = "x86_64")] // TODO: x86 - { - use crate::paging::PAGE_SIZE; - // Allocate 64 KiB of stack space for the backup stack. - const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4; - let frames = crate::memory::allocate_p2frame(4) - .expect("failed to allocate pages for backup interrupt stack"); + // Put them in the 1st entry of the IST. + #[cfg(target_arch = "x86_64")] // TODO: x86 + { + use crate::paging::PAGE_SIZE; + // Allocate 64 KiB of stack space for the backup stack. + const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4; + let frames = crate::memory::allocate_p2frame(4) + .expect("failed to allocate pages for backup interrupt stack"); - use crate::paging::{RmmA, RmmArch}; + use crate::paging::{RmmA, RmmArch}; - // Physical pages are mapped linearly. So is the linearly mapped virtual memory. - let base_address = RmmA::phys_to_virt(frames.base()); + // Physical pages are mapped linearly. So is the linearly mapped virtual memory. + let base_address = RmmA::phys_to_virt(frames.base()); - // Stack always grows downwards. - let address = base_address.data() + BACKUP_STACK_SIZE; + // Stack always grows downwards. + let address = base_address.data() + BACKUP_STACK_SIZE; - (*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64; + (*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64; + } + + index + }; + + set_exceptions(current_idt); + current_idt[2].set_ist(backup_ist); + current_idt[8].set_ist(backup_ist); + current_idt[18].set_ist(backup_ist); + + #[cfg(target_arch = "x86_64")] + assert_eq!( + __generic_interrupts_end as usize - __generic_interrupts_start as usize, + 224 * 8 + ); + + #[cfg(target_arch = "x86_64")] + for i in 0..224 { + current_idt[i + 32] + .set_func(mem::transmute(__generic_interrupts_start as usize + i * 8)); } - index - }; + // reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions + *current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF; - set_exceptions(current_idt); - current_idt[2].set_ist(backup_ist); - current_idt[8].set_ist(backup_ist); - current_idt[18].set_ist(backup_ist); + if cpu_id == LogicalCpuId::BSP { + // Set up IRQs + current_idt[32].set_func(irq::pit_stack); + current_idt[33].set_func(irq::keyboard); + current_idt[34].set_func(irq::cascade); + current_idt[35].set_func(irq::com2); + current_idt[36].set_func(irq::com1); + current_idt[37].set_func(irq::lpt2); + current_idt[38].set_func(irq::floppy); + current_idt[39].set_func(irq::lpt1); + current_idt[40].set_func(irq::rtc); + current_idt[41].set_func(irq::pci1); + current_idt[42].set_func(irq::pci2); + current_idt[43].set_func(irq::pci3); + current_idt[44].set_func(irq::mouse); + current_idt[45].set_func(irq::fpu); + current_idt[46].set_func(irq::ata1); + current_idt[47].set_func(irq::ata2); + current_idt[48].set_func(irq::lapic_timer); + current_idt[49].set_func(irq::lapic_error); - #[cfg(target_arch = "x86_64")] - assert_eq!( - __generic_interrupts_end as usize - __generic_interrupts_start as usize, - 224 * 8 - ); + // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error. + *current_reservations[1].get_mut() |= 0x0003_FFFF; + } else { + // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU + current_idt[49].set_func(irq::lapic_error); - #[cfg(target_arch = "x86_64")] - for i in 0..224 { - current_idt[i + 32].set_func(mem::transmute(__generic_interrupts_start as usize + i * 8)); + // reserve bit 49 + *current_reservations[1].get_mut() |= 1 << 17; + } + + #[cfg(target_arch = "x86")] + use_default_irqs!(current_idt); + + // Set IPI handlers + current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup); + current_idt[IpiKind::Switch as usize].set_func(ipi::switch); + current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb); + current_idt[IpiKind::Pit as usize].set_func(ipi::pit); + idt.set_reserved_mut(IpiKind::Wakeup as u8, true); + idt.set_reserved_mut(IpiKind::Switch as u8, true); + idt.set_reserved_mut(IpiKind::Tlb as u8, true); + idt.set_reserved_mut(IpiKind::Pit as u8, true); + + #[cfg(target_arch = "x86")] + { + let current_idt = &mut idt.entries; + // Set syscall function + current_idt[0x80].set_func(syscall::syscall); + current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); + idt.set_reserved_mut(0x80, true); + } + + #[cfg(feature = "profiling")] + crate::profiling::maybe_setup_timer(idt, cpu_id); + + dtables::lidt(&idtr); } - - // reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions - *current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF; - - if cpu_id == LogicalCpuId::BSP { - // Set up IRQs - current_idt[32].set_func(irq::pit_stack); - current_idt[33].set_func(irq::keyboard); - current_idt[34].set_func(irq::cascade); - current_idt[35].set_func(irq::com2); - current_idt[36].set_func(irq::com1); - current_idt[37].set_func(irq::lpt2); - current_idt[38].set_func(irq::floppy); - current_idt[39].set_func(irq::lpt1); - current_idt[40].set_func(irq::rtc); - current_idt[41].set_func(irq::pci1); - current_idt[42].set_func(irq::pci2); - current_idt[43].set_func(irq::pci3); - current_idt[44].set_func(irq::mouse); - current_idt[45].set_func(irq::fpu); - current_idt[46].set_func(irq::ata1); - current_idt[47].set_func(irq::ata2); - current_idt[48].set_func(irq::lapic_timer); - current_idt[49].set_func(irq::lapic_error); - - // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error. - *current_reservations[1].get_mut() |= 0x0003_FFFF; - } else { - // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU - current_idt[49].set_func(irq::lapic_error); - - // reserve bit 49 - *current_reservations[1].get_mut() |= 1 << 17; - } - - #[cfg(target_arch = "x86")] - use_default_irqs!(current_idt); - - // Set IPI handlers - current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup); - current_idt[IpiKind::Switch as usize].set_func(ipi::switch); - current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb); - current_idt[IpiKind::Pit as usize].set_func(ipi::pit); - idt.set_reserved_mut(IpiKind::Wakeup as u8, true); - idt.set_reserved_mut(IpiKind::Switch as u8, true); - idt.set_reserved_mut(IpiKind::Tlb as u8, true); - idt.set_reserved_mut(IpiKind::Pit as u8, true); - - #[cfg(target_arch = "x86")] - { - let current_idt = &mut idt.entries; - // Set syscall function - current_idt[0x80].set_func(syscall::syscall); - current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); - idt.set_reserved_mut(0x80, true); - } - - #[cfg(feature = "profiling")] - crate::profiling::maybe_setup_timer(idt, cpu_id); - - dtables::lidt(&idtr); } bitflags! { diff --git a/src/arch/x86_shared/interrupt/mod.rs b/src/arch/x86_shared/interrupt/mod.rs index 4d794fb73a..a01e7dbfd7 100644 --- a/src/arch/x86_shared/interrupt/mod.rs +++ b/src/arch/x86_shared/interrupt/mod.rs @@ -8,7 +8,9 @@ pub use super::idt::{available_irqs_iter, is_reserved, set_reserved}; /// Clear interrupts #[inline(always)] pub unsafe fn disable() { - core::arch::asm!("cli", options(nomem, nostack)); + unsafe { + core::arch::asm!("cli", options(nomem, nostack)); + } } /// Set interrupts and halt @@ -16,7 +18,9 @@ pub unsafe fn disable() { /// Performing enable followed by halt is not guaranteed to be atomic, use this instead! #[inline(always)] pub unsafe fn enable_and_halt() { - core::arch::asm!("sti; hlt", options(nomem, nostack)); + unsafe { + core::arch::asm!("sti; hlt", options(nomem, nostack)); + } } /// Set interrupts and nop @@ -24,13 +28,17 @@ pub unsafe fn enable_and_halt() { /// Simply enabling interrupts does not gurantee that they will trigger, use this instead! #[inline(always)] pub unsafe fn enable_and_nop() { - core::arch::asm!("sti; nop", options(nomem, nostack)); + unsafe { + core::arch::asm!("sti; nop", options(nomem, nostack)); + } } /// Halt instruction #[inline(always)] pub unsafe fn halt() { - core::arch::asm!("hlt", options(nomem, nostack)); + unsafe { + core::arch::asm!("hlt", options(nomem, nostack)); + } } /// Pause instruction diff --git a/src/arch/x86_shared/interrupt/trace.rs b/src/arch/x86_shared/interrupt/trace.rs index 9df85d66a6..4bbf262eb9 100644 --- a/src/arch/x86_shared/interrupt/trace.rs +++ b/src/arch/x86_shared/interrupt/trace.rs @@ -8,24 +8,28 @@ pub struct StackTrace { impl StackTrace { #[inline(always)] pub unsafe fn start() -> Option { - let mut fp: usize; - #[cfg(target_arch = "x86")] - core::arch::asm!("mov {}, ebp", out(reg) fp); - #[cfg(target_arch = "x86_64")] - core::arch::asm!("mov {}, rbp", out(reg) fp); - let pc_ptr = fp.checked_add(mem::size_of::())?; - Some(Self { - fp, - pc_ptr: pc_ptr as *const usize, - }) + unsafe { + let mut fp: usize; + #[cfg(target_arch = "x86")] + core::arch::asm!("mov {}, ebp", out(reg) fp); + #[cfg(target_arch = "x86_64")] + core::arch::asm!("mov {}, rbp", out(reg) fp); + let pc_ptr = fp.checked_add(mem::size_of::())?; + Some(Self { + fp, + pc_ptr: pc_ptr as *const usize, + }) + } } pub unsafe fn next(self) -> Option { - let fp = *(self.fp as *const usize); - let pc_ptr = fp.checked_add(mem::size_of::())?; - Some(Self { - fp: fp, - pc_ptr: pc_ptr as *const usize, - }) + unsafe { + let fp = *(self.fp as *const usize); + let pc_ptr = fp.checked_add(mem::size_of::())?; + Some(Self { + fp: fp, + pc_ptr: pc_ptr as *const usize, + }) + } } } diff --git a/src/arch/x86_shared/paging/mod.rs b/src/arch/x86_shared/paging/mod.rs index e892cb5577..1c7959147b 100644 --- a/src/arch/x86_shared/paging/mod.rs +++ b/src/arch/x86_shared/paging/mod.rs @@ -32,40 +32,44 @@ pub const PAGE_MASK: usize = RmmA::PAGE_OFFSET_MASK; /// Setup page attribute table #[cold] unsafe fn init_pat() { - let uncacheable = 0; - let write_combining = 1; - let write_through = 4; - //let write_protected = 5; - let write_back = 6; - let uncached = 7; + unsafe { + let uncacheable = 0; + let write_combining = 1; + let write_through = 4; + //let write_protected = 5; + let write_back = 6; + let uncached = 7; - let pat0 = write_back; - let pat1 = write_through; - let pat2 = uncached; - let pat3 = uncacheable; + let pat0 = write_back; + let pat1 = write_through; + let pat2 = uncached; + let pat3 = uncacheable; - let pat4 = write_combining; - let pat5 = pat1; - let pat6 = pat2; - let pat7 = pat3; + let pat4 = write_combining; + let pat5 = pat1; + let pat6 = pat2; + let pat7 = pat3; - msr::wrmsr( - msr::IA32_PAT, - pat7 << 56 - | pat6 << 48 - | pat5 << 40 - | pat4 << 32 - | pat3 << 24 - | pat2 << 16 - | pat1 << 8 - | pat0, - ); + msr::wrmsr( + msr::IA32_PAT, + pat7 << 56 + | pat6 << 48 + | pat5 << 40 + | pat4 << 32 + | pat3 << 24 + | pat2 << 16 + | pat1 << 8 + | pat0, + ); + } } /// Initialize PAT #[cold] pub unsafe fn init() { - init_pat(); + unsafe { + init_pat(); + } } /// Page diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index f6eb3e9787..471e644c31 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -65,190 +65,192 @@ pub struct KernelArgs { } /// The entry to Rust, all things must be initialized -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { - let bootstrap = { - let args = args_ptr.read(); + unsafe { + let bootstrap = { + let args = args_ptr.read(); - // BSS should already be zero - { - assert_eq!(BSS_TEST_ZERO.get().read(), 0); - assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value()); - } + // BSS should already be zero + { + assert_eq!(BSS_TEST_ZERO.get().read(), 0); + assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value()); + } - KERNEL_BASE.store(args.kernel_base as usize, Ordering::SeqCst); - KERNEL_SIZE.store(args.kernel_size as usize, Ordering::SeqCst); + KERNEL_BASE.store(args.kernel_base as usize, Ordering::SeqCst); + KERNEL_SIZE.store(args.kernel_size as usize, Ordering::SeqCst); - // Convert env to slice - let env = slice::from_raw_parts( - (args.env_base as usize + crate::PHYS_OFFSET) as *const u8, - args.env_size as usize, - ); - - // Set up serial debug - #[cfg(feature = "serial_debug")] - device::serial::init(); - - // Set up graphical debug - #[cfg(feature = "graphical_debug")] - graphical_debug::init(env); - - #[cfg(feature = "system76_ec_debug")] - device::system76_ec::init(); - - // Initialize logger - crate::log::init_logger(|r| { - use core::fmt::Write; - let _ = writeln!( - super::debug::Writer::new(), - "{}:{} -- {}", - r.target(), - r.level(), - r.args() + // Convert env to slice + let env = slice::from_raw_parts( + (args.env_base as usize + crate::PHYS_OFFSET) as *const u8, + args.env_size as usize, ); - }); - info!("Redox OS starting..."); - info!( - "Kernel: {:X}:{:X}", - { args.kernel_base }, - { args.kernel_base } + { args.kernel_size } - ); - info!( - "Stack: {:X}:{:X}", - { args.stack_base }, - { args.stack_base } + { args.stack_size } - ); - info!( - "Env: {:X}:{:X}", - { args.env_base }, - { args.env_base } + { args.env_size } - ); - info!( - "RSDP: {:X}:{:X}", - { args.acpi_rsdp_base }, - { args.acpi_rsdp_base } + { args.acpi_rsdp_size } - ); - info!( - "Areas: {:X}:{:X}", - { args.areas_base }, - { args.areas_base } + { args.areas_size } - ); - info!( - "Bootstrap: {:X}:{:X}", - { args.bootstrap_base }, - { args.bootstrap_base } + { args.bootstrap_size } - ); + // Set up serial debug + #[cfg(feature = "serial_debug")] + device::serial::init(); - // Set up GDT before paging - gdt::init(); + // Set up graphical debug + #[cfg(feature = "graphical_debug")] + graphical_debug::init(env); - // Set up IDT before paging - idt::init(); + #[cfg(feature = "system76_ec_debug")] + device::system76_ec::init(); - // Initialize RMM - register_bootloader_areas(args.areas_base as usize, args.areas_size as usize); - register_memory_region( - args.kernel_base as usize, - args.kernel_size as usize, - BootloaderMemoryKind::Kernel, - ); - register_memory_region( - args.stack_base as usize, - args.stack_size as usize, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.env_base as usize, - args.env_size as usize, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.acpi_rsdp_base as usize, - args.acpi_rsdp_size as usize, - BootloaderMemoryKind::IdentityMap, - ); - register_memory_region( - args.bootstrap_base as usize, - args.bootstrap_size as usize, - BootloaderMemoryKind::IdentityMap, - ); - #[cfg(target_arch = "x86")] - crate::startup::memory::init(Some(0x100000), Some(0x40000000)); - #[cfg(target_arch = "x86_64")] - crate::startup::memory::init(Some(0x100000), None); - - // Initialize paging - paging::init(); - - // Set up GDT after paging with TLS - gdt::init_paging( - args.stack_base as usize + args.stack_size as usize, - LogicalCpuId::BSP, - ); - - // Set up IDT - idt::init_paging_bsp(); - - #[cfg(target_arch = "x86_64")] - crate::alternative::early_init(true); - - // Set up syscall instruction - interrupt::syscall::init(); - - // Reset AP variables - CPU_COUNT.store(1, Ordering::SeqCst); - AP_READY.store(false, Ordering::SeqCst); - BSP_READY.store(false, Ordering::SeqCst); - - // Setup kernel heap - allocator::init(); - - #[cfg(all(target_arch = "x86_64", feature = "profiling"))] - crate::profiling::init(); - - // Set up double buffer for graphical debug now that heap is available - #[cfg(feature = "graphical_debug")] - graphical_debug::init_heap(); - - idt::init_paging_post_heap(LogicalCpuId::BSP); - - // Activate memory logging - crate::log::init(); - - // Initialize miscellaneous processor features - #[cfg(target_arch = "x86_64")] - crate::misc::init(LogicalCpuId::BSP); - - // Initialize devices - device::init(); - - // Read ACPI tables, starts APs - #[cfg(feature = "acpi")] - { - acpi::init(if args.acpi_rsdp_base != 0 { - Some((args.acpi_rsdp_base as usize + crate::PHYS_OFFSET) as *const u8) - } else { - None + // Initialize logger + crate::log::init_logger(|r| { + use core::fmt::Write; + let _ = writeln!( + super::debug::Writer::new(), + "{}:{} -- {}", + r.target(), + r.level(), + r.args() + ); }); - device::init_after_acpi(); - } - // Initialize all of the non-core devices not otherwise needed to complete initialization - device::init_noncore(); + info!("Redox OS starting..."); + info!( + "Kernel: {:X}:{:X}", + { args.kernel_base }, + { args.kernel_base } + { args.kernel_size } + ); + info!( + "Stack: {:X}:{:X}", + { args.stack_base }, + { args.stack_base } + { args.stack_size } + ); + info!( + "Env: {:X}:{:X}", + { args.env_base }, + { args.env_base } + { args.env_size } + ); + info!( + "RSDP: {:X}:{:X}", + { args.acpi_rsdp_base }, + { args.acpi_rsdp_base } + { args.acpi_rsdp_size } + ); + info!( + "Areas: {:X}:{:X}", + { args.areas_base }, + { args.areas_base } + { args.areas_size } + ); + info!( + "Bootstrap: {:X}:{:X}", + { args.bootstrap_base }, + { args.bootstrap_base } + { args.bootstrap_size } + ); - BSP_READY.store(true, Ordering::SeqCst); + // Set up GDT before paging + gdt::init(); - crate::Bootstrap { - base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new( + // Set up IDT before paging + idt::init(); + + // Initialize RMM + register_bootloader_areas(args.areas_base as usize, args.areas_size as usize); + register_memory_region( + args.kernel_base as usize, + args.kernel_size as usize, + BootloaderMemoryKind::Kernel, + ); + register_memory_region( + args.stack_base as usize, + args.stack_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.env_base as usize, + args.env_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( + args.acpi_rsdp_base as usize, + args.acpi_rsdp_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + register_memory_region( args.bootstrap_base as usize, - )), - page_count: (args.bootstrap_size as usize) / crate::memory::PAGE_SIZE, - env, - } - }; + args.bootstrap_size as usize, + BootloaderMemoryKind::IdentityMap, + ); + #[cfg(target_arch = "x86")] + crate::startup::memory::init(Some(0x100000), Some(0x40000000)); + #[cfg(target_arch = "x86_64")] + crate::startup::memory::init(Some(0x100000), None); - crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap); + // Initialize paging + paging::init(); + + // Set up GDT after paging with TLS + gdt::init_paging( + args.stack_base as usize + args.stack_size as usize, + LogicalCpuId::BSP, + ); + + // Set up IDT + idt::init_paging_bsp(); + + #[cfg(target_arch = "x86_64")] + crate::alternative::early_init(true); + + // Set up syscall instruction + interrupt::syscall::init(); + + // Reset AP variables + CPU_COUNT.store(1, Ordering::SeqCst); + AP_READY.store(false, Ordering::SeqCst); + BSP_READY.store(false, Ordering::SeqCst); + + // Setup kernel heap + allocator::init(); + + #[cfg(all(target_arch = "x86_64", feature = "profiling"))] + crate::profiling::init(); + + // Set up double buffer for graphical debug now that heap is available + #[cfg(feature = "graphical_debug")] + graphical_debug::init_heap(); + + idt::init_paging_post_heap(LogicalCpuId::BSP); + + // Activate memory logging + crate::log::init(); + + // Initialize miscellaneous processor features + #[cfg(target_arch = "x86_64")] + crate::misc::init(LogicalCpuId::BSP); + + // Initialize devices + device::init(); + + // Read ACPI tables, starts APs + #[cfg(feature = "acpi")] + { + acpi::init(if args.acpi_rsdp_base != 0 { + Some((args.acpi_rsdp_base as usize + crate::PHYS_OFFSET) as *const u8) + } else { + None + }); + device::init_after_acpi(); + } + + // Initialize all of the non-core devices not otherwise needed to complete initialization + device::init_noncore(); + + BSP_READY.store(true, Ordering::SeqCst); + + crate::Bootstrap { + base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new( + args.bootstrap_base as usize, + )), + page_count: (args.bootstrap_size as usize) / crate::memory::PAGE_SIZE, + env, + } + }; + + crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap); + } } #[repr(C, packed)] @@ -263,56 +265,58 @@ pub struct KernelArgsAp { /// Entry to rust for an AP pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { - let cpu_id = { - let args = &*args_ptr; - let cpu_id = LogicalCpuId::new(args.cpu_id as u32); - let bsp_table = args.page_table as usize; - let _stack_start = args.stack_start as usize; - let stack_end = args.stack_end as usize; + unsafe { + let cpu_id = { + let args = &*args_ptr; + let cpu_id = LogicalCpuId::new(args.cpu_id as u32); + let bsp_table = args.page_table as usize; + let _stack_start = args.stack_start as usize; + let stack_end = args.stack_end as usize; - assert_eq!(BSS_TEST_ZERO.get().read(), 0); - assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value()); + assert_eq!(BSS_TEST_ZERO.get().read(), 0); + assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value()); - // Set up GDT before paging - gdt::init(); + // Set up GDT before paging + gdt::init(); - // Set up IDT before paging - idt::init(); + // Set up IDT before paging + idt::init(); - // Initialize paging - RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table)); - paging::init(); + // Initialize paging + RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table)); + paging::init(); - // Set up GDT with TLS - gdt::init_paging(stack_end, cpu_id); + // Set up GDT with TLS + gdt::init_paging(stack_end, cpu_id); - #[cfg(all(target_arch = "x86_64", feature = "profiling"))] - crate::profiling::init(); + #[cfg(all(target_arch = "x86_64", feature = "profiling"))] + crate::profiling::init(); - // Set up IDT for AP - idt::init_paging_post_heap(cpu_id); + // Set up IDT for AP + idt::init_paging_post_heap(cpu_id); - #[cfg(target_arch = "x86_64")] - crate::alternative::early_init(false); + #[cfg(target_arch = "x86_64")] + crate::alternative::early_init(false); - // Set up syscall instruction - interrupt::syscall::init(); + // Set up syscall instruction + interrupt::syscall::init(); - // Initialize miscellaneous processor features - #[cfg(target_arch = "x86_64")] - crate::misc::init(cpu_id); + // Initialize miscellaneous processor features + #[cfg(target_arch = "x86_64")] + crate::misc::init(cpu_id); - // Initialize devices (for AP) - device::init_ap(); + // Initialize devices (for AP) + device::init_ap(); - AP_READY.store(true, Ordering::SeqCst); + AP_READY.store(true, Ordering::SeqCst); - cpu_id - }; + cpu_id + }; - while !BSP_READY.load(Ordering::SeqCst) { - interrupt::pause(); + while !BSP_READY.load(Ordering::SeqCst) { + interrupt::pause(); + } + + crate::kmain_ap(cpu_id); } - - crate::kmain_ap(cpu_id); } diff --git a/src/arch/x86_shared/stop.rs b/src/arch/x86_shared/stop.rs index 0a573011b1..6b6b1cb106 100644 --- a/src/arch/x86_shared/stop.rs +++ b/src/arch/x86_shared/stop.rs @@ -4,24 +4,27 @@ use crate::{context, scheme::acpi, time}; use crate::syscall::io::{Io, Pio}; pub unsafe fn kreset() -> ! { - log::info!("kreset"); + unsafe { + log::info!("kreset"); - // 8042 reset - { - println!("Reset with 8042"); - let mut port = Pio::::new(0x64); - while port.readf(2) {} - port.write(0xFE); + // 8042 reset + { + println!("Reset with 8042"); + let mut port = Pio::::new(0x64); + while port.readf(2) {} + port.write(0xFE); + } + + emergency_reset(); } - - emergency_reset(); } #[cfg(target_arch = "x86")] pub unsafe fn emergency_reset() -> ! { - // Use triple fault to guarantee reset - core::arch::asm!( - " + unsafe { + // Use triple fault to guarantee reset + core::arch::asm!( + " cli sidt [esp+16] // set IDT limit to zero @@ -29,15 +32,17 @@ pub unsafe fn emergency_reset() -> ! { lidt [esp+16] int $3 ", - options(noreturn) - ); + options(noreturn) + ); + } } #[cfg(target_arch = "x86_64")] pub unsafe fn emergency_reset() -> ! { - // Use triple fault to guarantee reset - core::arch::asm!( - " + unsafe { + // Use triple fault to guarantee reset + core::arch::asm!( + " cli sidt [rsp+16] // set IDT limit to zero @@ -45,8 +50,9 @@ pub unsafe fn emergency_reset() -> ! { lidt [rsp+16] int $3 ", - options(noreturn) - ); + options(noreturn) + ); + } } #[cfg(feature = "acpi")] @@ -81,29 +87,31 @@ fn userspace_acpi_shutdown() { } pub unsafe fn kstop() -> ! { - log::info!("Running kstop()"); + unsafe { + log::info!("Running kstop()"); - #[cfg(feature = "acpi")] - userspace_acpi_shutdown(); + #[cfg(feature = "acpi")] + userspace_acpi_shutdown(); - // Magic shutdown code for bochs and qemu (older versions). - for c in "Shutdown".bytes() { - let port = 0x8900; - println!("Shutdown with outb(0x{:X}, '{}')", port, c as char); - Pio::::new(port).write(c); - } + // Magic shutdown code for bochs and qemu (older versions). + for c in "Shutdown".bytes() { + let port = 0x8900; + println!("Shutdown with outb(0x{:X}, '{}')", port, c as char); + Pio::::new(port).write(c); + } - // Magic shutdown using qemu default ACPI method - { - let port = 0x604; - let data = 0x2000; - println!("Shutdown with outb(0x{:X}, 0x{:X})", port, data); - Pio::::new(port).write(data); - } + // Magic shutdown using qemu default ACPI method + { + let port = 0x604; + let data = 0x2000; + println!("Shutdown with outb(0x{:X}, 0x{:X})", port, data); + Pio::::new(port).write(data); + } - // Magic code for VMWare. Also a hard lock. - println!("Shutdown with cli hlt"); - loop { - core::arch::asm!("cli; hlt"); + // Magic code for VMWare. Also a hard lock. + println!("Shutdown with cli hlt"); + loop { + core::arch::asm!("cli; hlt"); + } } } diff --git a/src/common/mod.rs b/src/common/mod.rs index cb9fc3b5ea..64a5eebde6 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -8,7 +8,7 @@ macro_rules! dbg { () => { $crate::println!("[{}:{}]", file!(), line!()); }; - ($val:expr) => { + ($val:expr_2021) => { // Use of `match` here is intentional because it affects the lifetimes // of temporaries - https://stackoverflow.com/a/48732525/1063961 match $val { @@ -20,8 +20,8 @@ macro_rules! dbg { } }; // Trailing comma with single argument is ignored - ($val:expr,) => { $crate::dbg!($val) }; - ($($val:expr),+ $(,)?) => { + ($val:expr_2021,) => { $crate::dbg!($val) }; + ($($val:expr_2021),+ $(,)?) => { ($($crate::dbg!($val)),+,) }; } diff --git a/src/context/arch/aarch64.rs b/src/context/arch/aarch64.rs index ff90ca55ec..4602c36635 100644 --- a/src/context/arch/aarch64.rs +++ b/src/context/arch/aarch64.rs @@ -200,92 +200,101 @@ pub static EMPTY_CR3: Once = Once::new(); // SAFETY: EMPTY_CR3 must be initialized. pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { - debug_assert!(EMPTY_CR3.poll().is_some()); - *EMPTY_CR3.get_unchecked() + unsafe { + debug_assert!(EMPTY_CR3.poll().is_some()); + *EMPTY_CR3.get_unchecked() + } } #[target_feature(enable = "neon")] unsafe extern "C" fn fp_save(float_regs: &mut FloatRegisters) { - core::arch::asm!( - "stp q0, q1, [{3}, {0} + 16 * 0]", - "stp q2, q3, [{3}, {0} + 16 * 2]", - "stp q4, q5, [{3}, {0} + 16 * 4]", - "stp q6, q7, [{3}, {0} + 16 * 6]", - "stp q8, q9, [{3}, {0} + 16 * 8]", - "stp q10, q11, [{3}, {0} + 16 * 10]", - "stp q12, q13, [{3}, {0} + 16 * 12]", - "stp q14, q15, [{3}, {0} + 16 * 14]", - "stp q16, q17, [{3}, {0} + 16 * 16]", - "stp q18, q19, [{3}, {0} + 16 * 18]", - "stp q20, q21, [{3}, {0} + 16 * 20]", - "stp q22, q23, [{3}, {0} + 16 * 22]", - "stp q24, q25, [{3}, {0} + 16 * 24]", - "stp q26, q27, [{3}, {0} + 16 * 26]", - "stp q28, q29, [{3}, {0} + 16 * 28]", - "stp q30, q31, [{3}, {0} + 16 * 30]", - "mrs x9, fpcr", - "add {3}, {3}, {1}", - "str x9, [{3}]", - "mrs x9, fpsr", - "str x9, [{3}, {2} - {1}]", - const mem::offset_of!(FloatRegisters, fp_simd_regs), - const mem::offset_of!(FloatRegisters, fpcr), - const mem::offset_of!(FloatRegisters, fpsr), - inout(reg) float_regs => _, - ); + unsafe { + core::arch::asm!( + "stp q0, q1, [{3}, {0} + 16 * 0]", + "stp q2, q3, [{3}, {0} + 16 * 2]", + "stp q4, q5, [{3}, {0} + 16 * 4]", + "stp q6, q7, [{3}, {0} + 16 * 6]", + "stp q8, q9, [{3}, {0} + 16 * 8]", + "stp q10, q11, [{3}, {0} + 16 * 10]", + "stp q12, q13, [{3}, {0} + 16 * 12]", + "stp q14, q15, [{3}, {0} + 16 * 14]", + "stp q16, q17, [{3}, {0} + 16 * 16]", + "stp q18, q19, [{3}, {0} + 16 * 18]", + "stp q20, q21, [{3}, {0} + 16 * 20]", + "stp q22, q23, [{3}, {0} + 16 * 22]", + "stp q24, q25, [{3}, {0} + 16 * 24]", + "stp q26, q27, [{3}, {0} + 16 * 26]", + "stp q28, q29, [{3}, {0} + 16 * 28]", + "stp q30, q31, [{3}, {0} + 16 * 30]", + "mrs x9, fpcr", + "add {3}, {3}, {1}", + "str x9, [{3}]", + "mrs x9, fpsr", + "str x9, [{3}, {2} - {1}]", + const mem::offset_of!(FloatRegisters, fp_simd_regs), + const mem::offset_of!(FloatRegisters, fpcr), + const mem::offset_of!(FloatRegisters, fpsr), + inout(reg) float_regs => _, + ); + } } #[target_feature(enable = "neon")] unsafe extern "C" fn fp_load(float_regs: &mut FloatRegisters) { - core::arch::asm!( - "ldp q0, q1, [{3}, {0} + 16 * 0]", - "ldp q2, q3, [{3}, {0} + 16 * 2]", - "ldp q4, q5, [{3}, {0} + 16 * 4]", - "ldp q6, q7, [{3}, {0} + 16 * 6]", - "ldp q8, q9, [{3}, {0} + 16 * 8]", - "ldp q10, q11, [{3}, {0} + 16 * 10]", - "ldp q12, q13, [{3}, {0} + 16 * 12]", - "ldp q14, q15, [{3}, {0} + 16 * 14]", - "ldp q16, q17, [{3}, {0} + 16 * 16]", - "ldp q18, q19, [{3}, {0} + 16 * 18]", - "ldp q20, q21, [{3}, {0} + 16 * 20]", - "ldp q22, q23, [{3}, {0} + 16 * 22]", - "ldp q24, q25, [{3}, {0} + 16 * 24]", - "ldp q26, q27, [{3}, {0} + 16 * 26]", - "ldp q28, q29, [{3}, {0} + 16 * 28]", - "ldp q30, q31, [{3}, {0} + 16 * 30]", - "add {3}, {3}, {1}", - "ldr x9, [{3}]", - "msr fpcr, x9", - "ldr x9, [{3}, {2} - {1}]", - "msr fpsr, x9", - const mem::offset_of!(FloatRegisters, fp_simd_regs), - const mem::offset_of!(FloatRegisters, fpcr), - const mem::offset_of!(FloatRegisters, fpsr), - inout(reg) float_regs => _, - ); + unsafe { + core::arch::asm!( + "ldp q0, q1, [{3}, {0} + 16 * 0]", + "ldp q2, q3, [{3}, {0} + 16 * 2]", + "ldp q4, q5, [{3}, {0} + 16 * 4]", + "ldp q6, q7, [{3}, {0} + 16 * 6]", + "ldp q8, q9, [{3}, {0} + 16 * 8]", + "ldp q10, q11, [{3}, {0} + 16 * 10]", + "ldp q12, q13, [{3}, {0} + 16 * 12]", + "ldp q14, q15, [{3}, {0} + 16 * 14]", + "ldp q16, q17, [{3}, {0} + 16 * 16]", + "ldp q18, q19, [{3}, {0} + 16 * 18]", + "ldp q20, q21, [{3}, {0} + 16 * 20]", + "ldp q22, q23, [{3}, {0} + 16 * 22]", + "ldp q24, q25, [{3}, {0} + 16 * 24]", + "ldp q26, q27, [{3}, {0} + 16 * 26]", + "ldp q28, q29, [{3}, {0} + 16 * 28]", + "ldp q30, q31, [{3}, {0} + 16 * 30]", + "add {3}, {3}, {1}", + "ldr x9, [{3}]", + "msr fpcr, x9", + "ldr x9, [{3}, {2} - {1}]", + "msr fpsr, x9", + const mem::offset_of!(FloatRegisters, fp_simd_regs), + const mem::offset_of!(FloatRegisters, fpcr), + const mem::offset_of!(FloatRegisters, fpsr), + inout(reg) float_regs => _, + ); + } } pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { - fp_save(&mut *(prev.kfx.as_mut_ptr() as *mut FloatRegisters)); + unsafe { + fp_save(&mut *(prev.kfx.as_mut_ptr() as *mut FloatRegisters)); - prev.arch.fx_loadable = true; + prev.arch.fx_loadable = true; - if next.arch.fx_loadable { - fp_load(&mut *(next.kfx.as_mut_ptr() as *mut FloatRegisters)); + if next.arch.fx_loadable { + fp_load(&mut *(next.kfx.as_mut_ptr() as *mut FloatRegisters)); + } + + PercpuBlock::current() + .new_addrsp_tmp + .set(next.addr_space.clone()); + + switch_to_inner(&mut prev.arch, &mut next.arch) } - - PercpuBlock::current() - .new_addrsp_tmp - .set(next.addr_space.clone()); - - switch_to_inner(&mut prev.arch, &mut next.arch) } #[naked] unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { - core::arch::naked_asm!( - " + unsafe { + core::arch::naked_asm!( + " str x19, [x0, #{off_x19}] ldr x19, [x1, #{off_x19}] @@ -359,28 +368,29 @@ unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { b {switch_hook} ", - off_x19 = const(offset_of!(Context, x19)), - off_x20 = const(offset_of!(Context, x20)), - off_x21 = const(offset_of!(Context, x21)), - off_x22 = const(offset_of!(Context, x22)), - off_x23 = const(offset_of!(Context, x23)), - off_x24 = const(offset_of!(Context, x24)), - off_x25 = const(offset_of!(Context, x25)), - off_x26 = const(offset_of!(Context, x26)), - off_x27 = const(offset_of!(Context, x27)), - off_x28 = const(offset_of!(Context, x28)), - off_x29 = const(offset_of!(Context, fp)), - off_x30 = const(offset_of!(Context, lr)), - off_elr_el1 = const(offset_of!(Context, elr_el1)), - off_sp_el0 = const(offset_of!(Context, sp_el0)), - off_tpidr_el0 = const(offset_of!(Context, tpidr_el0)), - off_tpidrro_el0 = const(offset_of!(Context, tpidrro_el0)), - off_spsr_el1 = const(offset_of!(Context, spsr_el1)), - off_esr_el1 = const(offset_of!(Context, esr_el1)), - off_sp = const(offset_of!(Context, sp)), + off_x19 = const(offset_of!(Context, x19)), + off_x20 = const(offset_of!(Context, x20)), + off_x21 = const(offset_of!(Context, x21)), + off_x22 = const(offset_of!(Context, x22)), + off_x23 = const(offset_of!(Context, x23)), + off_x24 = const(offset_of!(Context, x24)), + off_x25 = const(offset_of!(Context, x25)), + off_x26 = const(offset_of!(Context, x26)), + off_x27 = const(offset_of!(Context, x27)), + off_x28 = const(offset_of!(Context, x28)), + off_x29 = const(offset_of!(Context, fp)), + off_x30 = const(offset_of!(Context, lr)), + off_elr_el1 = const(offset_of!(Context, elr_el1)), + off_sp_el0 = const(offset_of!(Context, sp_el0)), + off_tpidr_el0 = const(offset_of!(Context, tpidr_el0)), + off_tpidrro_el0 = const(offset_of!(Context, tpidrro_el0)), + off_spsr_el1 = const(offset_of!(Context, spsr_el1)), + off_esr_el1 = const(offset_of!(Context, esr_el1)), + off_sp = const(offset_of!(Context, sp)), - switch_hook = sym crate::context::switch_finish_hook, - ); + switch_hook = sym crate::context::switch_finish_hook, + ); + } } /// Allocates a new empty utable diff --git a/src/context/arch/riscv64.rs b/src/context/arch/riscv64.rs index 1f484811ef..a7bee5f1c9 100644 --- a/src/context/arch/riscv64.rs +++ b/src/context/arch/riscv64.rs @@ -134,23 +134,28 @@ pub static EMPTY_CR3: Once = Once::new(); // SAFETY: EMPTY_CR3 must be initialized. pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { - debug_assert!(EMPTY_CR3.poll().is_some()); - *EMPTY_CR3.get_unchecked() + unsafe { + debug_assert!(EMPTY_CR3.poll().is_some()); + *EMPTY_CR3.get_unchecked() + } } /// Switch to the next context by restoring its stack and registers pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { - // FIXME floating point - PercpuBlock::current() - .new_addrsp_tmp - .set(next.addr_space.clone()); + unsafe { + // FIXME floating point + PercpuBlock::current() + .new_addrsp_tmp + .set(next.addr_space.clone()); - switch_to_inner(&mut prev.arch, &mut next.arch); + switch_to_inner(&mut prev.arch, &mut next.arch); + } } #[naked] unsafe extern "C" fn switch_to_inner(prev: &mut Context, next: &mut Context) { - core::arch::naked_asm!(r#" + unsafe { + core::arch::naked_asm!(r#" sd s1, {off_s1}(a0) ld s1, {off_s1}(a1) @@ -203,24 +208,25 @@ unsafe extern "C" fn switch_to_inner(prev: &mut Context, next: &mut Context) { j {switch_hook} "#, - off_s1 = const(offset_of!(Context, s1)), - off_s2 = const(offset_of!(Context, s2)), - off_s3 = const(offset_of!(Context, s3)), - off_s4 = const(offset_of!(Context, s4)), - off_s5 = const(offset_of!(Context, s5)), - off_s6 = const(offset_of!(Context, s6)), - off_s7 = const(offset_of!(Context, s7)), - off_s8 = const(offset_of!(Context, s8)), - off_s9 = const(offset_of!(Context, s9)), - off_s10 = const(offset_of!(Context, s10)), - off_s11 = const(offset_of!(Context, s11)), - off_sp = const(offset_of!(Context, sp)), - off_ra = const(offset_of!(Context, ra)), - off_fp = const(offset_of!(Context, fp)), - off_sstatus = const(offset_of!(Context, sstatus)), + off_s1 = const(offset_of!(Context, s1)), + off_s2 = const(offset_of!(Context, s2)), + off_s3 = const(offset_of!(Context, s3)), + off_s4 = const(offset_of!(Context, s4)), + off_s5 = const(offset_of!(Context, s5)), + off_s6 = const(offset_of!(Context, s6)), + off_s7 = const(offset_of!(Context, s7)), + off_s8 = const(offset_of!(Context, s8)), + off_s9 = const(offset_of!(Context, s9)), + off_s10 = const(offset_of!(Context, s10)), + off_s11 = const(offset_of!(Context, s11)), + off_sp = const(offset_of!(Context, sp)), + off_ra = const(offset_of!(Context, ra)), + off_fp = const(offset_of!(Context, fp)), + off_sstatus = const(offset_of!(Context, sstatus)), - switch_hook = sym crate::context::switch_finish_hook, - ); + switch_hook = sym crate::context::switch_finish_hook, + ); + } } /// Allocates a new empty utable diff --git a/src/context/arch/x86.rs b/src/context/arch/x86.rs index 4a98d9ebc4..ebdece8bf2 100644 --- a/src/context/arch/x86.rs +++ b/src/context/arch/x86.rs @@ -209,57 +209,62 @@ pub static EMPTY_CR3: Once = Once::new(); // SAFETY: EMPTY_CR3 must be initialized. pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { - debug_assert!(EMPTY_CR3.poll().is_some()); - *EMPTY_CR3.get_unchecked() + unsafe { + debug_assert!(EMPTY_CR3.poll().is_some()); + *EMPTY_CR3.get_unchecked() + } } /// Switch to the next context by restoring its stack and registers pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { - if let Some(ref stack) = next.kstack { - crate::gdt::set_tss_stack(stack.initial_top() as usize); - } - crate::gdt::set_userspace_io_allowed(next.arch.userspace_io_allowed); + unsafe { + if let Some(ref stack) = next.kstack { + crate::gdt::set_tss_stack(stack.initial_top() as usize); + } + crate::gdt::set_userspace_io_allowed(next.arch.userspace_io_allowed); - core::arch::asm!(" + core::arch::asm!(" fxsave [{prev_fx}] fxrstor [{next_fx}] ", prev_fx = in(reg) prev.kfx.as_mut_ptr(), - next_fx = in(reg) next.kfx.as_ptr(), - ); + next_fx = in(reg) next.kfx.as_ptr(), + ); - { - let gdt = &mut (&mut *pcr()).gdt; + { + let gdt = &mut (&mut *pcr()).gdt; - prev.arch.fsbase = gdt[GDT_USER_FS].offset() as usize; - gdt[GDT_USER_FS].set_offset(next.arch.fsbase as u32); - prev.arch.gsbase = gdt[GDT_USER_GS].offset() as usize; - gdt[GDT_USER_GS].set_offset(next.arch.gsbase as u32); + prev.arch.fsbase = gdt[GDT_USER_FS].offset() as usize; + gdt[GDT_USER_FS].set_offset(next.arch.fsbase as u32); + prev.arch.gsbase = gdt[GDT_USER_GS].offset() as usize; + gdt[GDT_USER_GS].set_offset(next.arch.gsbase as u32); + } + PercpuBlock::current() + .new_addrsp_tmp + .set(next.addr_space.clone()); + + core::arch::asm!( + "call {inner}", + inner = sym switch_to_inner, + in("ecx") &mut prev.arch, + in("edx") &mut next.arch, + ); } - PercpuBlock::current() - .new_addrsp_tmp - .set(next.addr_space.clone()); - - core::arch::asm!( - "call {inner}", - inner = sym switch_to_inner, - in("ecx") &mut prev.arch, - in("edx") &mut next.arch, - ); } // Check disassembly! #[naked] unsafe extern "cdecl" fn switch_to_inner() { - use Context as Cx; + unsafe { + use Context as Cx; - core::arch::naked_asm!( - // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): - // - // - the current parameters are passed in the registers `edi`, `esi`, - // - we can modify scratch registers, e.g. rax - // - we cannot change callee-preserved registers arbitrarily, e.g. ebx, which is why we - // store them here in the first place. - concat!(" + core::arch::naked_asm!( + // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): + // + // - the current parameters are passed in the registers `edi`, `esi`, + // - we can modify scratch registers, e.g. rax + // - we cannot change callee-preserved registers arbitrarily, e.g. ebx, which is why we + // store them here in the first place. + concat!(" // ecx is prev, edx is next // Save old registers, and load new ones @@ -297,16 +302,17 @@ unsafe extern "cdecl" fn switch_to_inner() { "), - off_eflags = const(offset_of!(Cx, eflags)), + off_eflags = const(offset_of!(Cx, eflags)), - off_ebx = const(offset_of!(Cx, ebx)), - off_edi = const(offset_of!(Cx, edi)), - off_esi = const(offset_of!(Cx, esi)), - off_ebp = const(offset_of!(Cx, ebp)), - off_esp = const(offset_of!(Cx, esp)), + off_ebx = const(offset_of!(Cx, ebx)), + off_edi = const(offset_of!(Cx, edi)), + off_esi = const(offset_of!(Cx, esi)), + off_ebp = const(offset_of!(Cx, ebp)), + off_esp = const(offset_of!(Cx, esp)), - switch_hook = sym crate::context::switch_finish_hook, - ); + switch_hook = sym crate::context::switch_finish_hook, + ); + } } /// Allocates a new identically mapped ktable and empty utable (same memory on x86) diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 21cecac460..0cab729248 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -227,56 +227,59 @@ pub static EMPTY_CR3: Once = Once::new(); // SAFETY: EMPTY_CR3 must be initialized. pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { - debug_assert!(EMPTY_CR3.poll().is_some()); - *EMPTY_CR3.get_unchecked() + unsafe { + debug_assert!(EMPTY_CR3.poll().is_some()); + *EMPTY_CR3.get_unchecked() + } } /// Switch to the next context by restoring its stack and registers pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { - // Update contexts' timestamps - let switch_time = crate::time::monotonic(); - prev.cpu_time += switch_time.saturating_sub(prev.switch_time); - next.switch_time = switch_time; + unsafe { + // Update contexts' timestamps + let switch_time = crate::time::monotonic(); + prev.cpu_time += switch_time.saturating_sub(prev.switch_time); + next.switch_time = switch_time; - let pcr = crate::gdt::pcr(); + let pcr = crate::gdt::pcr(); - if let Some(ref stack) = next.kstack { - crate::gdt::set_tss_stack(pcr, stack.initial_top() as usize); - } - crate::gdt::set_userspace_io_allowed(pcr, next.arch.userspace_io_allowed); + if let Some(ref stack) = next.kstack { + crate::gdt::set_tss_stack(pcr, stack.initial_top() as usize); + } + crate::gdt::set_userspace_io_allowed(pcr, next.arch.userspace_io_allowed); - core::arch::asm!( - alternative2!( - feature1: "xsaveopt", - then1: [" + core::arch::asm!( + alternative2!( + feature1: "xsaveopt", + then1: [" mov eax, 0xffffffff mov edx, eax xsaveopt64 [{prev_fx}] xrstor64 [{next_fx}] "], - feature2: "xsave", - then2: [" + feature2: "xsave", + then2: [" mov eax, 0xffffffff mov edx, eax xsave64 [{prev_fx}] xrstor64 [{next_fx}] "], - default: [" + default: [" fxsave64 [{prev_fx}] fxrstor64 [{next_fx}] "] - ), - prev_fx = in(reg) prev.kfx.as_mut_ptr(), - next_fx = in(reg) next.kfx.as_ptr(), - out("eax") _, - out("edx") _, - ); + ), + prev_fx = in(reg) prev.kfx.as_mut_ptr(), + next_fx = in(reg) next.kfx.as_ptr(), + out("eax") _, + out("edx") _, + ); - { - core::arch::asm!( - alternative!( - feature: "fsgsbase", - then: [" + { + core::arch::asm!( + alternative!( + feature: "fsgsbase", + then: [" mov rax, [{next}+{fsbase_off}] mov rcx, [{next}+{gsbase_off}] @@ -290,10 +293,10 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { mov [{prev}+{fsbase_off}], rdx mov [{prev}+{gsbase_off}], rax "], - // TODO: Most applications will set FSBASE, but won't touch GSBASE. Maybe avoid - // wrmsr or even the swapgs+rdgsbase+wrgsbase+swapgs sequence if they are already - // equal? - default: [" + // TODO: Most applications will set FSBASE, but won't touch GSBASE. Maybe avoid + // wrmsr or even the swapgs+rdgsbase+wrgsbase+swapgs sequence if they are already + // equal? + default: [" mov ecx, {MSR_FSBASE} mov rdx, [{next}+{fsbase_off}] mov eax, edx @@ -308,35 +311,37 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { // {prev} "] - ), - out("rax") _, - out("rdx") _, - out("ecx") _, prev = in(reg) addr_of_mut!(prev.arch), next = in(reg) addr_of!(next.arch), - MSR_FSBASE = const msr::IA32_FS_BASE, - MSR_KERNEL_GSBASE = const msr::IA32_KERNEL_GSBASE, - gsbase_off = const offset_of!(Context, gsbase), - fsbase_off = const offset_of!(Context, fsbase), - ); + ), + out("rax") _, + out("rdx") _, + out("ecx") _, prev = in(reg) addr_of_mut!(prev.arch), next = in(reg) addr_of!(next.arch), + MSR_FSBASE = const msr::IA32_FS_BASE, + MSR_KERNEL_GSBASE = const msr::IA32_KERNEL_GSBASE, + gsbase_off = const offset_of!(Context, gsbase), + fsbase_off = const offset_of!(Context, fsbase), + ); + } + + (*pcr).percpu.new_addrsp_tmp.set(next.addr_space.clone()); + + switch_to_inner(&mut prev.arch, &mut next.arch) } - - (*pcr).percpu.new_addrsp_tmp.set(next.addr_space.clone()); - - switch_to_inner(&mut prev.arch, &mut next.arch) } // Check disassembly! #[naked] unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { - use Context as Cx; + unsafe { + use Context as Cx; - core::arch::naked_asm!( - // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): - // - // - the current parameters are passed in the registers `rdi`, `rsi`, - // - we can modify scratch registers, e.g. rax - // - we cannot change callee-preserved registers arbitrarily, e.g. rbx, which is why we - // store them here in the first place. - concat!(" + core::arch::naked_asm!( + // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): + // + // - the current parameters are passed in the registers `rdi`, `rsi`, + // - we can modify scratch registers, e.g. rax + // - we cannot change callee-preserved registers arbitrarily, e.g. rbx, which is why we + // store them here in the first place. + concat!(" // Save old registers, and load new ones mov [rdi + {off_rbx}], rbx mov rbx, [rsi + {off_rbx}] @@ -378,18 +383,19 @@ unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Conte "), - off_rflags = const(offset_of!(Cx, rflags)), + off_rflags = const(offset_of!(Cx, rflags)), - off_rbx = const(offset_of!(Cx, rbx)), - off_r12 = const(offset_of!(Cx, r12)), - off_r13 = const(offset_of!(Cx, r13)), - off_r14 = const(offset_of!(Cx, r14)), - off_r15 = const(offset_of!(Cx, r15)), - off_rbp = const(offset_of!(Cx, rbp)), - off_rsp = const(offset_of!(Cx, rsp)), + off_rbx = const(offset_of!(Cx, rbx)), + off_r12 = const(offset_of!(Cx, r12)), + off_r13 = const(offset_of!(Cx, r13)), + off_r14 = const(offset_of!(Cx, r14)), + off_r15 = const(offset_of!(Cx, r15)), + off_rbp = const(offset_of!(Cx, rbp)), + off_rsp = const(offset_of!(Cx, rsp)), - switch_hook = sym crate::context::switch_finish_hook, - ); + switch_hook = sym crate::context::switch_finish_hook, + ); + } } /// Allocates a new identically mapped ktable and empty utable (same memory on x86_64). diff --git a/src/context/context.rs b/src/context/context.rs index a575930107..624971b79a 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -320,7 +320,7 @@ impl Context { &mut self, addr_space: Option>, ) -> Option> { - if let (Some(ref old), Some(ref new)) = (&self.addr_space, &addr_space) + if let (&Some(ref old), &Some(ref new)) = (&self.addr_space, &addr_space) && Arc::ptr_eq(old, new) { return addr_space; @@ -346,17 +346,18 @@ impl Context { addr_space.clone(), ); - if let Some(ref new) = addr_space { - let new_addrsp = new.acquire_read(); - new_addrsp.used_by.atomic_set(this_percpu.cpu_id); + match addr_space { + Some(ref new) => { + let new_addrsp = new.acquire_read(); + new_addrsp.used_by.atomic_set(this_percpu.cpu_id); - unsafe { - new_addrsp.table.utable.make_current(); + unsafe { + new_addrsp.table.utable.make_current(); + } } - } else { - unsafe { + _ => unsafe { crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3()); - } + }, } } else { assert!(!self.running); diff --git a/src/context/memory.rs b/src/context/memory.rs index 6234d67399..33089957a8 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -2206,15 +2206,13 @@ impl GrantInfo { flags } pub fn file_ref(&self) -> Option<&GrantFileRef> { - if let Provider::FmapBorrowed { ref file_ref, .. } - | Provider::Allocated { - cow_file_ref: Some(ref file_ref), - .. - } = self.provider - { - Some(file_ref) - } else { - None + match self.provider { + Provider::FmapBorrowed { ref file_ref, .. } + | Provider::Allocated { + cow_file_ref: Some(ref file_ref), + .. + } => Some(file_ref), + _ => None, } } } @@ -2513,94 +2511,99 @@ fn correct_inner<'l>( let mut guard = foreign_address_space.acquire_upgradeable_read(); let src_page = src_base.next_by(pages_from_grant_start); - if let Some(_) = guard.grants.contains(src_page) { - let src_frame = if let Some((phys, _)) = - guard.table.utable.translate(src_page.start_address()) - { - Frame::containing(phys) - } else { - // Grant was valid (TODO check), but we need to correct the underlying page. - // TODO: Access mode + match guard.grants.contains(src_page) { + Some(_) => { + let src_frame = match guard.table.utable.translate(src_page.start_address()) { + Some((phys, _)) => Frame::containing(phys), + _ => { + // Grant was valid (TODO check), but we need to correct the underlying page. + // TODO: Access mode - // TODO: Reasonable maximum? - let new_recursion_level = recursion_level - .checked_add(1) - .filter(|new_lvl| *new_lvl < 16) - .ok_or(PfError::RecursionLimitExceeded)?; + // TODO: Reasonable maximum? + let new_recursion_level = recursion_level + .checked_add(1) + .filter(|new_lvl| *new_lvl < 16) + .ok_or(PfError::RecursionLimitExceeded)?; - drop(guard); - drop(flusher); - drop(addr_space_guard); + drop(guard); + drop(flusher); + drop(addr_space_guard); - // FIXME: Can this result in invalid address space state? - let ext_addrspace = &foreign_address_space; - let (frame, _, _) = { - let g = ext_addrspace.acquire_write(); - correct_inner( - ext_addrspace, - g, - src_page, - AccessMode::Read, - new_recursion_level, - )? + // FIXME: Can this result in invalid address space state? + let ext_addrspace = &foreign_address_space; + let (frame, _, _) = { + let g = ext_addrspace.acquire_write(); + correct_inner( + ext_addrspace, + g, + src_page, + AccessMode::Read, + new_recursion_level, + )? + }; + + addr_space_guard = addr_space_lock.acquire_write(); + addr_space = &mut *addr_space_guard; + flusher = Flusher::with_cpu_set( + &mut addr_space.used_by, + &addr_space_lock.tlb_ack, + ); + guard = foreign_address_space.acquire_upgradeable_read(); + + frame + } }; - addr_space_guard = addr_space_lock.acquire_write(); - addr_space = &mut *addr_space_guard; - flusher = - Flusher::with_cpu_set(&mut addr_space.used_by, &addr_space_lock.tlb_ack); - guard = foreign_address_space.acquire_upgradeable_read(); + let info = + get_page_info(src_frame).expect("all allocated frames need a PageInfo"); - frame - }; + match info.add_ref(RefKind::Shared) { + Ok(()) => src_frame, + Err(AddRefError::CowToShared) => { + let CowResult { + new_frame, + old_frame, + } = cow(src_frame, info, RefKind::Shared)?; - let info = get_page_info(src_frame).expect("all allocated frames need a PageInfo"); + if let Some(old_frame) = old_frame { + flusher.queue(old_frame, None, TlbShootdownActions::FREE); + flusher.flush(); + } - match info.add_ref(RefKind::Shared) { - Ok(()) => src_frame, - Err(AddRefError::CowToShared) => { - let CowResult { - new_frame, - old_frame, - } = cow(src_frame, info, RefKind::Shared)?; + let mut guard = RwLockUpgradableGuard::upgrade(guard); - if let Some(old_frame) = old_frame { - flusher.queue(old_frame, None, TlbShootdownActions::FREE); - flusher.flush(); + // TODO: flusher + unsafe { + guard + .table + .utable + .remap_with_full(src_page.start_address(), |_, f| { + (new_frame.base(), f) + }); + } + + new_frame } - - let mut guard = RwLockUpgradableGuard::upgrade(guard); - - // TODO: flusher - unsafe { - guard - .table - .utable - .remap_with_full(src_page.start_address(), |_, f| { - (new_frame.base(), f) - }); - } - - new_frame + Err(AddRefError::SharedToCow) => unreachable!(), + Err(AddRefError::RcOverflow) => return Err(PfError::Oom), } - Err(AddRefError::SharedToCow) => unreachable!(), - Err(AddRefError::RcOverflow) => return Err(PfError::Oom), } - } else { - // Grant did not exist, but we did own a Provider::External mapping, and cannot - // simply let the current context fail. TODO: But all borrowed memory shouldn't - // really be lazy though? TODO: Should a grant be created? + _ => { + // Grant did not exist, but we did own a Provider::External mapping, and cannot + // simply let the current context fail. TODO: But all borrowed memory shouldn't + // really be lazy though? TODO: Should a grant be created? - let mut guard = RwLockUpgradableGuard::upgrade(guard); + let mut guard = RwLockUpgradableGuard::upgrade(guard); - // TODO: Should this be called? - log::warn!("Mapped zero page since grant didn't exist"); - map_zeroed( - &mut guard.table.utable, - src_page, - grant_flags, - access == AccessMode::Write, - )? + // TODO: Should this be called? + log::warn!("Mapped zero page since grant didn't exist"); + map_zeroed( + &mut guard.table.utable, + src_page, + grant_flags, + access == AccessMode::Write, + )? + } } } // TODO: NonfatalInternalError if !MAP_LAZY and this page fault occurs. diff --git a/src/context/switch.rs b/src/context/switch.rs index 015e6c56d4..2259be33e9 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -107,14 +107,19 @@ pub fn tick() { /// # Safety /// This function involves unsafe operations such as resetting state and releasing locks. pub unsafe extern "C" fn switch_finish_hook() { - if let Some(switch_result) = PercpuBlock::current().switch_internals.switch_result.take() { - drop(switch_result); - } else { - // TODO: unreachable_unchecked()? - crate::arch::stop::emergency_reset(); + unsafe { + match PercpuBlock::current().switch_internals.switch_result.take() { + Some(switch_result) => { + drop(switch_result); + } + _ => { + // TODO: unreachable_unchecked()? + crate::arch::stop::emergency_reset(); + } + } + arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); + crate::percpu::switch_arch_hook(); } - arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); - crate::percpu::switch_arch_hook(); } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -215,91 +220,97 @@ pub fn switch() -> SwitchResult { }; // Switch process states, TSS stack pointer, and store new context ID - if let Some((mut prev_context_guard, mut next_context_guard)) = switch_context_opt { - // Update context states and prepare for the switch. - let prev_context = &mut *prev_context_guard; - let next_context = &mut *next_context_guard; + match switch_context_opt { + Some((mut prev_context_guard, mut next_context_guard)) => { + // Update context states and prepare for the switch. + let prev_context = &mut *prev_context_guard; + let next_context = &mut *next_context_guard; - // Set the previous context as "not running" - prev_context.running = false; + // Set the previous context as "not running" + prev_context.running = false; - // Set the next context as "running" - next_context.running = true; - // Set the CPU ID for the next context - next_context.cpu_id = Some(cpu_id); + // Set the next context as "running" + next_context.running = true; + // Set the CPU ID for the next context + next_context.cpu_id = Some(cpu_id); - let percpu = PercpuBlock::current(); - unsafe { - percpu.switch_internals.set_current_context(Arc::clone( - ArcRwSpinlockWriteGuard::rwlock(&next_context_guard), - )); + let percpu = PercpuBlock::current(); + unsafe { + percpu.switch_internals.set_current_context(Arc::clone( + ArcRwSpinlockWriteGuard::rwlock(&next_context_guard), + )); + } + + // FIXME set the switch result in arch::switch_to instead + let prev_context = unsafe { + mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *prev_context_guard) + }; + let next_context = unsafe { + mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *next_context_guard) + }; + + percpu + .switch_internals + .switch_result + .set(Some(SwitchResultInner { + _prev_guard: prev_context_guard, + _next_guard: next_context_guard, + })); + + /*let (ptrace_session, ptrace_flags) = if let Some((session, bp)) = ptrace::sessions() + .get(&next_context.pid) + .map(|s| (Arc::downgrade(s), s.data.lock().breakpoint)) + { + (Some(session), bp.map_or(PtraceFlags::empty(), |f| f.flags)) + } else { + (None, PtraceFlags::empty()) + };*/ + let ptrace_flags = PtraceFlags::empty(); + + //*percpu.ptrace_session.borrow_mut() = ptrace_session; + percpu.ptrace_flags.set(ptrace_flags); + prev_context.inside_syscall = + percpu.inside_syscall.replace(next_context.inside_syscall); + + #[cfg(feature = "syscall_debug")] + { + prev_context.syscall_debug_info = percpu + .syscall_debug_info + .replace(next_context.syscall_debug_info); + prev_context.syscall_debug_info.on_switch_from(); + next_context.syscall_debug_info.on_switch_to(); + } + + percpu + .switch_internals + .being_sigkilled + .set(next_context.being_sigkilled); + + unsafe { + arch::switch_to(prev_context, next_context); + } + + // NOTE: After switch_to is called, the return address can even be different from the + // current return address, meaning that we cannot use local variables here, and that we + // need to use the `switch_finish_hook` to be able to release the locks. Newly created + // contexts will return directly to the function pointer passed to context::spawn, and not + // reach this code until the next context switch back. + if next_context.userspace { + percpu.stats.set_state(cpu_stats::CpuState::User); + } else { + percpu.stats.set_state(cpu_stats::CpuState::Kernel); + } + + SwitchResult::Switched } + _ => { + // No target was found, unset global lock and return + arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); - // FIXME set the switch result in arch::switch_to instead - let prev_context = - unsafe { mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *prev_context_guard) }; - let next_context = - unsafe { mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *next_context_guard) }; + percpu.stats.set_state(cpu_stats::CpuState::Idle); - percpu - .switch_internals - .switch_result - .set(Some(SwitchResultInner { - _prev_guard: prev_context_guard, - _next_guard: next_context_guard, - })); - - /*let (ptrace_session, ptrace_flags) = if let Some((session, bp)) = ptrace::sessions() - .get(&next_context.pid) - .map(|s| (Arc::downgrade(s), s.data.lock().breakpoint)) - { - (Some(session), bp.map_or(PtraceFlags::empty(), |f| f.flags)) - } else { - (None, PtraceFlags::empty()) - };*/ - let ptrace_flags = PtraceFlags::empty(); - - //*percpu.ptrace_session.borrow_mut() = ptrace_session; - percpu.ptrace_flags.set(ptrace_flags); - prev_context.inside_syscall = percpu.inside_syscall.replace(next_context.inside_syscall); - - #[cfg(feature = "syscall_debug")] - { - prev_context.syscall_debug_info = percpu - .syscall_debug_info - .replace(next_context.syscall_debug_info); - prev_context.syscall_debug_info.on_switch_from(); - next_context.syscall_debug_info.on_switch_to(); + SwitchResult::AllContextsIdle } - - percpu - .switch_internals - .being_sigkilled - .set(next_context.being_sigkilled); - - unsafe { - arch::switch_to(prev_context, next_context); - } - - // NOTE: After switch_to is called, the return address can even be different from the - // current return address, meaning that we cannot use local variables here, and that we - // need to use the `switch_finish_hook` to be able to release the locks. Newly created - // contexts will return directly to the function pointer passed to context::spawn, and not - // reach this code until the next context switch back. - if next_context.userspace { - percpu.stats.set_state(cpu_stats::CpuState::User); - } else { - percpu.stats.set_state(cpu_stats::CpuState::Kernel); - } - - SwitchResult::Switched - } else { - // No target was found, unset global lock and return - arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst); - - percpu.stats.set_state(cpu_stats::CpuState::Idle); - - SwitchResult::AllContextsIdle } } diff --git a/src/devices/graphical_debug/display.rs b/src/devices/graphical_debug/display.rs index 99f2eadea9..34c3842692 100644 --- a/src/devices/graphical_debug/display.rs +++ b/src/devices/graphical_debug/display.rs @@ -48,42 +48,46 @@ impl Display { /// Sync from offscreen to onscreen, unsafe because it trusts provided x, y, w, h pub(super) unsafe fn sync(&mut self, x: usize, y: usize, w: usize, mut h: usize) { - if let Some(offscreen) = &self.offscreen { - let mut y = y; - while h > 0 { - let src_y = (self.offset_y + y) % self.height; - let src_offset = src_y * self.stride + x; - let dst_offset = y * self.stride + x; + unsafe { + if let Some(offscreen) = &self.offscreen { + let mut y = y; + while h > 0 { + let src_y = (self.offset_y + y) % self.height; + let src_offset = src_y * self.stride + x; + let dst_offset = y * self.stride + x; - ptr::copy( - offscreen.as_ptr().add(src_offset), - self.onscreen_ptr.add(dst_offset), - w, - ); + ptr::copy( + offscreen.as_ptr().add(src_offset), + self.onscreen_ptr.add(dst_offset), + w, + ); - y += 1; - h -= 1; + y += 1; + h -= 1; + } } } } // sync the whole screen (faster) pub(super) unsafe fn sync_screen(&mut self) { - if let Some(offscreen) = &self.offscreen { - let stride_bytes = self.stride; - let first_part_len = (self.height - self.offset_y) * stride_bytes; - let second_part_len = self.offset_y * stride_bytes; + unsafe { + if let Some(offscreen) = &self.offscreen { + let stride_bytes = self.stride; + let first_part_len = (self.height - self.offset_y) * stride_bytes; + let second_part_len = self.offset_y * stride_bytes; - ptr::copy( - offscreen.as_ptr().add(self.offset_y * stride_bytes), - self.onscreen_ptr, - first_part_len, - ); - ptr::copy( - offscreen.as_ptr(), - self.onscreen_ptr.add(first_part_len), - second_part_len, - ); + ptr::copy( + offscreen.as_ptr().add(self.offset_y * stride_bytes), + self.onscreen_ptr, + first_part_len, + ); + ptr::copy( + offscreen.as_ptr(), + self.onscreen_ptr.add(first_part_len), + second_part_len, + ); + } } } } diff --git a/src/devices/uart_16550.rs b/src/devices/uart_16550.rs index 2a30a42a92..bfc6baa3c0 100644 --- a/src/devices/uart_16550.rs +++ b/src/devices/uart_16550.rs @@ -67,14 +67,14 @@ impl SerialPort> { impl SerialPort> { #[allow(dead_code)] pub unsafe fn new(base: usize) -> &'static mut SerialPort> { - &mut *(base as *mut Self) + unsafe { &mut *(base as *mut Self) } } } impl SerialPort> { #[allow(dead_code)] pub unsafe fn new(base: usize) -> &'static mut SerialPort> { - &mut *(base as *mut Self) + unsafe { &mut *(base as *mut Self) } } } diff --git a/src/dtb/irqchip.rs b/src/dtb/irqchip.rs index cec0ba057c..3b1d099fcf 100644 --- a/src/dtb/irqchip.rs +++ b/src/dtb/irqchip.rs @@ -228,14 +228,17 @@ impl IrqChipList { debug_assert!(queue[0..queue_idx].contains(&connection.parent)); if let Some(parent_interrupt) = connection.parent_interrupt { let parent = &self.chips[connection.parent]; - if let Ok(virq) = parent.ic.irq_xlate(parent_interrupt) { - // assert is unused - irq_desc[virq].basic.child_ic_idx = Some(cur_idx); - } else { - error!( - "Cannot connect irq chip {} to parent irq {} : {:?}", - cur_chip.phandle, parent.phandle, parent_interrupt - ); + match parent.ic.irq_xlate(parent_interrupt) { + Ok(virq) => { + // assert is unused + irq_desc[virq].basic.child_ic_idx = Some(cur_idx); + } + _ => { + error!( + "Cannot connect irq chip {} to parent irq {} : {:?}", + cur_chip.phandle, parent.phandle, parent_interrupt + ); + } } } } @@ -295,12 +298,17 @@ impl IrqChipCore { pub fn trigger_virq(&mut self, virq: u32) { if virq < 1024 { let desc = &mut self.irq_desc[virq as usize]; - if let Some(handler) = &mut desc.handler { - handler.irq_handler(virq); - } else if let Some(ic_idx) = desc.basic.child_ic_idx { - self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq); - } else { - irq_trigger(virq as u8); + match &mut desc.handler { + Some(handler) => { + handler.irq_handler(virq); + } + _ => { + if let Some(ic_idx) = desc.basic.child_ic_idx { + self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq); + } else { + irq_trigger(virq as u8); + } + } } } } @@ -336,7 +344,9 @@ impl IrqChipCore { } pub unsafe fn acknowledge(irq: usize) { - IRQ_CHIP.irq_eoi(irq as u32); + unsafe { + IRQ_CHIP.irq_eoi(irq as u32); + } } const INIT_HANDLER: Option> = None; diff --git a/src/externs.rs b/src/externs.rs index 42f111dc41..a4c57e8f4a 100644 --- a/src/externs.rs +++ b/src/externs.rs @@ -8,39 +8,41 @@ const WORD_SIZE: usize = mem::size_of::(); /// /// This faster implementation works by copying bytes not one-by-one, but in /// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures). -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, len: usize) -> *mut u8 { - // TODO: Alignment? Some sources claim that even on relatively modern µ-arches, unaligned - // accesses spanning two pages, can take dozens of cycles. That means chunk-based memcpy can - // even be slower for small lengths if alignment is not taken into account. - // - // TODO: Optimize out smaller loops by first checking if len < WORD_SIZE, and possibly if - // dest + WORD_SIZE spans two pages, then doing one unaligned copy, then aligning up, and then - // doing one last unaligned copy? - // - // TODO: While we use the -fno-builtin equivalent, can we guarantee LLVM won't insert memcpy - // call inside here? Maybe write it in assembly? + unsafe { + // TODO: Alignment? Some sources claim that even on relatively modern µ-arches, unaligned + // accesses spanning two pages, can take dozens of cycles. That means chunk-based memcpy can + // even be slower for small lengths if alignment is not taken into account. + // + // TODO: Optimize out smaller loops by first checking if len < WORD_SIZE, and possibly if + // dest + WORD_SIZE spans two pages, then doing one unaligned copy, then aligning up, and then + // doing one last unaligned copy? + // + // TODO: While we use the -fno-builtin equivalent, can we guarantee LLVM won't insert memcpy + // call inside here? Maybe write it in assembly? - let mut i = 0_usize; + let mut i = 0_usize; - // First we copy len / WORD_SIZE chunks... + // First we copy len / WORD_SIZE chunks... - let chunks = len / WORD_SIZE; + let chunks = len / WORD_SIZE; - while i < chunks * WORD_SIZE { - dest.add(i) - .cast::() - .write_unaligned(src.add(i).cast::().read_unaligned()); - i += WORD_SIZE; + while i < chunks * WORD_SIZE { + dest.add(i) + .cast::() + .write_unaligned(src.add(i).cast::().read_unaligned()); + i += WORD_SIZE; + } + + // .. then we copy len % WORD_SIZE bytes + while i < len { + dest.add(i).write(src.add(i).read()); + i += 1; + } + + dest } - - // .. then we copy len % WORD_SIZE bytes - while i < len { - dest.add(i).write(src.add(i).read()); - i += 1; - } - - dest } /// Memmove @@ -49,48 +51,50 @@ pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, len: usize) -> *m /// /// This faster implementation works by copying bytes not one-by-one, but in /// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures). -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, len: usize) -> *mut u8 { - let chunks = len / WORD_SIZE; + unsafe { + let chunks = len / WORD_SIZE; - // TODO: also require dest - src < len before choosing to copy backwards? - if src < dest as *const u8 { - // We have to copy backwards if copying upwards. + // TODO: also require dest - src < len before choosing to copy backwards? + if src < dest as *const u8 { + // We have to copy backwards if copying upwards. - let mut i = len; + let mut i = len; - while i != chunks * WORD_SIZE { - i -= 1; - dest.add(i).write(src.add(i).read()); + while i != chunks * WORD_SIZE { + i -= 1; + dest.add(i).write(src.add(i).read()); + } + + while i > 0 { + i -= WORD_SIZE; + + dest.add(i) + .cast::() + .write_unaligned(src.add(i).cast::().read_unaligned()); + } + } else { + // We have to copy forward if copying downwards. + + let mut i = 0_usize; + + while i < chunks * WORD_SIZE { + dest.add(i) + .cast::() + .write_unaligned(src.add(i).cast::().read_unaligned()); + + i += WORD_SIZE; + } + + while i < len { + dest.add(i).write(src.add(i).read()); + i += 1; + } } - while i > 0 { - i -= WORD_SIZE; - - dest.add(i) - .cast::() - .write_unaligned(src.add(i).cast::().read_unaligned()); - } - } else { - // We have to copy forward if copying downwards. - - let mut i = 0_usize; - - while i < chunks * WORD_SIZE { - dest.add(i) - .cast::() - .write_unaligned(src.add(i).cast::().read_unaligned()); - - i += WORD_SIZE; - } - - while i < len { - dest.add(i).write(src.add(i).read()); - i += 1; - } + dest } - - dest } /// Memset @@ -99,26 +103,28 @@ pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, len: usize) -> * /// /// This faster implementation works by setting bytes not one-by-one, but in /// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures). -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn memset(dest: *mut u8, byte: i32, len: usize) -> *mut u8 { - let byte = byte as u8; + unsafe { + let byte = byte as u8; - let mut i = 0; + let mut i = 0; - let broadcasted = usize::from_ne_bytes([byte; WORD_SIZE]); - let chunks = len / WORD_SIZE; + let broadcasted = usize::from_ne_bytes([byte; WORD_SIZE]); + let chunks = len / WORD_SIZE; - while i < chunks * WORD_SIZE { - dest.add(i).cast::().write_unaligned(broadcasted); - i += WORD_SIZE; + while i < chunks * WORD_SIZE { + dest.add(i).cast::().write_unaligned(broadcasted); + i += WORD_SIZE; + } + + while i < len { + dest.add(i).write(byte); + i += 1; + } + + dest } - - while i < len { - dest.add(i).write(byte); - i += 1; - } - - dest } /// Memcmp @@ -127,38 +133,40 @@ pub unsafe extern "C" fn memset(dest: *mut u8, byte: i32, len: usize) -> *mut u8 /// /// This faster implementation works by comparing bytes not one-by-one, but in /// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures). -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, len: usize) -> i32 { - let mut i = 0_usize; + unsafe { + let mut i = 0_usize; - // First compare WORD_SIZE chunks... - let chunks = len / WORD_SIZE; + // First compare WORD_SIZE chunks... + let chunks = len / WORD_SIZE; - while i < chunks * WORD_SIZE { - let a = s1.add(i).cast::().read_unaligned(); - let b = s2.add(i).cast::().read_unaligned(); + while i < chunks * WORD_SIZE { + let a = s1.add(i).cast::().read_unaligned(); + let b = s2.add(i).cast::().read_unaligned(); - if a != b { - // x86 has had bswap since the 80486, and the compiler will likely use the faster - // movbe. AArch64 has the REV instruction, which I think is universally available. - let diff = usize::from_be(a).wrapping_sub(usize::from_be(b)) as isize; + if a != b { + // x86 has had bswap since the 80486, and the compiler will likely use the faster + // movbe. AArch64 has the REV instruction, which I think is universally available. + let diff = usize::from_be(a).wrapping_sub(usize::from_be(b)) as isize; - // TODO: If chunk size == 32 bits, diff can be returned directly. - return diff.signum() as i32; + // TODO: If chunk size == 32 bits, diff can be returned directly. + return diff.signum() as i32; + } + i += WORD_SIZE; } - i += WORD_SIZE; - } - // ... and then compare bytes. - while i < len { - let a = s1.add(i).read(); - let b = s2.add(i).read(); + // ... and then compare bytes. + while i < len { + let a = s1.add(i).read(); + let b = s2.add(i).read(); - if a != b { - return i32::from(a) - i32::from(b); + if a != b { + return i32::from(a) - i32::from(b); + } + i += 1; } - i += 1; - } - 0 + 0 + } } diff --git a/src/main.rs b/src/main.rs index dc7098f383..d890ec28b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -274,7 +274,7 @@ macro_rules! linker_offsets( $( #[inline] pub fn $name() -> usize { - extern "C" { + unsafe extern "C" { // TODO: UnsafeCell? static $name: u8; } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index bf294ed4ae..8a3f4065a2 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -230,29 +230,31 @@ pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) { } pub unsafe fn deallocate_frame(frame: Frame) { - deallocate_p2frame(frame, 0) + unsafe { deallocate_p2frame(frame, 0) } } // Helper function for quickly mapping device memory pub unsafe fn map_device_memory(addr: PhysicalAddress, len: usize) -> VirtualAddress { - let mut mapper_lock = KernelMapper::lock(); - let mapper = mapper_lock - .get_mut() - .expect("KernelMapper mapper locked re-entrant in map_device_memory"); - let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data())); - let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data())); - for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE { - let (_, flush) = mapper - .map_linearly( - base.add(page_idx * crate::memory::PAGE_SIZE), - PageFlags::new() - .write(true) - .custom_flag(EntryFlags::NO_CACHE.bits(), true), - ) - .expect("failed to linearly map SDT"); - flush.flush(); + unsafe { + let mut mapper_lock = KernelMapper::lock(); + let mapper = mapper_lock + .get_mut() + .expect("KernelMapper mapper locked re-entrant in map_device_memory"); + let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data())); + let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data())); + for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE { + let (_, flush) = mapper + .map_linearly( + base.add(page_idx * crate::memory::PAGE_SIZE), + PageFlags::new() + .write(true) + .custom_flag(EntryFlags::NO_CACHE.bits(), true), + ) + .expect("failed to linearly map SDT"); + flush.flush(); + } + RmmA::phys_to_virt(addr) } - RmmA::phys_to_virt(addr) } const ORDER_COUNT: u32 = 11; @@ -1063,8 +1065,10 @@ impl FrameAllocator for TheFrameAllocator { allocate_p2frame(order).map(|f| f.base()) } unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) { - let order = count.data().next_power_of_two().trailing_zeros(); - deallocate_p2frame(Frame::containing(address), order) + unsafe { + let order = count.data().next_power_of_two().trailing_zeros(); + deallocate_p2frame(Frame::containing(address), order) + } } unsafe fn usage(&self) -> FrameUsage { FrameUsage::new( diff --git a/src/panic.rs b/src/panic.rs index 8efcc3a46e..eddc0f0ea7 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -59,37 +59,39 @@ fn rust_begin_unwind(info: &PanicInfo) -> ! { /// Get a stack trace #[inline(never)] pub unsafe fn stack_trace() { - let mapper = KernelMapper::lock(); + unsafe { + let mapper = KernelMapper::lock(); - let mut frame = StackTrace::start(); + let mut frame = StackTrace::start(); - //Maximum 64 frames - for _ in 0..64 { - if let Some(frame_) = frame { - let fp_virt = VirtualAddress::new(frame_.fp); - let pc_virt = VirtualAddress::new(frame_.pc_ptr as usize); - if fp_virt.data() >= USER_END_OFFSET - && pc_virt.data() >= USER_END_OFFSET - && (fp_virt.data() as *const usize).is_aligned() - && (pc_virt.data() as *const usize).is_aligned() - && mapper.translate(fp_virt).is_some() - && mapper.translate(pc_virt).is_some() - { - let pc = *frame_.pc_ptr; - if pc == 0 { - println!(" {:>016x}: EMPTY RETURN", frame_.fp); - break; + //Maximum 64 frames + for _ in 0..64 { + if let Some(frame_) = frame { + let fp_virt = VirtualAddress::new(frame_.fp); + let pc_virt = VirtualAddress::new(frame_.pc_ptr as usize); + if fp_virt.data() >= USER_END_OFFSET + && pc_virt.data() >= USER_END_OFFSET + && (fp_virt.data() as *const usize).is_aligned() + && (pc_virt.data() as *const usize).is_aligned() + && mapper.translate(fp_virt).is_some() + && mapper.translate(pc_virt).is_some() + { + let pc = *frame_.pc_ptr; + if pc == 0 { + println!(" {:>016x}: EMPTY RETURN", frame_.fp); + break; + } else { + println!(" FP {:>016x}: PC {:>016x}", frame_.fp, pc); + symbol_trace(pc); + frame = frame_.next(); + } } else { - println!(" FP {:>016x}: PC {:>016x}", frame_.fp, pc); - symbol_trace(pc); - frame = frame_.next(); + println!(" {:>016x}: GUARD PAGE", frame_.fp); + break; } } else { - println!(" {:>016x}: GUARD PAGE", frame_.fp); break; } - } else { - break; } } } @@ -98,45 +100,47 @@ pub unsafe fn stack_trace() { //TODO: Do not create Elf object for every symbol lookup #[inline(never)] pub unsafe fn symbol_trace(addr: usize) { - let kernel_ptr = crate::KERNEL_OFFSET as *const u8; - let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst)); + unsafe { + let kernel_ptr = crate::KERNEL_OFFSET as *const u8; + let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst)); - if let Ok(elf) = Elf::from(kernel_slice) { - let mut strtab_opt = None; - for section in elf.sections() { - if section.sh_type == ::goblin::elf::section_header::SHT_STRTAB { - strtab_opt = Some(section); - break; + if let Ok(elf) = Elf::from(kernel_slice) { + let mut strtab_opt = None; + for section in elf.sections() { + if section.sh_type == ::goblin::elf::section_header::SHT_STRTAB { + strtab_opt = Some(section); + break; + } } - } - if let Some(symbols) = elf.symbols() { - for sym in symbols { - if sym::st_type(sym.st_info) == sym::STT_FUNC - && addr >= sym.st_value as usize - && addr < (sym.st_value + sym.st_size) as usize - { - println!( - " {:>016X}+{:>04X}", - sym.st_value, - addr - sym.st_value as usize - ); + if let Some(symbols) = elf.symbols() { + for sym in symbols { + if sym::st_type(sym.st_info) == sym::STT_FUNC + && addr >= sym.st_value as usize + && addr < (sym.st_value + sym.st_size) as usize + { + println!( + " {:>016X}+{:>04X}", + sym.st_value, + addr - sym.st_value as usize + ); - if let Some(strtab) = strtab_opt { - let start = strtab.sh_offset as usize + sym.st_name as usize; - let mut end = start; - while end < elf.data.len() { - let b = elf.data[end]; - end += 1; - if b == 0 { - break; + if let Some(strtab) = strtab_opt { + let start = strtab.sh_offset as usize + sym.st_name as usize; + let mut end = start; + while end < elf.data.len() { + let b = elf.data[end]; + end += 1; + if b == 0 { + break; + } } - } - if end > start { - let sym_slice = &elf.data[start..end - 1]; - if let Ok(sym_name) = str::from_utf8(sym_slice) { - println!(" {:#}", demangle(sym_name)); + if end > start { + let sym_slice = &elf.data[start..end - 1]; + if let Ok(sym_name) = str::from_utf8(sym_slice) { + println!(" {:#}", demangle(sym_name)); + } } } } diff --git a/src/percpu.rs b/src/percpu.rs index ce461534d1..f35376fb3c 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -11,7 +11,8 @@ use crate::{ context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu}, cpu_set::{LogicalCpuId, MAX_CPU_COUNT}, cpu_stats::CpuStats, - ptrace::Session,syscall::debug::SyscallDebugInfo + ptrace::Session, + syscall::debug::SyscallDebugInfo, }; #[cfg(feature = "sys_stat")] @@ -119,45 +120,50 @@ impl PercpuBlock { crate::paging::RmmA::invalidate_all(); } - if let Some(ref addrsp) = &*self.current_addrsp.borrow() { + if let &Some(ref addrsp) = &*self.current_addrsp.borrow() { addrsp.tlb_ack.fetch_add(1, Ordering::Release); } } } pub unsafe fn switch_arch_hook() { - let percpu = PercpuBlock::current(); + unsafe { + let percpu = PercpuBlock::current(); - let cur_addrsp = percpu.current_addrsp.borrow(); - let next_addrsp = percpu.new_addrsp_tmp.take(); + let cur_addrsp = percpu.current_addrsp.borrow(); + let next_addrsp = percpu.new_addrsp_tmp.take(); - let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) { - (Some(ref p), Some(ref n)) => Arc::ptr_eq(p, n), - (Some(_), None) | (None, Some(_)) => false, - (None, None) => true, - }; - if retain_pgtbl { - // If we are not switching to a different address space, we can simply return early. - } - if let Some(ref prev_addrsp) = &*cur_addrsp { - prev_addrsp - .acquire_read() - .used_by - .atomic_clear(percpu.cpu_id); - } + let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) { + (&Some(ref p), &Some(ref n)) => Arc::ptr_eq(p, n), + (Some(_), None) | (None, Some(_)) => false, + (None, None) => true, + }; + if retain_pgtbl { + // If we are not switching to a different address space, we can simply return early. + } + if let &Some(ref prev_addrsp) = &*cur_addrsp { + prev_addrsp + .acquire_read() + .used_by + .atomic_clear(percpu.cpu_id); + } - drop(cur_addrsp); + drop(cur_addrsp); - // Tell future TLB shootdown handlers that old_addrsp_tmp is no longer the current address - // space. - *percpu.current_addrsp.borrow_mut() = next_addrsp; + // Tell future TLB shootdown handlers that old_addrsp_tmp is no longer the current address + // space. + *percpu.current_addrsp.borrow_mut() = next_addrsp; - if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() { - let next = next_addrsp.acquire_read(); + match &*percpu.current_addrsp.borrow() { + Some(next_addrsp) => { + let next = next_addrsp.acquire_read(); - next.used_by.atomic_set(percpu.cpu_id); - next.table.utable.make_current(); - } else { - crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3()); + next.used_by.atomic_set(percpu.cpu_id); + next.table.utable.make_current(); + } + _ => { + crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3()); + } + } } } impl PercpuBlock { diff --git a/src/scheme/irq.rs b/src/scheme/irq.rs index ec49b041b3..170f735e1d 100644 --- a/src/scheme/irq.rs +++ b/src/scheme/irq.rs @@ -164,35 +164,37 @@ impl IrqScheme { phandle: usize, path_str: &str, ) -> Result<(Handle, InternalFlags)> { - let addr: Vec = path_str - .split(',') - .map(|x| u32::from_str(x).or(Err(Error::new(ENOENT)))) - .try_collect()?; - let ic_idx = IRQ_CHIP - .phandle_to_ic_idx(phandle as u32) - .ok_or(Error::new(ENOENT))?; - Ok({ - if flags & O_CREAT == 0 && flags & O_STAT == 0 { - return Err(Error::new(EINVAL)); - } - let irq_number = IRQ_CHIP - .irq_xlate(ic_idx, addr.as_slice()) - .or(Err(Error::new(ENOENT)))?; - log::debug!("open_phandle_irq virq={}", irq_number); - if flags & O_STAT == 0 { - if is_reserved(LogicalCpuId::new(0), irq_number as u8) { - return Err(Error::new(EEXIST)); + unsafe { + let addr: Vec = path_str + .split(',') + .map(|x| u32::from_str(x).or(Err(Error::new(ENOENT)))) + .try_collect()?; + let ic_idx = IRQ_CHIP + .phandle_to_ic_idx(phandle as u32) + .ok_or(Error::new(ENOENT))?; + Ok({ + if flags & O_CREAT == 0 && flags & O_STAT == 0 { + return Err(Error::new(EINVAL)); } - set_reserved(LogicalCpuId::new(0), irq_number as u8, true); - } - ( - Handle::Irq { - ack: AtomicUsize::new(0), - irq: irq_number as u8, - }, - InternalFlags::empty(), - ) - }) + let irq_number = IRQ_CHIP + .irq_xlate(ic_idx, addr.as_slice()) + .or(Err(Error::new(ENOENT)))?; + log::debug!("open_phandle_irq virq={}", irq_number); + if flags & O_STAT == 0 { + if is_reserved(LogicalCpuId::new(0), irq_number as u8) { + return Err(Error::new(EEXIST)); + } + set_reserved(LogicalCpuId::new(0), irq_number as u8, true); + } + ( + Handle::Irq { + ack: AtomicUsize::new(0), + irq: irq_number as u8, + }, + InternalFlags::empty(), + ) + }) + } } } diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 21bd50a852..1d041d2642 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -247,15 +247,18 @@ impl SchemeList { return Err(Error::new(ENODEV)); }; - if let Some(ref mut names) = self.names.get_mut(&to) { - if names - .insert(name.to_string().into_boxed_str(), id) - .is_some() - { - return Err(Error::new(EEXIST)); + match self.names.get_mut(&to) { + Some(ref mut names) => { + if names + .insert(name.to_string().into_boxed_str(), id) + .is_some() + { + return Err(Error::new(EEXIST)); + } + } + _ => { + panic!("scheme namespace not found"); } - } else { - panic!("scheme namespace not found"); } } @@ -344,13 +347,16 @@ impl SchemeList { let (new_scheme, t) = scheme_fn(id); assert!(self.map.insert(id, new_scheme).is_none()); - if let Some(ref mut names) = self.names.get_mut(&ns) { - assert!(names - .insert(name.to_string().into_boxed_str(), id) - .is_none()); - } else { - // Nonexistent namespace, posssibly null namespace - return Err(Error::new(ENODEV)); + match self.names.get_mut(&ns) { + Some(ref mut names) => { + assert!(names + .insert(name.to_string().into_boxed_str(), id) + .is_none()); + } + _ => { + // Nonexistent namespace, posssibly null namespace + return Err(Error::new(ENODEV)); + } } Ok((id, t)) } diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 415e7e26f8..aec40f9830 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -259,12 +259,11 @@ impl ProcScheme { let operation_name = operation_str.ok_or(Error::new(EINVAL))?; let (mut handle, positioned) = match ty { OpenTy::Ctxt(context) => { - if let Some((kind, positioned)) = - self.openat_context(operation_name, Arc::clone(&context))? - { - (Handle { context, kind }, positioned) - } else { - return Err(Error::new(EINVAL)); + match self.openat_context(operation_name, Arc::clone(&context))? { + Some((kind, positioned)) => (Handle { context, kind }, positioned), + _ => { + return Err(Error::new(EINVAL)); + } } } OpenTy::Auth => { @@ -1006,7 +1005,7 @@ impl ContextHandle { verify_scheme(hopefully_this_scheme)?; let mut handles = HANDLES.write(); - let Handle { + let &Handle { kind: ContextHandle::AddrSpace { ref addrspace }, .. } = handles.get(&number).ok_or(Error::new(EBADF))? @@ -1201,7 +1200,7 @@ impl ContextHandle { buf.copy_common_bytes_from_slice(src_buf) } - ContextHandle::AddrSpace { ref addrspace } => { + &ContextHandle::AddrSpace { ref addrspace } => { let Ok(offset) = usize::try_from(offset) else { return Ok(0); }; @@ -1239,7 +1238,7 @@ impl ContextHandle { } ContextHandle::Filetable { data, .. } => read_from(buf, &data, offset), - ContextHandle::MmapMinAddr(ref addrspace) => { + &ContextHandle::MmapMinAddr(ref addrspace) => { buf.write_usize(addrspace.acquire_read().mmap_min)?; Ok(mem::size_of::()) } diff --git a/src/scheme/sys/context.rs b/src/scheme/sys/context.rs index 219c152c4d..bd70d19354 100644 --- a/src/scheme/sys/context.rs +++ b/src/scheme/sys/context.rs @@ -21,14 +21,15 @@ pub fn resource() -> Result> { let mut stat_string = String::new(); // TODO: All user programs must have some grant in order for executable memory to even // exist, but is this a good indicator of whether it is user or kernel? - stat_string.push(if let Ok(addr_space) = context.addr_space() { - if addr_space.acquire_read().grants.is_empty() { - 'K' - } else { - 'U' + stat_string.push(match context.addr_space() { + Ok(addr_space) => { + if addr_space.acquire_read().grants.is_empty() { + 'K' + } else { + 'U' + } } - } else { - 'R' + _ => 'R', }); match context.status { context::Status::Runnable => { @@ -49,10 +50,13 @@ pub fn resource() -> Result> { stat_string.push('+'); } - let cpu_string = if let Some(cpu_id) = context.cpu_id { - format!("{}", cpu_id) - } else { - format!("?") + let cpu_string = match context.cpu_id { + Some(cpu_id) => { + format!("{}", cpu_id) + } + _ => { + format!("?") + } }; let affinity = context.sched_affinity.to_string(); diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index 57a3b69168..6b1834c94f 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -190,7 +190,7 @@ impl KernelScheme for SysScheme { Handle::TopLevel | Handle::Resource { data: None, .. } => { return Err(Error::new(EISDIR)) } - Handle::Resource { + &Handle::Resource { data: Some(ref data), .. } => { diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 11321a3842..2977fd2782 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -955,7 +955,7 @@ impl UserInner { .get_mut(tag as usize) .ok_or(Error::new(EINVAL))? { - State::Waiting { ref mut fds, .. } => { + &mut State::Waiting { ref mut fds, .. } => { fds.take().ok_or(Error::new(ENOENT))?.remove(0) } _ => return Err(Error::new(ENOENT)), @@ -1105,11 +1105,14 @@ impl UserInner { .map(RwLock::into_inner) .collect(); - if let Some(context) = context.upgrade() { - context.write().unblock(); - *o = State::Responded(response); - } else { - states.remove(tag as usize); + match context.upgrade() { + Some(context) => { + context.write().unblock(); + *o = State::Responded(response); + } + _ => { + states.remove(tag as usize); + } } let unpin = true; @@ -1333,7 +1336,7 @@ impl UserInner { .get_mut(request_id) .ok_or(Error::new(EINVAL))? { - State::Waiting { ref mut fds, .. } => *fds = Some(descs), + &mut State::Waiting { ref mut fds, .. } => *fds = Some(descs), _ => return Err(Error::new(ENOENT)), }; @@ -1389,7 +1392,7 @@ impl UserInner { .get_mut(request_id) .ok_or(Error::new(EINVAL))? { - State::Waiting { ref mut fds, .. } => fds.take().ok_or(Error::new(ENOENT))?, + &mut State::Waiting { ref mut fds, .. } => fds.take().ok_or(Error::new(ENOENT))?, _ => return Err(Error::new(ENOENT)), }; diff --git a/src/startup/memory.rs b/src/startup/memory.rs index 5d14d10ead..ad00af4f4d 100644 --- a/src/startup/memory.rs +++ b/src/startup/memory.rs @@ -168,275 +168,281 @@ pub fn register_bootloader_areas(areas_base: usize, areas_size: usize) { } unsafe fn add_memory(areas: &mut [MemoryArea], area_i: &mut usize, mut area: MemoryEntry) { - for reservation in (*MEMORY_MAP.get()).non_free() { - if area.end > reservation.start && area.end <= reservation.end { - log::info!( - "Memory {:X}:{:X} overlaps with reservation {:X}:{:X}", - area.start, - area.end, - reservation.start, - reservation.end - ); - area.end = reservation.start; - } - if area.start >= area.end { - return; - } + unsafe { + for reservation in (*MEMORY_MAP.get()).non_free() { + if area.end > reservation.start && area.end <= reservation.end { + log::info!( + "Memory {:X}:{:X} overlaps with reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + area.end = reservation.start; + } + if area.start >= area.end { + return; + } - if area.start >= reservation.start && area.start < reservation.end { - log::info!( - "Memory {:X}:{:X} overlaps with reservation {:X}:{:X}", - area.start, - area.end, - reservation.start, - reservation.end - ); - area.start = reservation.end; - } - if area.start >= area.end { - return; - } + if area.start >= reservation.start && area.start < reservation.end { + log::info!( + "Memory {:X}:{:X} overlaps with reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + area.start = reservation.end; + } + if area.start >= area.end { + return; + } - if area.start <= reservation.start && area.end > reservation.start { - log::info!( - "Memory {:X}:{:X} contains reservation {:X}:{:X}", - area.start, - area.end, - reservation.start, - reservation.end - ); - debug_assert!(area.start < reservation.start && reservation.end < area.end, + if area.start <= reservation.start && area.end > reservation.start { + log::info!( + "Memory {:X}:{:X} contains reservation {:X}:{:X}", + area.start, + area.end, + reservation.start, + reservation.end + ); + debug_assert!(area.start < reservation.start && reservation.end < area.end, "Should've contained reservation entirely: memory block {:X}:{:X} reservation {:X}:{:X}", area.start, area.end, reservation.start, reservation.end ); - // recurse on first part of split memory block + // recurse on first part of split memory block - add_memory( - areas, - area_i, - MemoryEntry { - end: reservation.start, - ..area - }, - ); + add_memory( + areas, + area_i, + MemoryEntry { + end: reservation.start, + ..area + }, + ); - // and continue with the second part - area.start = reservation.end; - } - debug_assert!( - area.intersect(reservation).is_none(), - "Intersects with reservation! memory block {:X}:{:X} reservation {:X}:{:X}", - area.start, - area.end, - reservation.start, - reservation.end - ); - debug_assert!( - area.start < area.end, - "Empty memory block {:X}:{:X}", - area.start, - area.end - ); - } - - // Combine overlapping memory areas - let mut other_i = 0; - while other_i < *area_i { - let other = &areas[other_i]; - let other = MemoryEntry { - start: other.base.data(), - end: other.base.data().saturating_add(other.size), - kind: BootloaderMemoryKind::Free, - }; - if let Some(union) = area.combine(&other) { - log::debug!( - "{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", + // and continue with the second part + area.start = reservation.end; + } + debug_assert!( + area.intersect(reservation).is_none(), + "Intersects with reservation! memory block {:X}:{:X} reservation {:X}:{:X}", area.start, area.end, - other.start, - other.end, - union.start, - union.end + reservation.start, + reservation.end + ); + debug_assert!( + area.start < area.end, + "Empty memory block {:X}:{:X}", + area.start, + area.end ); - area = union; - *area_i -= 1; // delete the original memory chunk - areas[other_i] = areas[*area_i]; - } else { - other_i += 1; } - } - areas[*area_i].base = PhysicalAddress::new(area.start); - areas[*area_i].size = area.end - area.start; - *area_i += 1; + // Combine overlapping memory areas + let mut other_i = 0; + while other_i < *area_i { + let other = &areas[other_i]; + let other = MemoryEntry { + start: other.base.data(), + end: other.base.data().saturating_add(other.size), + kind: BootloaderMemoryKind::Free, + }; + if let Some(union) = area.combine(&other) { + log::debug!( + "{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", + area.start, + area.end, + other.start, + other.end, + union.start, + union.end + ); + area = union; + *area_i -= 1; // delete the original memory chunk + areas[other_i] = areas[*area_i]; + } else { + other_i += 1; + } + } + + areas[*area_i].base = PhysicalAddress::new(area.start); + areas[*area_i].size = area.end - area.start; + *area_i += 1; + } } unsafe fn map_memory(areas: &[MemoryArea], mut bump_allocator: &mut BumpAllocator) { - let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) - .expect("failed to create Mapper"); + unsafe { + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); - if cfg!(target_arch = "x86") { - // Pre-allocate all kernel PD entries so that when the page table is copied, - // these entries are synced between processes - for i in 512..1024 { - use rmm::{FrameAllocator, PageEntry}; + if cfg!(target_arch = "x86") { + // Pre-allocate all kernel PD entries so that when the page table is copied, + // these entries are synced between processes + for i in 512..1024 { + use rmm::{FrameAllocator, PageEntry}; - let phys = mapper - .allocator_mut() - .allocate_one() - .expect("failed to map page table"); - let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE; - mapper - .table() - .set_entry(i, PageEntry::new(phys.data(), flags)); - } - } - - // Map all physical areas at PHYS_OFFSET - for area in areas.iter() { - for i in 0..area.size / PAGE_SIZE { - let phys = area.base.add(i * PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - let kernel_area = (*MEMORY_MAP.get()).kernel().unwrap(); - let kernel_base = kernel_area.start; - let kernel_size = kernel_area.end - kernel_area.start; - // Map kernel at KERNEL_OFFSET and identity map too - for i in 0..kernel_size / A::PAGE_SIZE { - let phys = PhysicalAddress::new(kernel_base + i * PAGE_SIZE); - let virt = VirtualAddress::new(KERNEL_OFFSET + i * PAGE_SIZE); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - - let virt = A::phys_to_virt(phys); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - - for area in (*MEMORY_MAP.get()).identity_mapped() { - let base = area.start; - let size = area.end - area.start; - for i in 0..size / PAGE_SIZE { - let phys = PhysicalAddress::new(base + i * PAGE_SIZE); - let virt = A::phys_to_virt(phys); - let flags = page_flags::(virt); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - //map dev mem - for area in (*MEMORY_MAP.get()).devices() { - let base = area.start; - let size = area.end - area.start; - for i in 0..size / PAGE_SIZE { - let phys = PhysicalAddress::new(base + i * PAGE_SIZE); - let virt = A::phys_to_virt(phys); - // use the same mair_el1 value with bootloader, - // mair_el1 == 0x00000000000044FF - // set mem_attr == device memory - let flags = page_flags::(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - // Ensure graphical debug region remains paged - #[cfg(feature = "graphical_debug")] - { - use crate::devices::graphical_debug::FRAMEBUFFER; - - let (phys, virt, size) = *FRAMEBUFFER.lock(); - - let pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; - for i in 0..pages { - let phys = PhysicalAddress::new(phys + i * PAGE_SIZE); - let virt = VirtualAddress::new(virt + i * PAGE_SIZE); - let flags = PageFlags::new().write(true).write_combining(true); - let flush = mapper - .map_phys(virt, phys, flags) - .expect("failed to map frame"); - flush.ignore(); // Not the active table - } - } - - log::debug!("Table: {:X}", mapper.table().phys().data()); - for i in 0..A::PAGE_ENTRIES { - if let Some(entry) = mapper.table().entry(i) { - if entry.present() { - log::debug!("{}: {:X}", i, entry.data()); + let phys = mapper + .allocator_mut() + .allocate_one() + .expect("failed to map page table"); + let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE; + mapper + .table() + .set_entry(i, PageEntry::new(phys.data(), flags)); } } - } - // Use the new table - mapper.make_current(); + // Map all physical areas at PHYS_OFFSET + for area in areas.iter() { + for i in 0..area.size / PAGE_SIZE { + let phys = area.base.add(i * PAGE_SIZE); + let virt = A::phys_to_virt(phys); + let flags = page_flags::(virt); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + let kernel_area = (*MEMORY_MAP.get()).kernel().unwrap(); + let kernel_base = kernel_area.start; + let kernel_size = kernel_area.end - kernel_area.start; + // Map kernel at KERNEL_OFFSET and identity map too + for i in 0..kernel_size / A::PAGE_SIZE { + let phys = PhysicalAddress::new(kernel_base + i * PAGE_SIZE); + let virt = VirtualAddress::new(KERNEL_OFFSET + i * PAGE_SIZE); + let flags = page_flags::(virt); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + + let virt = A::phys_to_virt(phys); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + + for area in (*MEMORY_MAP.get()).identity_mapped() { + let base = area.start; + let size = area.end - area.start; + for i in 0..size / PAGE_SIZE { + let phys = PhysicalAddress::new(base + i * PAGE_SIZE); + let virt = A::phys_to_virt(phys); + let flags = page_flags::(virt); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + //map dev mem + for area in (*MEMORY_MAP.get()).devices() { + let base = area.start; + let size = area.end - area.start; + for i in 0..size / PAGE_SIZE { + let phys = PhysicalAddress::new(base + i * PAGE_SIZE); + let virt = A::phys_to_virt(phys); + // use the same mair_el1 value with bootloader, + // mair_el1 == 0x00000000000044FF + // set mem_attr == device memory + let flags = page_flags::(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + // Ensure graphical debug region remains paged + #[cfg(feature = "graphical_debug")] + { + use crate::devices::graphical_debug::FRAMEBUFFER; + + let (phys, virt, size) = *FRAMEBUFFER.lock(); + + let pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + for i in 0..pages { + let phys = PhysicalAddress::new(phys + i * PAGE_SIZE); + let virt = VirtualAddress::new(virt + i * PAGE_SIZE); + let flags = PageFlags::new().write(true).write_combining(true); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); + flush.ignore(); // Not the active table + } + } + + log::debug!("Table: {:X}", mapper.table().phys().data()); + for i in 0..A::PAGE_ENTRIES { + if let Some(entry) = mapper.table().entry(i) { + if entry.present() { + log::debug!("{}: {:X}", i, entry.data()); + } + } + } + + // Use the new table + mapper.make_current(); + } } pub unsafe fn init(low_limit: Option, high_limit: Option) { - let physmem_limit = MemoryEntry { - start: align_up(low_limit.unwrap_or(0)), - end: align_down(high_limit.unwrap_or(usize::MAX)), - kind: BootloaderMemoryKind::Free, - }; + unsafe { + let physmem_limit = MemoryEntry { + start: align_up(low_limit.unwrap_or(0)), + end: align_down(high_limit.unwrap_or(usize::MAX)), + kind: BootloaderMemoryKind::Free, + }; - let areas = &mut *crate::memory::AREAS.get(); - let mut area_i = 0; + let areas = &mut *crate::memory::AREAS.get(); + let mut area_i = 0; - // Copy initial memory map, and page align it - for area in (*MEMORY_MAP.get()).free() { - log::debug!("{:X}:{:X}", area.start, area.end); + // Copy initial memory map, and page align it + for area in (*MEMORY_MAP.get()).free() { + log::debug!("{:X}:{:X}", area.start, area.end); - if let Some(area) = area.intersect(&physmem_limit) { - add_memory(areas, &mut area_i, area); + if let Some(area) = area.intersect(&physmem_limit) { + add_memory(areas, &mut area_i, area); + } } - } - areas[..area_i].sort_unstable_by_key(|area| area.base); - crate::memory::AREA_COUNT.get().write(area_i as u16); + areas[..area_i].sort_unstable_by_key(|area| area.base); + crate::memory::AREA_COUNT.get().write(area_i as u16); - // free memory map in now ready - let areas = crate::memory::areas(); + // free memory map in now ready + let areas = crate::memory::areas(); - // First, calculate how much memory we have - let mut size = 0; - for area in areas.iter() { - if area.size > 0 { - log::debug!("{:X?}", area); - size += area.size; + // First, calculate how much memory we have + let mut size = 0; + for area in areas.iter() { + if area.size > 0 { + log::debug!("{:X?}", area); + size += area.size; + } } + + log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE); + + // Create a basic allocator for the first pages + let mut bump_allocator = BumpAllocator::::new(areas, 0); + + map_memory(areas, &mut bump_allocator); + + // Create the physical memory map + let offset = bump_allocator.offset(); + log::info!( + "Permanently used: {} KB", + (offset + (KILOBYTE - 1)) / KILOBYTE + ); + + crate::memory::init_mm(bump_allocator); } - - log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE); - - // Create a basic allocator for the first pages - let mut bump_allocator = BumpAllocator::::new(areas, 0); - - map_memory(areas, &mut bump_allocator); - - // Create the physical memory map - let offset = bump_allocator.offset(); - log::info!( - "Permanently used: {} KB", - (offset + (KILOBYTE - 1)) / KILOBYTE - ); - - crate::memory::init_mm(bump_allocator); } diff --git a/src/sync/wait_queue.rs b/src/sync/wait_queue.rs index fa9d7962a7..39a4489c58 100644 --- a/src/sync/wait_queue.rs +++ b/src/sync/wait_queue.rs @@ -31,15 +31,20 @@ impl WaitQueue { loop { let mut inner = self.inner.lock(); - if let Some(t) = inner.pop_front() { - return Ok(t); - } else if block { - if !self.condition.wait(inner, reason) { - return Err(Error::new(EINTR)); + match inner.pop_front() { + Some(t) => { + return Ok(t); + } + _ => { + if block { + if !self.condition.wait(inner, reason) { + return Err(Error::new(EINTR)); + } + continue; + } else { + return Err(Error::new(EAGAIN)); + } } - continue; - } else { - return Err(Error::new(EAGAIN)); } } } diff --git a/src/syscall/debug.rs b/src/syscall/debug.rs index 348016a429..f1728068e4 100644 --- a/src/syscall/debug.rs +++ b/src/syscall/debug.rs @@ -38,7 +38,7 @@ fn debug_buf(ptr: usize, len: usize) -> Result> { }) } unsafe fn read_struct(ptr: usize) -> Result { - UserSlice::ro(ptr, mem::size_of::()).and_then(|slice| slice.read_exact::()) + unsafe { UserSlice::ro(ptr, mem::size_of::()).and_then(|slice| slice.read_exact::()) } } //TODO: calling format_call with arguments from another process space will not work diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 62460ce28b..195a5525a2 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -138,8 +138,10 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) { } pub unsafe fn bootstrap_mem(bootstrap: &crate::Bootstrap) -> &'static [u8] { - core::slice::from_raw_parts( - CurrentRmmArch::phys_to_virt(bootstrap.base.base()).data() as *const u8, - bootstrap.page_count * PAGE_SIZE, - ) + unsafe { + core::slice::from_raw_parts( + CurrentRmmArch::phys_to_virt(bootstrap.base.base()).data() as *const u8, + bootstrap.page_count * PAGE_SIZE, + ) + } } diff --git a/src/syscall/usercopy.rs b/src/syscall/usercopy.rs index 84d1190978..e49f0a6167 100644 --- a/src/syscall/usercopy.rs +++ b/src/syscall/usercopy.rs @@ -108,19 +108,21 @@ impl UserSlice { } } pub unsafe fn read_exact(self) -> Result { - let mut t: T = core::mem::zeroed(); - let slice = unsafe { - core::slice::from_raw_parts_mut( - (&mut t as *mut T).cast::(), - core::mem::size_of::(), - ) - }; + unsafe { + let mut t: T = core::mem::zeroed(); + let slice = unsafe { + core::slice::from_raw_parts_mut( + (&mut t as *mut T).cast::(), + core::mem::size_of::(), + ) + }; - self.limit(core::mem::size_of::()) - .ok_or(Error::new(EINVAL))? - .copy_to_slice(slice)?; + self.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_to_slice(slice)?; - Ok(t) + Ok(t) + } } pub fn copy_common_bytes_to_slice(self, slice: &mut [u8]) -> Result { let min = core::cmp::min(self.len(), slice.len());