diff --git a/config.toml.example b/config.toml.example index c7a71487fe..b3cdb9e7fb 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1,5 +1,7 @@ [arch.x86_64.features] smap = "auto" fsgsbase = "auto" +xsave = "auto" +xsaveopt = "auto" # vim: ft=toml diff --git a/src/arch/x86_64/alternative.rs b/src/arch/x86_64/alternative.rs index 9eb92875a0..6b9274d91e 100644 --- a/src/arch/x86_64/alternative.rs +++ b/src/arch/x86_64/alternative.rs @@ -1,12 +1,16 @@ use core::mem::size_of; +use raw_cpuid::{ExtendedRegisterType, ExtendedRegisterStateLocation}; use spin::Once; -use x86::controlregs::Cr4; +use x86::controlregs::{Cr4, Xcr0}; use crate::context::memory::PageSpan; -use crate::cpuid::{has_ext_feat, cpuid_always}; +use crate::cpuid::{has_ext_feat, cpuid_always, feature_info}; use crate::paging::{KernelMapper, Page, PageFlags, PAGE_SIZE, VirtualAddress}; +#[cfg(all(cpu_feature_never = "xsave", not(cpu_feature_never = "xsaveopt")))] +compile_error!("cannot force-disable xsave without force-disabling xsaveopt"); + #[repr(C)] #[derive(Clone, Copy, Debug)] pub struct AltReloc { @@ -49,6 +53,38 @@ pub unsafe fn early_init(bsp: bool) { enable |= KcpuFeatures::FSGSBASE; } + if cfg!(not(cpu_feature_never = "xsave")) && feature_info().has_xsave() { + x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE); + + let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE; + let ext_state_info = cpuid_always().get_extended_state_info().expect("must be present if XSAVE is supported"); + + enable |= KcpuFeatures::XSAVE; + enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt()); + + let info = xsave::XsaveInfo { + ymm_upper_offset: feature_info().has_avx().then(|| { + xcr0 |= Xcr0::XCR0_AVX_STATE; + x86::controlregs::xcr0_write(xcr0); + + let state = ext_state_info.iter().find(|state| { + state.register() == ExtendedRegisterType::Avx + && state.location() == ExtendedRegisterStateLocation::Xcr0 + }).expect("CPUID said AVX was supported but there's no state info"); + + if state.size() as usize != 16 * core::mem::size_of::() { + log::warn!("Unusual AVX state size {}", state.size()); + } + + state.offset() + }), + xsave_size: ext_state_info.xsave_area_size_enabled_features(), + }; + log::info!("INFO: {:?}", info); + + xsave::XSAVE_INFO.call_once(|| info); + } + if !bsp { return; } @@ -75,6 +111,8 @@ pub unsafe fn early_init(bsp: bool) { let feature_is_enabled = match name { "smap" => enable.contains(KcpuFeatures::SMAP), "fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE), + "xsave" => enable.contains(KcpuFeatures::XSAVE), + "xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT), //_ => panic!("unknown altcode relocation: {}", name), _ => true, }; @@ -121,6 +159,8 @@ bitflags! { pub struct KcpuFeatures: usize { const SMAP = 1; const FSGSBASE = 2; + const XSAVE = 4; + const XSAVEOPT = 8; } } @@ -129,3 +169,34 @@ static FEATURES: Once = Once::new(); pub fn features() -> KcpuFeatures { *FEATURES.get().expect("early_cpu_init was not called") } + +#[cfg(not(cpu_feature_never = "xsave"))] +mod xsave { + use super::*; + + #[derive(Debug)] + pub struct XsaveInfo { + pub ymm_upper_offset: Option, + pub xsave_size: u32, + } + pub(in super) static XSAVE_INFO: Once = Once::new(); + + pub fn info() -> &'static XsaveInfo { + XSAVE_INFO.get().unwrap() + } +} + +pub fn kfx_size() -> usize { + #[cfg(not(cpu_feature_never = "xsave"))] + { + FXSAVE_SIZE + XSAVE_HEADER_SIZE + xsave::info().xsave_size as usize + } + #[cfg(cpu_feature_never = "xsave")] + { + // FXSAVE size + FXSAVE_SIZE + } +} + +pub const FXSAVE_SIZE: usize = 512; +pub const XSAVE_HEADER_SIZE: usize = 64; diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 5b2782da4f..23ccb88f37 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -1,4 +1,4 @@ -use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures}; +use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures, FeatureInfo}; pub fn cpuid() -> Option { // CPUID is always available on x86_64 systems. @@ -15,6 +15,11 @@ pub fn cpuid_always() -> CpuId { } }) } + +pub fn feature_info() -> FeatureInfo { + cpuid_always().get_feature_info().expect("x86_64 requires CPUID leaf=0x01 to be present") +} + pub fn has_ext_feat(feat: impl FnOnce(ExtendedFeatures) -> bool) -> bool { cpuid_always() .get_extended_feature_info() diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index e143f1a08b..a900337bb0 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -92,3 +92,5 @@ pub use arch_copy_to_user as arch_copy_from_user; pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] { core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) } + +pub use alternative::kfx_size; diff --git a/src/common/aligned_box.rs b/src/common/aligned_box.rs index 3c1cdd68ff..10a5fd091d 100644 --- a/src/common/aligned_box.rs +++ b/src/common/aligned_box.rs @@ -1,4 +1,5 @@ -use core::{alloc::GlobalAlloc, mem}; +use core::alloc::Layout; +use core::alloc::GlobalAlloc; use crate::common::unique::Unique; use crate::memory::Enomem; @@ -6,33 +7,39 @@ use crate::memory::Enomem; // Necessary because GlobalAlloc::dealloc requires the layout to be the same, and therefore Box // cannot be used for increased alignment directly. // TODO: move to common? -pub struct AlignedBox { +pub struct AlignedBox { inner: Unique, } pub unsafe trait ValidForZero {} unsafe impl ValidForZero for [u8; N] {} +unsafe impl ValidForZero for u8 {} unsafe impl ValidForZero for crate::syscall::data::Stat {} unsafe impl ValidForZero for crate::syscall::data::StatVfs {} -impl AlignedBox { - const LAYOUT: core::alloc::Layout = { - const fn max(a: usize, b: usize) -> usize { - if a > b { a } else { b } - } - - match core::alloc::Layout::from_size_align(mem::size_of::(), max(mem::align_of::(), ALIGN)) { - Ok(l) => l, - Err(_) => panic!("layout validation failed at compile time"), - } +impl AlignedBox { + fn layout(&self) -> Layout { + layout_upgrade_align(Layout::for_value::(&*self), ALIGN) + } +} +const fn layout_upgrade_align(layout: Layout, align: usize) -> Layout { + const fn max(a: usize, b: usize) -> usize { + if a > b { a } else { b } + } + let Ok(x) = Layout::from_size_align(layout.size(), max(align, layout.align())) else { + panic!("failed to calculate layout"); }; + x +} + +impl AlignedBox { #[inline(always)] pub fn try_zeroed() -> Result where T: ValidForZero, { Ok(unsafe { - let ptr = crate::ALLOCATOR.alloc_zeroed(Self::LAYOUT); + let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::new::(), ALIGN)); if ptr.is_null() { return Err(Enomem); } @@ -42,36 +49,63 @@ impl AlignedBox { }) } } - -impl core::fmt::Debug for AlignedBox { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "[aligned box at {:p}, size {} alignment {}]", self.inner.as_ptr(), mem::size_of::(), mem::align_of::()) +impl AlignedBox<[T], ALIGN> { + #[inline] + pub fn try_zeroed_slice(len: usize) -> Result + where + T: ValidForZero, + { + Ok(unsafe { + let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::array::(len).unwrap(), ALIGN)); + if ptr.is_null() { + return Err(Enomem); + } + Self { + inner: Unique::new_unchecked(core::ptr::slice_from_raw_parts_mut(ptr.cast(), len)), + } + }) } } -impl Drop for AlignedBox { + +impl core::fmt::Debug for AlignedBox { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[aligned box at {:p}, size {} alignment {}]", self.inner.as_ptr(), self.layout().size(), self.layout().align()) + } +} +impl Drop for AlignedBox { fn drop(&mut self) { unsafe { + let layout = self.layout(); core::ptr::drop_in_place(self.inner.as_ptr()); - crate::ALLOCATOR.dealloc(self.inner.as_ptr().cast(), Self::LAYOUT); + crate::ALLOCATOR.dealloc(self.inner.as_ptr().cast(), layout); } } } -impl core::ops::Deref for AlignedBox { +impl core::ops::Deref for AlignedBox { type Target = T; fn deref(&self) -> &Self::Target { unsafe { &*self.inner.as_ptr() } } } -impl core::ops::DerefMut for AlignedBox { +impl core::ops::DerefMut for AlignedBox { fn deref_mut(&mut self) -> &mut Self::Target { unsafe { &mut *self.inner.as_ptr() } } } impl Clone for AlignedBox { fn clone(&self) -> Self { - let mut new = Self::try_zeroed().unwrap_or_else(|_| alloc::alloc::handle_alloc_error(Self::LAYOUT)); + let mut new = Self::try_zeroed().unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout())); T::clone_from(&mut new, self); new } } +impl Clone for AlignedBox<[T], ALIGN> { + fn clone(&self) -> Self { + let mut new = Self::try_zeroed_slice(self.len()).unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout())); + for i in 0..self.len() { + new[i].clone_from(&self[i]); + } + new + } +} diff --git a/src/common/unique.rs b/src/common/unique.rs index 5d22f93a12..120f1c98eb 100644 --- a/src/common/unique.rs +++ b/src/common/unique.rs @@ -4,18 +4,18 @@ use core::{fmt, ptr::NonNull}; /// only correct if the pointer is never accessed from multiple /// locations across threads. Which is always, if the pointer is /// unique. -pub struct Unique(NonNull); +pub struct Unique(NonNull); -impl Copy for Unique {} -impl Clone for Unique { +impl Copy for Unique {} +impl Clone for Unique { fn clone(&self) -> Self { *self } } -unsafe impl Send for Unique {} -unsafe impl Sync for Unique {} +unsafe impl Send for Unique {} +unsafe impl Sync for Unique {} -impl Unique { +impl Unique { pub fn new(ptr: *mut T) -> Self { Self(NonNull::new(ptr).expect("Did not expect pointer to be null")) } @@ -26,7 +26,7 @@ impl Unique { self.0.as_ptr() } } -impl fmt::Debug for Unique { +impl fmt::Debug for Unique { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{:?}", self.0) } diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 8e87502eb6..ca485c4c69 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -21,9 +21,12 @@ pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false); const ST_RESERVED: u128 = 0xFFFF_FFFF_FFFF_0000_0000_0000_0000_0000; -pub const KFX_SIZE: usize = 512; +#[cfg(cpu_feature_never = "xsave")] pub const KFX_ALIGN: usize = 16; +#[cfg(not(cpu_feature_never = "xsave"))] +pub const KFX_ALIGN: usize = 64; + #[derive(Clone, Debug)] #[repr(C)] pub struct Context { @@ -136,11 +139,31 @@ pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { /// Switch to the next context by restoring its stack and registers pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { - core::arch::asm!(" - fxsave64 [{prev_fx}] - fxrstor64 [{next_fx}] - ", prev_fx = in(reg) prev.kfx.as_mut_ptr(), + core::arch::asm!( + alternative2!( + feature1: "xsaveopt", + then1: [" + mov eax, 0xffffffff + mov edx, eax + xsaveopt [{prev_fx}] + xrstor [{next_fx}] + "], + feature2: "xsave", + then2: [" + mov eax, 0xffffffff + mov edx, eax + xsave [{prev_fx}] + xrstor [{next_fx}] + "], + default: [" + fxsave64 [{prev_fx}] + fxrstor64 [{next_fx}] + "] + ), + prev_fx = in(reg) prev.kfx.as_mut_ptr(), next_fx = in(reg) next.kfx.as_ptr(), + out("eax") _, + out("edx") _, ); { diff --git a/src/context/context.rs b/src/context/context.rs index e939080988..43cb667e1c 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -257,11 +257,11 @@ pub struct Context { /// The architecture specific context pub arch: arch::Context, /// Kernel FX - used to store SIMD and FPU registers on context switch - pub kfx: AlignedBox<[u8; arch::KFX_SIZE], {arch::KFX_ALIGN}>, + pub kfx: AlignedBox<[u8], {arch::KFX_ALIGN}>, /// Kernel stack pub kstack: Option>, /// Kernel signal backup: Registers, Kernel FX, Kernel Stack, Signal number - pub ksig: Option<(arch::Context, AlignedBox<[u8; arch::KFX_SIZE], {arch::KFX_ALIGN}>, Option>, u8)>, + pub ksig: Option<(arch::Context, AlignedBox<[u8], {arch::KFX_ALIGN}>, Option>, u8)>, /// Restore ksig context on next switch pub ksig_restore: bool, /// Address space containing a page table lock, and grants. Normally this will have a value, @@ -325,7 +325,7 @@ impl Context { pending: VecDeque::new(), wake: None, arch: arch::Context::new(), - kfx: AlignedBox::<[u8; arch::KFX_SIZE], {arch::KFX_ALIGN}>::try_zeroed()?, + kfx: AlignedBox::<[u8], {arch::KFX_ALIGN}>::try_zeroed_slice(crate::arch::kfx_size())?, kstack: None, ksig: None, ksig_restore: false,