vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support XSAVE, XSAVEOPT, and AVX2.

Browse Source
This commit is contained in:
4lDO2
2023-09-14 10:54:30 +02:00
parent 38e669c807
commit dde8f78903
8 changed files with 177 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.toml.example
+2
View File
@@ -1,5 +1,7 @@
[arch.x86_64.features]
smap = "auto"
fsgsbase = "auto"
xsave = "auto"
xsaveopt = "auto"
# vim: ft=toml
src/arch/x86_64/alternative.rs
+73 -2
View File
@@ -1,12 +1,16 @@
use core::mem::size_of;
use raw_cpuid::{ExtendedRegisterType, ExtendedRegisterStateLocation};
use spin::Once;
use x86::controlregs::Cr4;
use x86::controlregs::{Cr4, Xcr0};
use crate::context::memory::PageSpan;
use crate::cpuid::{has_ext_feat, cpuid_always};
use crate::cpuid::{has_ext_feat, cpuid_always, feature_info};
use crate::paging::{KernelMapper, Page, PageFlags, PAGE_SIZE, VirtualAddress};
#[cfg(all(cpu_feature_never = "xsave", not(cpu_feature_never = "xsaveopt")))]
compile_error!("cannot force-disable xsave without force-disabling xsaveopt");
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct AltReloc {
@@ -49,6 +53,38 @@ pub unsafe fn early_init(bsp: bool) {
enable |= KcpuFeatures::FSGSBASE;
}
if cfg!(not(cpu_feature_never = "xsave")) && feature_info().has_xsave() {
x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE);
let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE;
let ext_state_info = cpuid_always().get_extended_state_info().expect("must be present if XSAVE is supported");
enable |= KcpuFeatures::XSAVE;
enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt());
let info = xsave::XsaveInfo {
ymm_upper_offset: feature_info().has_avx().then(|| {
xcr0 |= Xcr0::XCR0_AVX_STATE;
x86::controlregs::xcr0_write(xcr0);
let state = ext_state_info.iter().find(|state| {
state.register() == ExtendedRegisterType::Avx
&& state.location() == ExtendedRegisterStateLocation::Xcr0
}).expect("CPUID said AVX was supported but there's no state info");
if state.size() as usize != 16 * core::mem::size_of::<u128>() {
log::warn!("Unusual AVX state size {}", state.size());
}
state.offset()
}),
xsave_size: ext_state_info.xsave_area_size_enabled_features(),
};
log::info!("INFO: {:?}", info);
xsave::XSAVE_INFO.call_once(|| info);
}
if !bsp {
return;
}
@@ -75,6 +111,8 @@ pub unsafe fn early_init(bsp: bool) {
let feature_is_enabled = match name {
"smap" => enable.contains(KcpuFeatures::SMAP),
"fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE),
"xsave" => enable.contains(KcpuFeatures::XSAVE),
"xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT),
//_ => panic!("unknown altcode relocation: {}", name),
_ => true,
};
@@ -121,6 +159,8 @@ bitflags! {
pub struct KcpuFeatures: usize {
const SMAP = 1;
const FSGSBASE = 2;
const XSAVE = 4;
const XSAVEOPT = 8;
}
}
@@ -129,3 +169,34 @@ static FEATURES: Once<KcpuFeatures> = Once::new();
pub fn features() -> KcpuFeatures {
*FEATURES.get().expect("early_cpu_init was not called")
}
#[cfg(not(cpu_feature_never = "xsave"))]
mod xsave {
use super::*;
#[derive(Debug)]
pub struct XsaveInfo {
pub ymm_upper_offset: Option<u32>,
pub xsave_size: u32,
}
pub(in super) static XSAVE_INFO: Once<XsaveInfo> = Once::new();
pub fn info() -> &'static XsaveInfo {
XSAVE_INFO.get().unwrap()
}
}
pub fn kfx_size() -> usize {
#[cfg(not(cpu_feature_never = "xsave"))]
{
FXSAVE_SIZE + XSAVE_HEADER_SIZE + xsave::info().xsave_size as usize
}
#[cfg(cpu_feature_never = "xsave")]
{
// FXSAVE size
FXSAVE_SIZE
}
}
pub const FXSAVE_SIZE: usize = 512;
pub const XSAVE_HEADER_SIZE: usize = 64;
src/arch/x86_64/cpuid.rs
+6 -1
View File
@@ -1,4 +1,4 @@
use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures};
use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures, FeatureInfo};
pub fn cpuid() -> Option<CpuId> {
// CPUID is always available on x86_64 systems.
@@ -15,6 +15,11 @@ pub fn cpuid_always() -> CpuId {
}
})
}
pub fn feature_info() -> FeatureInfo {
cpuid_always().get_feature_info().expect("x86_64 requires CPUID leaf=0x01 to be present")
}
pub fn has_ext_feat(feat: impl FnOnce(ExtendedFeatures) -> bool) -> bool {
cpuid_always()
.get_extended_feature_info()
src/arch/x86_64/mod.rs
+2
View File
@@ -92,3 +92,5 @@ pub use arch_copy_to_user as arch_copy_from_user;
pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] {
core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE)
}
pub use alternative::kfx_size;
src/common/aligned_box.rs
+56 -22
View File
@@ -1,4 +1,5 @@
use core::{alloc::GlobalAlloc, mem};
use core::alloc::Layout;
use core::alloc::GlobalAlloc;
use crate::common::unique::Unique;
use crate::memory::Enomem;
@@ -6,33 +7,39 @@ use crate::memory::Enomem;
// Necessary because GlobalAlloc::dealloc requires the layout to be the same, and therefore Box
// cannot be used for increased alignment directly.
// TODO: move to common?
pub struct AlignedBox<T, const ALIGN: usize> {
pub struct AlignedBox<T: ?Sized, const ALIGN: usize> {
inner: Unique<T>,
}
pub unsafe trait ValidForZero {}
unsafe impl<const N: usize> ValidForZero for [u8; N] {}
unsafe impl ValidForZero for u8 {}
unsafe impl ValidForZero for crate::syscall::data::Stat {}
unsafe impl ValidForZero for crate::syscall::data::StatVfs {}
impl<T, const ALIGN: usize> AlignedBox<T, ALIGN> {
const LAYOUT: core::alloc::Layout = {
const fn max(a: usize, b: usize) -> usize {
if a > b { a } else { b }
}
match core::alloc::Layout::from_size_align(mem::size_of::<T>(), max(mem::align_of::<T>(), ALIGN)) {
Ok(l) => l,
Err(_) => panic!("layout validation failed at compile time"),
}
impl<T: ?Sized, const ALIGN: usize> AlignedBox<T, ALIGN> {
fn layout(&self) -> Layout {
layout_upgrade_align(Layout::for_value::<T>(&*self), ALIGN)
}
}
const fn layout_upgrade_align(layout: Layout, align: usize) -> Layout {
const fn max(a: usize, b: usize) -> usize {
if a > b { a } else { b }
}
let Ok(x) = Layout::from_size_align(layout.size(), max(align, layout.align())) else {
panic!("failed to calculate layout");
};
x
}
impl<T, const ALIGN: usize> AlignedBox<T, ALIGN> {
#[inline(always)]
pub fn try_zeroed() -> Result<Self, Enomem>
where
T: ValidForZero,
{
Ok(unsafe {
let ptr = crate::ALLOCATOR.alloc_zeroed(Self::LAYOUT);
let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::new::<T>(), ALIGN));
if ptr.is_null() {
return Err(Enomem);
}
@@ -42,36 +49,63 @@ impl<T, const ALIGN: usize> AlignedBox<T, ALIGN> {
})
}
}
impl<T, const ALIGN: usize> core::fmt::Debug for AlignedBox<T, ALIGN> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(f, "[aligned box at {:p}, size {} alignment {}]", self.inner.as_ptr(), mem::size_of::<T>(), mem::align_of::<T>())
impl<T, const ALIGN: usize> AlignedBox<[T], ALIGN> {
#[inline]
pub fn try_zeroed_slice(len: usize) -> Result<Self, Enomem>
where
T: ValidForZero,
{
Ok(unsafe {
let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::array::<T>(len).unwrap(), ALIGN));
if ptr.is_null() {
return Err(Enomem);
}
Self {
inner: Unique::new_unchecked(core::ptr::slice_from_raw_parts_mut(ptr.cast(), len)),
}
})
}
}
impl<T, const ALIGN: usize> Drop for AlignedBox<T, ALIGN> {
impl<T: ?Sized, const ALIGN: usize> core::fmt::Debug for AlignedBox<T, ALIGN> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(f, "[aligned box at {:p}, size {} alignment {}]", self.inner.as_ptr(), self.layout().size(), self.layout().align())
}
}
impl<T: ?Sized, const ALIGN: usize> Drop for AlignedBox<T, ALIGN> {
fn drop(&mut self) {
unsafe {
let layout = self.layout();
core::ptr::drop_in_place(self.inner.as_ptr());
crate::ALLOCATOR.dealloc(self.inner.as_ptr().cast(), Self::LAYOUT);
crate::ALLOCATOR.dealloc(self.inner.as_ptr().cast(), layout);
}
}
}
impl<T, const ALIGN: usize> core::ops::Deref for AlignedBox<T, ALIGN> {
impl<T: ?Sized, const ALIGN: usize> core::ops::Deref for AlignedBox<T, ALIGN> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { &*self.inner.as_ptr() }
}
}
impl<T, const ALIGN: usize> core::ops::DerefMut for AlignedBox<T, ALIGN> {
impl<T: ?Sized, const ALIGN: usize> core::ops::DerefMut for AlignedBox<T, ALIGN> {
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { &mut *self.inner.as_ptr() }
}
}
impl<T: Clone + ValidForZero, const ALIGN: usize> Clone for AlignedBox<T, ALIGN> {
fn clone(&self) -> Self {
let mut new = Self::try_zeroed().unwrap_or_else(|_| alloc::alloc::handle_alloc_error(Self::LAYOUT));
let mut new = Self::try_zeroed().unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout()));
T::clone_from(&mut new, self);
new
}
}
impl<T: Clone + ValidForZero, const ALIGN: usize> Clone for AlignedBox<[T], ALIGN> {
fn clone(&self) -> Self {
let mut new = Self::try_zeroed_slice(self.len()).unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout()));
for i in 0..self.len() {
new[i].clone_from(&self[i]);
}
new
}
}
src/common/unique.rs
+7 -7
View File
@@ -4,18 +4,18 @@ use core::{fmt, ptr::NonNull};
/// only correct if the pointer is never accessed from multiple
/// locations across threads. Which is always, if the pointer is
/// unique.
pub struct Unique<T>(NonNull<T>);
pub struct Unique<T: ?Sized>(NonNull<T>);
impl<T> Copy for Unique<T> {}
impl<T> Clone for Unique<T> {
impl<T: ?Sized> Copy for Unique<T> {}
impl<T: ?Sized> Clone for Unique<T> {
fn clone(&self) -> Self {
*self
}
}
unsafe impl<T> Send for Unique<T> {}
unsafe impl<T> Sync for Unique<T> {}
unsafe impl<T: ?Sized> Send for Unique<T> {}
unsafe impl<T: ?Sized> Sync for Unique<T> {}
impl<T> Unique<T> {
impl<T: ?Sized> Unique<T> {
pub fn new(ptr: *mut T) -> Self {
Self(NonNull::new(ptr).expect("Did not expect pointer to be null"))
}
@@ -26,7 +26,7 @@ impl<T> Unique<T> {
self.0.as_ptr()
}
}
impl<T> fmt::Debug for Unique<T> {
impl<T: ?Sized> fmt::Debug for Unique<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self.0)
}
src/context/arch/x86_64.rs
+28 -5
View File
@@ -21,9 +21,12 @@ pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false);
const ST_RESERVED: u128 = 0xFFFF_FFFF_FFFF_0000_0000_0000_0000_0000;
pub const KFX_SIZE: usize = 512;
#[cfg(cpu_feature_never = "xsave")]
pub const KFX_ALIGN: usize = 16;
#[cfg(not(cpu_feature_never = "xsave"))]
pub const KFX_ALIGN: usize = 64;
#[derive(Clone, Debug)]
#[repr(C)]
pub struct Context {
@@ -136,11 +139,31 @@ pub unsafe fn empty_cr3() -> rmm::PhysicalAddress {
/// Switch to the next context by restoring its stack and registers
pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
core::arch::asm!("
fxsave64 [{prev_fx}]
fxrstor64 [{next_fx}]
", prev_fx = in(reg) prev.kfx.as_mut_ptr(),
core::arch::asm!(
alternative2!(
feature1: "xsaveopt",
then1: ["
mov eax, 0xffffffff
mov edx, eax
xsaveopt [{prev_fx}]
xrstor [{next_fx}]
"],
feature2: "xsave",
then2: ["
mov eax, 0xffffffff
mov edx, eax
xsave [{prev_fx}]
xrstor [{next_fx}]
"],
default: ["
fxsave64 [{prev_fx}]
fxrstor64 [{next_fx}]
"]
),
prev_fx = in(reg) prev.kfx.as_mut_ptr(),
next_fx = in(reg) next.kfx.as_ptr(),
out("eax") _,
out("edx") _,
);
{
src/context/context.rs
+3 -3
View File
@@ -257,11 +257,11 @@ pub struct Context {
/// The architecture specific context
pub arch: arch::Context,
/// Kernel FX - used to store SIMD and FPU registers on context switch
pub kfx: AlignedBox<[u8; arch::KFX_SIZE], {arch::KFX_ALIGN}>,
pub kfx: AlignedBox<[u8], {arch::KFX_ALIGN}>,
/// Kernel stack
pub kstack: Option<Box<[u8]>>,
/// Kernel signal backup: Registers, Kernel FX, Kernel Stack, Signal number
pub ksig: Option<(arch::Context, AlignedBox<[u8; arch::KFX_SIZE], {arch::KFX_ALIGN}>, Option<Box<[u8]>>, u8)>,
pub ksig: Option<(arch::Context, AlignedBox<[u8], {arch::KFX_ALIGN}>, Option<Box<[u8]>>, u8)>,
/// Restore ksig context on next switch
pub ksig_restore: bool,
/// Address space containing a page table lock, and grants. Normally this will have a value,
@@ -325,7 +325,7 @@ impl Context {
pending: VecDeque::new(),
wake: None,
arch: arch::Context::new(),
kfx: AlignedBox::<[u8; arch::KFX_SIZE], {arch::KFX_ALIGN}>::try_zeroed()?,
kfx: AlignedBox::<[u8], {arch::KFX_ALIGN}>::try_zeroed_slice(crate::arch::kfx_size())?,
kstack: None,
ksig: None,
ksig_restore: false,