diff --git a/linkers/x86_64.ld b/linkers/x86_64.ld index fc745492cd..4ba93e2922 100644 --- a/linkers/x86_64.ld +++ b/linkers/x86_64.ld @@ -20,6 +20,15 @@ SECTIONS { __text_end = .; __rodata_start = .; *(.rodata*) + __altcode_start = .; + *(.altcode) + __altcode_end = .; + __altrelocs_start = .; + *(.altrelocs) + __altrelocs_end = .; + __altfeatures_start = .; + *(.altfeatures) + __altfeatures_end = .; } .data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) { diff --git a/src/arch/x86_64/macros.rs b/src/arch/x86_64/macros.rs index 4e3566fc55..6f9c6d01c1 100644 --- a/src/arch/x86_64/macros.rs +++ b/src/arch/x86_64/macros.rs @@ -14,3 +14,93 @@ macro_rules! println { ($fmt:expr) => (print!(concat!($fmt, "\n"))); ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*)); } + + +#[repr(C)] +pub struct AltReloc { + pub code_start: usize, + pub code_end: usize, + pub altcode_start: usize, + pub altcode_end: usize, + pub name_start: usize, + pub name_end: usize, +} +macro_rules! expand_bool( + ($value:expr) => { + concat!($value) + } +); + +macro_rules! alternative( + (feature: $feature:literal, then: [$($then:expr),*], default: [$($default:expr),*]) => { + alternative2!(feature1: $feature, then1: [$($then),*], feature2: "", then2: [""], default: [$($default),*]) + } +); +macro_rules! saturating_sub( + ($lhs:literal, $rhs:literal) => { concat!( + "((", $lhs, ")>(", $rhs, "))*((", $lhs, ")-(", $rhs, "))", + ) } +); +// Use feature1 if present, otherwise try using feature2, otherwise use default. +// +// cpu_feature_always simply means it is always enabled. Thus, if feature2, which has lower +// priority, is "always" but feature1 is "auto", feature2 will still be checked for, and feature2 +// will become the fallback code. +// +// An empty string as feature is equivalent with "never". +macro_rules! alternative2( + (feature1: $feature1:literal, then1: [$($then1:expr),*], feature2: $feature2:literal, then2: [$($then2:expr),*], default: [$($default:expr),*]) => { + concat!(" + .set true, 1 + .set false, 0 + 40: + .if ", expand_bool!(cfg!(cpu_feature_always = $feature1)), " + ", $($then1,)* " + .elseif ", expand_bool!(cfg!(cpu_feature_always = $feature2)), " + ", $($then2,)* " + .else + ", $($default,)* " + .endif + 42: + .if ", expand_bool!(cfg!(cpu_feature_auto = $feature1)), " + .skip -", saturating_sub!("51f - 50f", "42b - 40b"), ", 0x90 + .endif + .if ", expand_bool!(cfg!(cpu_feature_auto = $feature2)), " + .skip -", saturating_sub!("61f - 60f", "42b - 40b"), ", 0x90 + .endif + 41: + ", + // FIXME: The assembler apparently complains "invalid number of bytes" despite it being + // quite obvious what saturating_sub does. + + // Declare them in reverse order. Last relocation wins! + alternative_auto!("6", $feature2, [$($then2),*]), + alternative_auto!("5", $feature1, [$($then1),*]), + ) + }; +); +macro_rules! alternative_auto( + ($first_digit:literal, $feature:literal, [$($then:expr),*]) => { concat!( + ".if ", expand_bool!(cfg!(cpu_feature_auto = $feature)), " + .pushsection .altcode.", $feature, " + ", $first_digit, "0: + ", $($then,)* " + ", $first_digit, "1: + .popsection + .pushsection .altfeatures.", $feature, " + 70: .ascii \"", $feature, "\" + 71: + .popsection + .pushsection .altrelocs.", $feature, " + .quad 70b + .quad 71b - 70b + .quad 40b + .quad 41b - 40b + .quad ", $first_digit, "0b + .quad ", $first_digit, "1b - ", $first_digit, "0b + .popsection + .endif + ", + ) } +); + diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index ddc149b236..4e1860efb0 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -59,28 +59,30 @@ pub mod flags { pub const FLAG_INTERRUPTS: usize = 1 << 9; } +// TODO: Maybe support rewriting relocations (using LD's --emit-relocs) when working with entire +// functions? #[naked] #[link_section = ".usercopy-fns"] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - // TODO: LFENCE (spectre_v1 mitigation)? + // TODO: spectre_v1 - #[cfg(cpu_feature_never = "smap")] - core::arch::asm!(" - xor eax, eax - mov rcx, rdx - rep movsb - ret - ", options(noreturn)); - - #[cfg(cpu_feature_always = "smap")] - core::arch::asm!(" - xor eax, eax - mov rcx, rdx - stac - rep movsb - clac - ret - ", options(noreturn)); + core::arch::asm!(alternative!( + feature: "smap", + then: [" + xor eax, eax + mov rcx, rdx + stac + rep movsb + clac + ret + "], + default: [" + xor eax, eax + mov rcx, rdx + rep movsb + ret + "] + ), options(noreturn)); } pub use arch_copy_to_user as arch_copy_from_user; diff --git a/src/lib.rs b/src/lib.rs index 684fbf0bd1..40995f8eaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -283,6 +283,9 @@ macro_rules! linker_offsets( ); pub mod kernel_executable_offsets { linker_offsets!(__text_start, __text_end, __rodata_start, __rodata_end, __data_start, __data_end, __bss_start, __bss_end, __usercopy_start, __usercopy_end); + + #[cfg(target_arch = "x86_64")] + linker_offsets!(__altrelocs_start, __altrelocs_end); } /// A unique number used internally by the kernel to identify CPUs.