Allow code-overwriting optimizations, use for smap.

src/arch/x86_64/alternative.rs

@@ -0,0 +1,105 @@
+use core::mem::size_of;
+
+use x86::controlregs::Cr4;
+
+use crate::context::memory::PageSpan;
+use crate::cpuid::has_ext_feat;
+use crate::paging::{KernelMapper, Page, PageFlags, PAGE_SIZE, VirtualAddress};
+
+#[repr(C)]
+#[derive(Clone, Copy, Debug)]
+pub struct AltReloc {
+    pub name_start: *const u8,
+    pub name_len: usize,
+    pub code_start: *mut u8,
+    pub origcode_len: usize,
+    pub altcode_start: *const u8,
+    pub altcode_len: usize,
+}
+
+#[cold]
+pub unsafe fn early_init(bsp: bool) {
+    let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start();
+    let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset;
+
+    assert_eq!(relocs_size % size_of::<AltReloc>(), 0);
+    let relocs = core::slice::from_raw_parts(relocs_offset as *const AltReloc, relocs_size / size_of::<AltReloc>());
+
+    let mut enable_smap = false;
+
+    if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) {
+        // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any
+        // userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This
+        // limits user-memory accesses to the UserSlice wrapper, so that no data outside of
+        // usercopy functions can be accidentally accessed by the kernel.
+        x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP);
+        // Clear CLAC in (the probably unlikely) case the bootloader set it earlier.
+        x86::bits64::rflags::clac();
+
+        enable_smap = true;
+    }
+    if !bsp {
+        return;
+    }
+
+    let mut mapper = KernelMapper::lock();
+    for reloc in relocs.iter().copied() {
+        let name = core::str::from_utf8(core::slice::from_raw_parts(reloc.name_start, reloc.name_len)).expect("invalid feature name");
+        let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len);
+
+        let total_length = core::cmp::max(reloc.altcode_len, reloc.origcode_len);
+
+        let dst_pages = PageSpan::between(
+            Page::containing_address(VirtualAddress::new(reloc.code_start as usize)),
+            Page::containing_address(VirtualAddress::new((reloc.code_start as usize + reloc.origcode_len).next_multiple_of(PAGE_SIZE))),
+        );
+        for page in dst_pages.pages() {
+            mapper.remap(page.start_address(), PageFlags::new().write(true).execute(true).global(true)).unwrap().flush();
+        }
+
+        let code = core::slice::from_raw_parts_mut(reloc.code_start, total_length);
+
+        log::info!("feature {} current {:x?} altcode {:x?}", name, code, altcode);
+
+        let feature_is_enabled = match name {
+            "smap" => enable_smap,
+            //_ => panic!("unknown altcode relocation: {}", name),
+            _ => true,
+        };
+
+        if !feature_is_enabled {
+            continue;
+        }
+
+        let (dst, dst_nops) = code.split_at_mut(reloc.altcode_len);
+        dst.copy_from_slice(altcode);
+
+        // XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X`
+        // only outputs the (slower) single-byte 0x90 NOP.
+
+        // This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November
+        // 2020).
+        const NOPS_TABLE: [&[u8]; 11] = [
+            &[0x90],
+            &[0x66, 0x90],
+            &[0x0f, 0x1f, 0x00],
+            &[0x0f, 0x1f, 0x40, 0x00],
+            &[0x0f, 0x1f, 0x44, 0x00, 0x00],
+            &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00],
+            &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00],
+            &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+            &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+            &[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+            &[0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+        ];
+
+        for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) {
+            chunk.copy_from_slice(NOPS_TABLE[chunk.len()]);
+        }
+        log::info!("feature {} new {:x?} altcode {:x?}", name, code, altcode);
+
+        for page in dst_pages.pages() {
+            mapper.remap(page.start_address(), PageFlags::new().write(false).execute(true).global(true)).unwrap().flush();
+        }
+    }
+}

src/arch/x86_64/cpuid.rs

@@ -1,4 +1,4 @@
-use raw_cpuid::{CpuId, CpuIdResult};
+use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures};
 
 pub fn cpuid() -> Option<CpuId> {
     // CPUID is always available on x86_64 systems.
@@ -15,3 +15,8 @@ pub fn cpuid_always() -> CpuId {
         }
     })
 }
+pub fn has_ext_feat(feat: impl FnOnce(ExtendedFeatures) -> bool) -> bool {
+    cpuid_always()
+        .get_extended_feature_info()
+        .map_or(false, feat)
+}

src/arch/x86_64/macros.rs

@@ -15,16 +15,6 @@ macro_rules! println {
     ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
 }
 
-
-#[repr(C)]
-pub struct AltReloc {
-    pub code_start: usize,
-    pub code_end: usize,
-    pub altcode_start: usize,
-    pub altcode_end: usize,
-    pub name_start: usize,
-    pub name_end: usize,
-}
 macro_rules! expand_bool(
     ($value:expr) => {
         concat!($value)
@@ -95,7 +85,7 @@ macro_rules! alternative_auto(
         .quad 70b
         .quad 71b - 70b
         .quad 40b
-        .quad 41b - 40b
+        .quad 42b - 40b
         .quad ", $first_digit, "0b
         .quad ", $first_digit, "1b - ", $first_digit, "0b
         .popsection

src/arch/x86_64/misc.rs

@@ -1,16 +1,9 @@
 use x86::controlregs::Cr4;
-use x86::cpuid::ExtendedFeatures;
 
 use crate::LogicalCpuId;
-use crate::cpuid::cpuid_always;
+use crate::cpuid::{cpuid_always, has_ext_feat};
 
 pub unsafe fn init(cpu_id: LogicalCpuId) {
-    let has_ext_feat = |feat: fn(ExtendedFeatures) -> bool| {
-        cpuid_always()
-            .get_extended_feature_info()
-            .map_or(false, feat)
-    };
-
     if has_ext_feat(|feat| feat.has_umip()) {
         // UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT,
         // SMSW and STR. KASLR is currently not implemented, but this protects against leaking
@@ -23,15 +16,6 @@ pub unsafe fn init(cpu_id: LogicalCpuId) {
         // obvious reasons.
         x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP);
     }
-    if has_ext_feat(|feat| feat.has_smap()) && cfg!(cpu_feature_always = "smap") {
-        // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any
-        // userspace-accessible pages, with the necessary exception of RFLAGS.AC = 1. This limits
-        // user-memory accesses to the UserSlice wrapper, so that no code outside of usercopy
-        // functions can be accidentally accessed by the kernel.
-        x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP);
-        // Clear CLAC in (the probably unlikely) case the bootloader set it earlier.
-        x86::bits64::rflags::clac();
-    }
 
     if let Some(feats) = cpuid_always().get_extended_processor_and_feature_identifiers() && feats.has_rdtscp() {
         x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into());

src/arch/x86_64/mod.rs

@@ -2,6 +2,8 @@ use crate::Bootstrap;
 
 use self::paging::PAGE_SIZE;
 
+pub mod alternative;
+
 #[macro_use]
 pub mod macros;
 

src/arch/x86_64/start.rs

@@ -137,6 +137,8 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
         // Set up IDT
         idt::init_paging_bsp();
 
+        crate::alternative::early_init(true);
+
         // Set up syscall instruction
         interrupt::syscall::init();
 
@@ -235,6 +237,8 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
         // Set up IDT for AP
         idt::init_paging_post_heap(false, cpu_id);
 
+        crate::alternative::early_init(false);
+
         // Set up syscall instruction
         interrupt::syscall::init();