diff --git a/linkers/i686.ld b/linkers/i686.ld index 4579b5390f..386ceff512 100644 --- a/linkers/i686.ld +++ b/linkers/i686.ld @@ -7,45 +7,42 @@ SECTIONS { . = KERNEL_OFFSET; . += SIZEOF_HEADERS; - . = ALIGN(4096); - .text : AT(ADDR(.text) - KERNEL_OFFSET) { + .text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) { __text_start = .; *(.text*) __usercopy_start = .; *(.usercopy-fns) __usercopy_end = .; - . = ALIGN(4096); - __text_end = .; } - .rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) { + .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { + __text_end = .; __rodata_start = .; *(.rodata*) - . = ALIGN(4096); - __rodata_end = .; } - .data : AT(ADDR(.data) - KERNEL_OFFSET) { + .data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) { + __rodata_end = .; __data_start = .; *(.data*) - . = ALIGN(4096); + . = ALIGN(4K); __data_end = .; __bss_start = .; *(.bss*) - . = ALIGN(4096); - __bss_end = .; + . = ALIGN(4K); } - .tdata : AT(ADDR(.tdata) - KERNEL_OFFSET) { + .tdata ALIGN(4K) : AT(ADDR(.tdata) - KERNEL_OFFSET) { + __bss_end = .; __tdata_start = .; *(.tdata*) - . = ALIGN(4096); + . = ALIGN(4K); __tdata_end = .; __tbss_start = .; *(.tbss*) . += 8; - . = ALIGN(4096); + . = ALIGN(4K); __tbss_end = .; } diff --git a/linkers/x86_64.ld b/linkers/x86_64.ld index 35aadf19a9..fc8a626d60 100644 --- a/linkers/x86_64.ld +++ b/linkers/x86_64.ld @@ -7,45 +7,39 @@ SECTIONS { . = KERNEL_OFFSET; . += SIZEOF_HEADERS; - . = ALIGN(4096); - .text : AT(ADDR(.text) - KERNEL_OFFSET) { + .text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) { __text_start = .; *(.text*) __usercopy_start = .; *(.usercopy-fns) __usercopy_end = .; - . = ALIGN(4096); - __text_end = .; } - .rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) { + .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { + __text_end = .; __rodata_start = .; *(.rodata*) - . = ALIGN(4096); - __rodata_end = .; } - .data : AT(ADDR(.data) - KERNEL_OFFSET) { + .data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) { + __rodata_end = .; __data_start = .; *(.data*) - . = ALIGN(4096); + . = ALIGN(4K); __data_end = .; __bss_start = .; *(.bss*) - . = ALIGN(4096); - __bss_end = .; } - .tdata : AT(ADDR(.tdata) - KERNEL_OFFSET) { + .tdata ALIGN(4K) : AT(ADDR(.tdata) - KERNEL_OFFSET) { + __bss_end = .; __tdata_start = .; *(.tdata*) - . = ALIGN(4096); __tdata_end = .; __tbss_start = .; *(.tbss*) - . += 8; - . = ALIGN(4096); + . = ALIGN(4K); __tbss_end = .; } diff --git a/src/arch/x86/gdt.rs b/src/arch/x86/gdt.rs index 14c6cdb06e..d6c5cc0e1e 100644 --- a/src/arch/x86/gdt.rs +++ b/src/arch/x86/gdt.rs @@ -2,26 +2,27 @@ use core::convert::TryInto; use core::mem; +use core::ptr::addr_of_mut; -use x86::segmentation::load_cs; use x86::bits32::task::TaskStateSegment; use x86::Ring; use x86::dtables::{self, DescriptorTablePointer}; use x86::segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector}; use x86::task; +use crate::paging::{RmmA, RmmArch, PAGE_SIZE}; + use super::cpuid::cpuid; pub const GDT_NULL: usize = 0; pub const GDT_KERNEL_CODE: usize = 1; pub const GDT_KERNEL_DATA: usize = 2; -pub const GDT_KERNEL_KPCR: usize = 3; +pub const GDT_KERNEL_PERCPU: usize = 3; pub const GDT_USER_CODE: usize = 4; pub const GDT_USER_DATA: usize = 5; pub const GDT_USER_FS: usize = 6; pub const GDT_USER_GS: usize = 7; pub const GDT_TSS: usize = 8; -pub const GDT_CPU_ID_CONTAINER: usize = 9; pub const GDT_A_PRESENT: u8 = 1 << 7; pub const GDT_A_RING_0: u8 = 0 << 5; @@ -41,19 +42,16 @@ pub const GDT_F_PAGE_SIZE: u8 = 1 << 7; pub const GDT_F_PROTECTED_MODE: u8 = 1 << 6; pub const GDT_F_LONG_MODE: u8 = 1 << 5; -static mut INIT_GDT: [GdtEntry; 4] = [ +static INIT_GDT: [GdtEntry; 3] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), // Kernel data GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), - // Kernel TLS - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), ]; -#[thread_local] -pub static mut GDT: [GdtEntry; 10] = [ +const BASE_GDT: [GdtEntry; 9] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code @@ -72,16 +70,15 @@ pub static mut GDT: [GdtEntry; 10] = [ GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), // TSS GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0), - // Unused entry which stores the CPU ID. This is necessary for paranoid interrupts as they have - // no other way of determining it. - GdtEntry::new(0, 0, 0, 0), ]; -#[repr(C, align(16))] +#[repr(C, align(4096))] pub struct ProcessorControlRegion { pub tcb_end: usize, pub user_rsp_tmp: usize, pub tss: TssWrapper, + pub self_ref: usize, + pub gdt: [GdtEntry; 9], } // NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding @@ -89,114 +86,108 @@ pub struct ProcessorControlRegion { #[repr(C, align(16))] pub struct TssWrapper(pub TaskStateSegment); -#[thread_local] -pub static mut KPCR: ProcessorControlRegion = ProcessorControlRegion { - tcb_end: 0, - user_rsp_tmp: 0, - tss: TssWrapper(TaskStateSegment::new()), -}; +pub unsafe fn pcr() -> *mut ProcessorControlRegion { + let mut ret: *mut ProcessorControlRegion; + core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(memoffset::offset_of!(ProcessorControlRegion, self_ref))); + ret +} #[cfg(feature = "pti")] pub unsafe fn set_tss_stack(stack: usize) { use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK}; - KPCR.tss.0.ss0 = (GDT_KERNEL_DATA << 3) as u16; - KPCR.tss.0.esp0 = (PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32; + addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16); + addr_of_mut!((*pcr()).tss.0.esp0).write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32); PTI_CONTEXT_STACK = stack; } #[cfg(not(feature = "pti"))] pub unsafe fn set_tss_stack(stack: usize) { - KPCR.tss.0.ss0 = (GDT_KERNEL_DATA << 3) as u16; - KPCR.tss.0.esp0 = stack as u32; + addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16); + addr_of_mut!((*pcr()).tss.0.esp0).write(stack as u32); } -// Initialize GDT +/// Initialize a minimal GDT without configuring percpu. pub unsafe fn init() { - { - // Setup the initial GDT with TLS, so we can setup the TLS GDT (a little confusing) - // This means that each CPU will have its own GDT, but we only need to define it once as a thread local - - let limit = (INIT_GDT.len() * mem::size_of::() - 1) - .try_into() - .expect("initial GDT way too large"); - let base = INIT_GDT.as_ptr() as *const SegmentDescriptor; - - let init_gdtr: DescriptorTablePointer = DescriptorTablePointer { - limit, - base, - }; - - // Load the initial GDT, before we have access to thread locals - dtables::lgdt(&init_gdtr); - } + // Load the initial GDT, before the kernel remaps itself. + dtables::lgdt(&DescriptorTablePointer { + limit: (INIT_GDT.len() * mem::size_of::() - 1) as u16, + base: INIT_GDT.as_ptr() as *const SegmentDescriptor, + }); // Load the segment descriptors - load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); + segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0)); + segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); } -/// Initialize GDT with TLS -pub unsafe fn init_paging(cpu_id: u32, tcb_offset: usize, stack_offset: usize) { - //TODO: will this work with multicore? - { - INIT_GDT[GDT_KERNEL_KPCR].set_offset(tcb_offset as u32); - segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0)); - } +/// Initialize GDT and configure percpu. +pub unsafe fn init_paging(stack_offset: usize) { + let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR frame"); + let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); - // Now that we have access to thread locals, begin by getting a pointer to the Processor - // Control Region. - let kpcr = &mut KPCR; - - // Then, setup the AP's individual GDT - let limit = (GDT.len() * mem::size_of::() - 1) - .try_into() - .expect("main GDT way too large"); - let base = GDT.as_ptr() as *const SegmentDescriptor; + pcr.self_ref = pcr as *const _ as usize; + pcr.gdt = BASE_GDT; + pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32); let gdtr: DescriptorTablePointer = DescriptorTablePointer { - limit, - base, + limit: (pcr.gdt.len() * mem::size_of::() - 1) as u16, + base: pcr.gdt.as_ptr() as *const SegmentDescriptor, }; - // Once we have fetched the real KPCR address, set the TLS segment to the TCB pointer there. - kpcr.tcb_end = (tcb_offset as *const usize).read(); - - GDT[GDT_KERNEL_KPCR].set_offset(tcb_offset as u32); + pcr.tcb_end = init_percpu(); { - // We can now access our TSS, via the KPCR, which is a thread local - let tss = &kpcr.tss.0 as *const _ as usize as u32; + let tss = &pcr.tss.0 as *const _ as usize as u32; - GDT[GDT_TSS].set_offset(tss); - GDT[GDT_TSS].set_limit(mem::size_of::() as u32); + pcr.gdt[GDT_TSS].set_offset(tss); + pcr.gdt[GDT_TSS].set_limit(mem::size_of::() as u32); } - // And finally, populate the last GDT entry with the current CPU ID, to allow paranoid - // interrupt handlers to safely use TLS. - (&mut GDT[GDT_CPU_ID_CONTAINER] as *mut GdtEntry).cast::().write(cpu_id); - - // Set the stack pointer to use when coming back from userspace. - set_tss_stack(stack_offset); - // Load the new GDT, which is correctly located in thread local storage. dtables::lgdt(&gdtr); // Reload the segment descriptors - load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); + segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0)); segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + // TODO: Use FS for kernel TLS on i686? + segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0)); + + // Set the stack pointer to use when coming back from userspace. + set_tss_stack(stack_offset); + // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); } +// TODO: Share code with x86. Maybe even with aarch64? +/// Copy tdata, clear tbss, calculate TCB end pointer +#[cold] +unsafe fn init_percpu() -> usize { + use crate::kernel_executable_offsets::*; + + let size = __tbss_end() - __tdata_start(); + assert_eq!(size % PAGE_SIZE, 0); + + let tbss_offset = __tbss_start() - __tdata_start(); + + let base_frame = crate::memory::allocate_frames(size / PAGE_SIZE).expect("failed to allocate percpu memory"); + let base = RmmA::phys_to_virt(base_frame.start_address()); + + let tls_end = base.data() + size; + + core::ptr::copy_nonoverlapping(__tdata_start() as *const u8, base.data() as *mut u8, tbss_offset); + core::ptr::write_bytes((base.data() + tbss_offset) as *mut u8, 0, size - tbss_offset); + + tls_end +} + #[derive(Copy, Clone, Debug)] #[repr(packed)] pub struct GdtEntry { diff --git a/src/arch/x86/paging/mod.rs b/src/arch/x86/paging/mod.rs index 336469bfa1..ea3562b6de 100644 --- a/src/arch/x86/paging/mod.rs +++ b/src/arch/x86/paging/mod.rs @@ -29,6 +29,7 @@ pub const ENTRY_COUNT: usize = RmmA::PAGE_ENTRIES; pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE; /// Setup page attribute table +#[cold] unsafe fn init_pat() { let uncacheable = 0; let write_combining = 1; @@ -60,124 +61,9 @@ unsafe fn init_pat() { ); } -/// Map percpu -unsafe fn map_percpu(cpu_id: usize, mapper: &mut PageMapper) -> PageFlushAll { - extern "C" { - /// The starting byte of the thread data segment - static mut __tdata_start: u8; - /// The ending byte of the thread data segment - static mut __tdata_end: u8; - /// The starting byte of the thread BSS segment - static mut __tbss_start: u8; - /// The ending byte of the thread BSS segment - static mut __tbss_end: u8; - } - - let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize; - let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id; - let end = start + size; - - let mut flush_all = PageFlushAll::new(); - let start_page = Page::containing_address(VirtualAddress::new(start)); - let end_page = Page::containing_address(VirtualAddress::new(end - 1)); - for page in Page::range_inclusive(start_page, end_page) { - let result = mapper.map( - page.start_address(), - PageFlags::new().write(true).global(cfg!(not(feature = "pti"))), - ) - .expect("failed to allocate page table frames while mapping percpu"); - flush_all.consume(result); - } - flush_all -} - -/// Copy tdata, clear tbss, set TCB self pointer -unsafe fn init_tcb(cpu_id: usize) -> usize { - extern "C" { - /// The starting byte of the thread data segment - static mut __tdata_start: u8; - /// The ending byte of the thread data segment - static mut __tdata_end: u8; - /// The starting byte of the thread BSS segment - static mut __tbss_start: u8; - /// The ending byte of the thread BSS segment - static mut __tbss_end: u8; - } - - let tcb_offset; - { - let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize; - let tbss_offset = &__tbss_start as *const _ as usize - &__tdata_start as *const _ as usize; - - let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id; - let end = start + size; - tcb_offset = end - mem::size_of::(); - - ptr::copy(&__tdata_start as *const u8, start as *mut u8, tbss_offset); - ptr::write_bytes((start + tbss_offset) as *mut u8, 0, size - tbss_offset); - - *(tcb_offset as *mut usize) = end; - } - tcb_offset -} - -/// Initialize paging -/// -/// Returns page table and thread control block offset -pub unsafe fn init( - cpu_id: usize, -) -> usize { - extern "C" { - /// The starting byte of the text (code) data segment. - static mut __text_start: u8; - /// The ending byte of the text (code) data segment. - static mut __text_end: u8; - /// The starting byte of the _.rodata_ (read-only data) segment. - static mut __rodata_start: u8; - /// The ending byte of the _.rodata_ (read-only data) segment. - static mut __rodata_end: u8; - /// The starting byte of the _.data_ segment. - static mut __data_start: u8; - /// The ending byte of the _.data_ segment. - static mut __data_end: u8; - /// The starting byte of the thread data segment - static mut __tdata_start: u8; - /// The ending byte of the thread data segment - static mut __tdata_end: u8; - /// The starting byte of the thread BSS segment - static mut __tbss_start: u8; - /// The ending byte of the thread BSS segment - static mut __tbss_end: u8; - /// The starting byte of the _.bss_ (uninitialized data) segment. - static mut __bss_start: u8; - /// The ending byte of the _.bss_ (uninitialized data) segment. - static mut __bss_end: u8; - } - +#[cold] +pub unsafe fn init() { init_pat(); - - let flush_all = map_percpu(cpu_id, KernelMapper::lock_manually(cpu_id).get_mut().expect("expected KernelMapper not to be locked re-entrant in paging::init")); - flush_all.flush(); - - init_tcb(cpu_id) -} - -pub unsafe fn init_ap( - cpu_id: usize, - bsp_table: &mut KernelMapper, -) -> usize { - init_pat(); - - { - let flush_all = map_percpu(cpu_id, bsp_table.get_mut().expect("KernelMapper locked re-entrant for AP")); - - // The flush can be ignored as this is not the active table. See later make_current(). - flush_all.ignore(); - }; - - bsp_table.make_current(); - - init_tcb(cpu_id) } /// Page diff --git a/src/arch/x86/start.rs b/src/arch/x86/start.rs index 77a437f26e..8a72ae9f9f 100644 --- a/src/arch/x86/start.rs +++ b/src/arch/x86/start.rs @@ -18,7 +18,7 @@ use crate::gdt; use crate::idt; use crate::interrupt; use crate::log::{self, info}; -use crate::paging::{self, KernelMapper, TableKind}; +use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKind}; /// Test of zero values in BSS. static BSS_TEST_ZERO: usize = 0; @@ -129,12 +129,11 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { args.areas_base as usize, args.areas_size as usize, args.bootstrap_base as usize, args.bootstrap_size as usize, ); - // Initialize paging - let tcb_offset = paging::init(0); + paging::init(); // Set up GDT after paging with TLS - gdt::init_paging(0, tcb_offset, args.stack_base as usize + args.stack_size as usize); + gdt::init_paging(args.stack_base as usize + args.stack_size as usize); // Set up IDT idt::init_paging_bsp(); @@ -230,16 +229,11 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { idt::init(); // Initialize paging - let tcb_offset = { - use crate::paging::{PageMapper, PhysicalAddress}; - use crate::rmm::FRAME_ALLOCATOR; - - let mut mapper = KernelMapper::lock_for_manual_mapper(cpu_id, PageMapper::new(TableKind::Kernel, PhysicalAddress::new(bsp_table), FRAME_ALLOCATOR)); - paging::init_ap(cpu_id, &mut mapper) - }; + RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table)); + paging::init(); // Set up GDT with TLS - gdt::init_paging(cpu_id as u32, tcb_offset, stack_end); + gdt::init_paging(stack_end); // Set up IDT for AP idt::init_paging_post_heap(false, cpu_id); diff --git a/src/arch/x86_64/consts.rs b/src/arch/x86_64/consts.rs index 1656c8c26f..d50af6bec8 100644 --- a/src/arch/x86_64/consts.rs +++ b/src/arch/x86_64/consts.rs @@ -20,16 +20,6 @@ /// Size of kernel heap pub const KERNEL_HEAP_SIZE: usize = 1 * 1024 * 1024; // 1 MB - /// Offset of temporary mapping for misc kernel bring-up actions - pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE; - - /// Offset to kernel percpu variables - pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_TMP_MISC_OFFSET - PML4_SIZE; - pub const KERNEL_PERCPU_PML4: usize = (KERNEL_PERCPU_OFFSET & PML4_MASK)/PML4_SIZE; - /// Size of kernel percpu variables - pub const KERNEL_PERCPU_SHIFT: u8 = 16; // 2^16 = 64 KiB - pub const KERNEL_PERCPU_SIZE: usize = 1_usize << KERNEL_PERCPU_SHIFT; - /// Offset of physmap // This needs to match RMM's PHYS_OFFSET pub const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 520c8cdc9d..ceaa9aac42 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,10 +1,10 @@ //! Global descriptor table -use core::cell::UnsafeCell; use core::convert::TryInto; use core::mem; -use x86::segmentation::load_cs; +use crate::paging::{PAGE_SIZE, RmmA, RmmArch}; + use x86::bits64::task::TaskStateSegment; use x86::Ring; use x86::dtables::{self, DescriptorTablePointer}; @@ -16,13 +16,11 @@ use super::cpuid::cpuid; pub const GDT_NULL: usize = 0; pub const GDT_KERNEL_CODE: usize = 1; pub const GDT_KERNEL_DATA: usize = 2; -pub const GDT_KERNEL_KPCR: usize = 3; -pub const GDT_USER_CODE32_UNUSED: usize = 4; -pub const GDT_USER_DATA: usize = 5; -pub const GDT_USER_CODE: usize = 6; -pub const GDT_TSS: usize = 7; -pub const GDT_TSS_HIGH: usize = 8; -pub const GDT_CPU_ID_CONTAINER: usize = 9; +pub const GDT_USER_CODE32_UNUSED: usize = 3; +pub const GDT_USER_DATA: usize = 4; +pub const GDT_USER_CODE: usize = 5; +pub const GDT_TSS: usize = 6; +pub const GDT_TSS_HIGH: usize = 7; pub const GDT_A_PRESENT: u8 = 1 << 7; pub const GDT_A_RING_0: u8 = 0 << 5; @@ -42,28 +40,24 @@ pub const GDT_F_PAGE_SIZE: u8 = 1 << 7; pub const GDT_F_PROTECTED_MODE: u8 = 1 << 6; pub const GDT_F_LONG_MODE: u8 = 1 << 5; -static mut INIT_GDT: [GdtEntry; 4] = [ +static mut INIT_GDT: [GdtEntry; 3] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), // Kernel data GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), - // Kernel TLS - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE) ]; -#[thread_local] -pub static GDT: UnsafeCell<[GdtEntry; 10]> = UnsafeCell::new([ +// Later copied into the actual GDT with various fields set. +const BASE_GDT: [GdtEntry; 8] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), // Kernel data GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), - // Kernel TLS - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), - // Dummy 32-bit user code - apparently necessary for SYSEXIT. We restrict it to ring 0 anyway. + // Dummy 32-bit user code - apparently necessary for SYSRET. We restrict it to ring 0 anyway. GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PROTECTED_MODE), // User data GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), @@ -73,165 +67,175 @@ pub static GDT: UnsafeCell<[GdtEntry; 10]> = UnsafeCell::new([ GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0), // TSS must be 16 bytes long, twice the normal size GdtEntry::new(0, 0, 0, 0), - // Unused entry which stores the CPU ID. This is necessary for paranoid interrupts as they have - // no other way of determining it. - GdtEntry::new(0, 0, 0, 0), -]); +]; -#[repr(C, align(16))] +#[repr(C, align(4096))] pub struct ProcessorControlRegion { + // TODO: When both KASLR and KPTI are implemented, the PCR may need to be split into two pages, + // such that "secret" kernel addresses are only stored in the protected half. + pub tcb_end: usize, pub user_rsp_tmp: usize, - pub tss: TssWrapper, + // TODO: The I/O permissions bitmap can require more than 8192 bytes of space. + pub tss: TaskStateSegment, + pub self_ref: usize, + // The GDT *must* be stored in the PCR! The paranoid interrupt handler, lacking a reliable way + // to correctly obtain GSBASE, uses SGDT to calculate the PCR offset. + pub gdt: [GdtEntry; 8], + // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it + // first to avoid cache invalidation. } -// NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding -// inserted before and after the TSS, the main TSS structure will remain intact. -#[repr(C, align(16))] -pub struct TssWrapper(pub TaskStateSegment); +const _: () = { + if memoffset::offset_of!(ProcessorControlRegion, tss) % 16 != 0 { + panic!("PCR is incorrectly defined, TSS alignment is too small"); + } + if memoffset::offset_of!(ProcessorControlRegion, gdt) % 8 != 0 { + panic!("PCR is incorrectly defined, GDT alignment is too small"); + } +}; -#[thread_local] -pub static KPCR: UnsafeCell = UnsafeCell::new(ProcessorControlRegion { - tcb_end: 0, - user_rsp_tmp: 0, - tss: TssWrapper(TaskStateSegment { - reserved: 0, - rsp: [0; 3], - reserved2: 0, - ist: [0; 7], - reserved3: 0, - reserved4: 0, - iomap_base: 0xFFFF - }), -}); +pub unsafe fn pcr() -> *mut ProcessorControlRegion { + // Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that + // obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably + // microcoded) instructions. + let mut ret: *mut ProcessorControlRegion; + core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(memoffset::offset_of!(ProcessorControlRegion, self_ref))); + ret +} #[cfg(feature = "pti")] pub unsafe fn set_tss_stack(stack: usize) { use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK}; - (*KPCR.get()).tss.0.rsp[0] = (PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64; + core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64); PTI_CONTEXT_STACK = stack; } #[cfg(not(feature = "pti"))] pub unsafe fn set_tss_stack(stack: usize) { - (*KPCR.get()).tss.0.rsp[0] = stack as u64; + // TODO: If this increases performance, read gs:[offset] directly + core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write(stack as u64); } -// Initialize GDT +// Initialize startup GDT +#[cold] pub unsafe fn init() { - { - // Setup the initial GDT with TLS, so we can setup the TLS GDT (a little confusing) - // This means that each CPU will have its own GDT, but we only need to define it once as a thread local + // Before the kernel can remap itself, it needs to switch to a GDT it controls. Start with a + // minimal kernel-only GDT. + dtables::lgdt(&DescriptorTablePointer { + limit: (INIT_GDT.len() * mem::size_of::() - 1) as u16, + base: INIT_GDT.as_ptr() as *const SegmentDescriptor, + }); - let limit = (INIT_GDT.len() * mem::size_of::() - 1) - .try_into() - .expect("initial GDT way too large"); - let base = INIT_GDT.as_ptr() as *const SegmentDescriptor; - - let init_gdtr: DescriptorTablePointer = DescriptorTablePointer { - limit, - base, - }; - - // Load the initial GDT, before we have access to thread locals - dtables::lgdt(&init_gdtr); - } - - // Load the segment descriptors - load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); - segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0)); + load_segments(); +} +#[cold] +unsafe fn load_segments() { + segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); + + segmentation::load_ds(SegmentSelector::from_raw(0)); + segmentation::load_es(SegmentSelector::from_raw(0)); + segmentation::load_fs(SegmentSelector::from_raw(0)); + + // What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However, + // GSBASE is set later, and percpu is not used until gdt::init_paging(). + segmentation::load_gs(SegmentSelector::from_raw(0)); } -/// Initialize GDT with TLS -pub unsafe fn init_paging(cpu_id: u32, tcb_offset: usize, stack_offset: usize) { - // Set temporary TLS segment to the self-pointer of the Thread Control Block. - x86::msr::wrmsr(x86::msr::IA32_GS_BASE, tcb_offset as u64); +/// Initialize GDT and PCR. +#[cold] +pub unsafe fn init_paging(stack_offset: usize) { + let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR"); + let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); - // Now that we have access to thread locals, begin by getting a pointer to the Processor - // Control Region. - let kpcr = KPCR.get(); + pcr.self_ref = pcr as *mut ProcessorControlRegion as usize; - // Then, setup the AP's individual GDT - let limit = ((*GDT.get()).len() * mem::size_of::() - 1) + // Setup the GDT. + pcr.gdt = BASE_GDT; + + let limit = (pcr.gdt.len() * mem::size_of::() - 1) .try_into() .expect("main GDT way too large"); - let base = GDT.get() as *const SegmentDescriptor; + let base = pcr.gdt.as_ptr() as *const SegmentDescriptor; let gdtr: DescriptorTablePointer = DescriptorTablePointer { limit, base, }; - // Once we have fetched the real KPCR address, set the TLS segment to the TCB pointer there. - (*kpcr).tcb_end = (tcb_offset as *const usize).read(); + pcr.tcb_end = init_percpu(); { - // We can now access our TSS, via the KPCR, which is a thread local - let tss = &(*kpcr).tss.0 as *const _ as usize as u64; + pcr.tss.iomap_base = 0xFFFF; + + let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64; let tss_lo = (tss & 0xFFFF_FFFF) as u32; let tss_hi = (tss >> 32) as u32; - (*GDT.get())[GDT_TSS].set_offset(tss_lo); - (*GDT.get())[GDT_TSS].set_limit(mem::size_of::() as u32); + pcr.gdt[GDT_TSS].set_offset(tss_lo); + pcr.gdt[GDT_TSS].set_limit(mem::size_of::() as u32); - (&mut (*GDT.get())[GDT_TSS_HIGH] as *mut GdtEntry) - .cast::() - .write(tss_hi); + (&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry).cast::().write(tss_hi); } - // And finally, populate the last GDT entry with the current CPU ID, to allow paranoid - // interrupt handlers to safely use TLS. - (&mut (*GDT.get())[GDT_CPU_ID_CONTAINER] as *mut GdtEntry) - .cast::() - .write(cpu_id); - - // Set the stack pointer to use when coming back from userspace. - set_tss_stack(stack_offset); - // Load the new GDT, which is correctly located in thread local storage. dtables::lgdt(&gdtr); - // Ensure that GS always points to the KPCR in kernel space. - x86::msr::wrmsr(x86::msr::IA32_GS_BASE, kpcr as *mut _ as usize as u64); - // Inside kernel space, GS should _always_ point to the TSS. When leaving userspace, `swapgs` - // is called again, making the userspace GS always point to user data. + // Load segments again, possibly resetting FSBASE and GSBASE. + load_segments(); + + // Ensure that GSBASE always points to the PCR in kernel space. + x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64); + + // While GSBASE points to the PCR in kernel space, userspace is free to set it to other values. + // Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is + // because entering usermode will entail executing the SWAPGS instruction. x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0); - // Set the User TLS segment to zero, before we create any contexts and start scheduling. + // Set the userspace FSBASE to zero. x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0); - // Reload the segment descriptors - load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0)); - segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0)); - - // NOTE: FS has already been updated while calling set_tcb. - // NOTE: We do not want to load GS again, since it has already been loaded into - // GDT_KERNEL_KPCR. Instead, we use the base MSR to allow for a 64-bit offset. + // Set the stack pointer to use when coming back from userspace. + set_tss_stack(stack_offset); // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); - let has_fsgsbase = cpuid().map_or(false, |cpuid| { + let cpu_supports_fsgsbase = cpuid().map_or(false, |cpuid| { cpuid.get_extended_feature_info().map_or(false, |extended_features| { extended_features.has_fsgsbase() }) }); if cfg!(feature = "x86_fsgsbase") { - assert!(has_fsgsbase, "running kernel with features not supported by the current CPU"); - } + assert!(cpu_supports_fsgsbase, "running kernel with features not supported by the current CPU"); - if has_fsgsbase { x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE); } } +/// Copy tdata, clear tbss, calculate TCB end pointer +#[cold] +unsafe fn init_percpu() -> usize { + use crate::kernel_executable_offsets::*; + + let size = __tbss_end() - __tdata_start(); + assert_eq!(size % PAGE_SIZE, 0); + + let tbss_offset = __tbss_start() - __tdata_start(); + + let base_frame = crate::memory::allocate_frames(size / PAGE_SIZE).expect("failed to allocate percpu memory"); + let base = RmmA::phys_to_virt(base_frame.start_address()); + + let tls_end = base.data() + size; + + core::ptr::copy_nonoverlapping(__tdata_start() as *const u8, base.data() as *mut u8, tbss_offset); + core::ptr::write_bytes((base.data() + tbss_offset) as *mut u8, 0, size - tbss_offset); + + tls_end +} + #[derive(Copy, Clone, Debug)] #[repr(packed)] pub struct GdtEntry { diff --git a/src/arch/x86_64/idt.rs b/src/arch/x86_64/idt.rs index 5107ab9206..6adffb7fa2 100644 --- a/src/arch/x86_64/idt.rs +++ b/src/arch/x86_64/idt.rs @@ -176,7 +176,7 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) { let address = base_address.data() + BACKUP_STACK_SIZE; // Put them in the 1st entry of the IST. - (*crate::gdt::KPCR.get()).tss.0.ist[usize::from(index - 1)] = address as u64; + (*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64; index }; diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index acdf4a519d..b7962a27ed 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -258,6 +258,7 @@ macro_rules! pop_preserved { " }; } macro_rules! swapgs_iff_ring3_fast { + // TODO: Spectre V1: LFENCE? () => { " // Check whether the last two bits RSP+8 (code segment) are equal to zero. test QWORD PTR [rsp + 8], 0x3 @@ -268,6 +269,7 @@ macro_rules! swapgs_iff_ring3_fast { " }; } macro_rules! swapgs_iff_ring3_fast_errorcode { + // TODO: Spectre V1: LFENCE? () => { " test QWORD PTR [rsp + 16], 0x3 jz 1f @@ -276,109 +278,87 @@ macro_rules! swapgs_iff_ring3_fast_errorcode { " }; } -#[cfg(feature = "x86_fsgsbase")] -macro_rules! save_gsbase_paranoid { - () => { " - // Unused: {IA32_GS_BASE} - rdgsbase rax - push rax - " } +#[cfg(feature = "x86_fsbase")] +macro_rules! read_gsbase_into_rdx { + () => { "rdgsbase rdx;" } } -#[cfg(feature = "x86_fsgsbase")] -macro_rules! restore_gsbase_paranoid { - () => { " - // Unused: {IA32_GS_BASE} - pop rax - wrgsbase rax - " } -} -#[cfg(not(feature = "x86_fsgsbase"))] -macro_rules! save_gsbase_paranoid { + +#[cfg(not(feature = "x86_fsbase"))] +macro_rules! read_gsbase_into_rdx { () => { " mov ecx, {IA32_GS_BASE} rdmsr shl rdx, 32 - or rax, rdx - - push rax - " } -} -#[cfg(not(feature = "x86_fsgsbase"))] -macro_rules! restore_gsbase_paranoid { - () => { " - pop rdx - - mov ecx, {IA32_GS_BASE} - mov eax, edx - shr rdx, 32 - wrmsr + or rdx, rax " } } -#[cfg(feature = "x86_fsgsbase")] -macro_rules! set_gsbase_paranoid { - () => { " - // Unused: {IA32_GS_BASE} - wrgsbase rdx - " } -} -#[cfg(not(feature = "x86_fsgsbase"))] -macro_rules! set_gsbase_paranoid { - () => { " - mov ecx, {IA32_GS_BASE} - mov eax, edx - shr rdx, 32 - wrmsr - " } -} - -macro_rules! save_and_set_gsbase_paranoid { - // For paranoid interrupt entries, we have to be extremely careful with how we use IA32_GS_BASE - // and IA32_KERNEL_GS_BASE. If FSGSBASE is enabled, then we have no way to differentiate these - // two, as paranoid interrupts (e.g. NMIs) can occur even in kernel mode. In fact, they can - // even occur within another IRQ, so we cannot check the the privilege level via the stack. +macro_rules! conditional_swapgs_paranoid { + // For regular interrupt handlers and the syscall handler, managing IA32_GS_BASE and + // IA32_KERNEL_GS_BASE (the "GSBASE registers") is more or less trivial when using the SWAPGS + // instruction. // - // What we do instead, is using a special entry in the GDT, since we know that the GDT will - // always be thread local, as it contains the TSS. This gives us more than 32 bits to work - // with, which already is the largest x2APIC ID that an x86 CPU can handle. Luckily we can also - // use the stack, even though there might be interrupts in between. + // The syscall handler simply runs SWAPGS, as syscalls can only originate from usermode, + // whereas interrupt handlers conditionally SWAPGS unless the interrupt was triggered from + // kernel mode, in which case the "swap state" is already valid, and there is no need to + // SWAPGS. // - // TODO: Linux uses the Interrupt Stack Table to figure out which NMIs were nested. Perhaps - // this could be done here, because if nested (sp > initial_sp), that means the NMI could not - // have come from userspace. But then, knowing the initial sp would somehow have to involve - // percpu, which brings us back to square one. But it might be useful if we would allow faults - // in NMIs. If we do detect a nested interrupt, then we can perform the iretq procedure - // ourselves, so that the newly nested NMI still blocks additional interrupts while still - // returning to the previously (faulting) NMI. See https://lwn.net/Articles/484932/, although I - // think the solution becomes a bit simpler when we cannot longer rely on GSBASE anymore. + // Handling GSBASE correctly for paranoid interrupts however, is not as simple. NMIs can occur + // between the check of whether an interrupt came from usermode, and the actual SWAPGS + // instruction. #DB can also be triggered inside of a kernel interrupt handler, due to + // breakpoints, even though setting up such breakpoints in the first place, is not yet + // supported by the kernel. + // + // Luckily, the GDT always resides in the PCR (at least after init_paging, but there are no + // interrupt handlers set up before that), allowing GSBASE to be calculated relatively cheaply. + // Out of the two GSBASE registers, at least one must be *the* kernel GSBASE, allowing for a + // simple conditional SWAPGS. + // + // (An alternative to conditionally executing SWAPGS, would be to save and restore GSBASE via + // e.g. the stack. That would nonetheless require saving and restoring both GSBASE registers, + // if the interrupt handler should be allowed to context switch, which the current #DB handler + // may do.) + // + // TODO: Handle nested NMIs like Linux does (https://lwn.net/Articles/484932/)?. () => { concat!( - save_gsbase_paranoid!(), - - // Allocate stack space for 8 bytes GDT base and 2 bytes size (ignored). - "sub rsp, 16\n", - // Set it to the GDT base. - "sgdt [rsp + 6]\n", - // Get the base pointer + // Put the GDT base pointer in RDI. " - mov rax, [rsp + 8] + sub rsp, 16 + sgdt [rsp + 6] + mov rdi, [rsp + 8] add rsp, 16 ", - // Load the lower 32 bits of that GDT entry. - "mov edx, [rax + {gdt_cpu_id_offset}]\n", - // Calculate the percpu offset. + // Calculate the PCR address by subtracting the offset of the GDT in the PCR struct. + "sub rdi, {PCR_GDT_OFFSET};", + + // Read the current IA32_GS_BASE value into RDX. + read_gsbase_into_rdx!(), + + // If they were not equal, the PCR address must instead be in IA32_KERNEL_GS_BASE, + // requiring a SWAPGS. GSBASE needs to be swapped back, so store the same flag in RBX. + + // TODO: Spectre V1: LFENCE? " - mov rbx, {KERNEL_PERCPU_OFFSET} - shl rdx, {KERNEL_PERCPU_SHIFT} - add rdx, rbx + cmp rdx, rdi + sete bl + je 1f + swapgs + 1: ", - // Set GSBASE to RAX accordingly - set_gsbase_paranoid!(), ) } } +macro_rules! conditional_swapgs_back_paranoid { + () => { " + test ebx, ebx + jnz 1f + swapgs + 1: + " } +} macro_rules! nop { () => { " - // Unused: {IA32_GS_BASE} {KERNEL_PERCPU_OFFSET} {KERNEL_PERCPU_SHIFT} {gdt_cpu_id_offset} + // Unused: {IA32_GS_BASE} {PCR_GDT_OFFSET} " } } @@ -399,7 +379,10 @@ macro_rules! interrupt_stack { _guard = $crate::ptrace::set_process_regs($stack); } - // TODO: Force the declarations to specify unsafe? + // TODO: Force the declarations to specify unsafe? For example, accessing a global + // core::cell::Cell is safe when running at the "bottom level" (interrupt from + // userspace or prior to entering userspace), but UB if simultaneously accessed + // from an interrupt. #[allow(unused_unsafe)] unsafe { @@ -442,10 +425,8 @@ macro_rules! interrupt_stack { inner = sym inner, IA32_GS_BASE = const(x86::msr::IA32_GS_BASE), - KERNEL_PERCPU_SHIFT = const(crate::KERNEL_PERCPU_SHIFT), - KERNEL_PERCPU_OFFSET = const(crate::KERNEL_PERCPU_OFFSET), - gdt_cpu_id_offset = const(crate::gdt::GDT_CPU_ID_CONTAINER * core::mem::size_of::()), + PCR_GDT_OFFSET = const(memoffset::offset_of!(crate::gdt::ProcessorControlRegion, gdt)), options(noreturn), @@ -453,7 +434,7 @@ macro_rules! interrupt_stack { } }; ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, swapgs_iff_ring3_fast!, nop!, nop!, swapgs_iff_ring3_fast!, is_paranoid: false, |$stack| $code); }; - ($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, save_and_set_gsbase_paranoid!, restore_gsbase_paranoid!, nop!, is_paranoid: true, |$stack| $code); } + ($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, conditional_swapgs_paranoid!, conditional_swapgs_back_paranoid!, nop!, is_paranoid: true, |$stack| $code); } } #[macro_export] diff --git a/src/arch/x86_64/paging/mod.rs b/src/arch/x86_64/paging/mod.rs index aebe64b4ee..f9726fbc10 100644 --- a/src/arch/x86_64/paging/mod.rs +++ b/src/arch/x86_64/paging/mod.rs @@ -1,11 +1,8 @@ //! # Paging //! Some code was borrowed from [Phil Opp's Blog](http://os.phil-opp.com/modifying-page-tables.html) -use core::{mem, ptr}; use x86::msr; -use self::mapper::PageFlushAll; - pub use rmm::{ Arch as RmmArch, Flusher, @@ -29,6 +26,7 @@ pub const ENTRY_COUNT: usize = RmmA::PAGE_ENTRIES; pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE; /// Setup page attribute table +#[cold] unsafe fn init_pat() { let uncacheable = 0; let write_combining = 1; @@ -60,124 +58,10 @@ unsafe fn init_pat() { ); } -/// Map percpu -unsafe fn map_percpu(cpu_id: usize, mapper: &mut PageMapper) -> PageFlushAll { - extern "C" { - /// The starting byte of the thread data segment - static mut __tdata_start: u8; - /// The ending byte of the thread data segment - static mut __tdata_end: u8; - /// The starting byte of the thread BSS segment - static mut __tbss_start: u8; - /// The ending byte of the thread BSS segment - static mut __tbss_end: u8; - } - - let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize; - let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id; - let end = start + size; - - let mut flush_all = PageFlushAll::new(); - let start_page = Page::containing_address(VirtualAddress::new(start)); - let end_page = Page::containing_address(VirtualAddress::new(end - 1)); - for page in Page::range_inclusive(start_page, end_page) { - let result = mapper.map( - page.start_address(), - PageFlags::new().write(true).global(cfg!(not(feature = "pti"))), - ) - .expect("failed to allocate page table frames while mapping percpu"); - flush_all.consume(result); - } - flush_all -} - -/// Copy tdata, clear tbss, set TCB self pointer -unsafe fn init_tcb(cpu_id: usize) -> usize { - extern "C" { - /// The starting byte of the thread data segment - static mut __tdata_start: u8; - /// The ending byte of the thread data segment - static mut __tdata_end: u8; - /// The starting byte of the thread BSS segment - static mut __tbss_start: u8; - /// The ending byte of the thread BSS segment - static mut __tbss_end: u8; - } - - let tcb_offset; - { - let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize; - let tbss_offset = &__tbss_start as *const _ as usize - &__tdata_start as *const _ as usize; - - let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id; - let end = start + size; - tcb_offset = end - mem::size_of::(); - - ptr::copy(&__tdata_start as *const u8, start as *mut u8, tbss_offset); - ptr::write_bytes((start + tbss_offset) as *mut u8, 0, size - tbss_offset); - - *(tcb_offset as *mut usize) = end; - } - tcb_offset -} - -/// Initialize paging -/// -/// Returns page table and thread control block offset -pub unsafe fn init( - cpu_id: usize, -) -> usize { - extern "C" { - /// The starting byte of the text (code) data segment. - static mut __text_start: u8; - /// The ending byte of the text (code) data segment. - static mut __text_end: u8; - /// The starting byte of the _.rodata_ (read-only data) segment. - static mut __rodata_start: u8; - /// The ending byte of the _.rodata_ (read-only data) segment. - static mut __rodata_end: u8; - /// The starting byte of the _.data_ segment. - static mut __data_start: u8; - /// The ending byte of the _.data_ segment. - static mut __data_end: u8; - /// The starting byte of the thread data segment - static mut __tdata_start: u8; - /// The ending byte of the thread data segment - static mut __tdata_end: u8; - /// The starting byte of the thread BSS segment - static mut __tbss_start: u8; - /// The ending byte of the thread BSS segment - static mut __tbss_end: u8; - /// The starting byte of the _.bss_ (uninitialized data) segment. - static mut __bss_start: u8; - /// The ending byte of the _.bss_ (uninitialized data) segment. - static mut __bss_end: u8; - } - +/// Initialize PAT +#[cold] +pub unsafe fn init() { init_pat(); - - let flush_all = map_percpu(cpu_id, KernelMapper::lock_manually(cpu_id).get_mut().expect("expected KernelMapper not to be locked re-entrant in paging::init")); - flush_all.flush(); - - init_tcb(cpu_id) -} - -pub unsafe fn init_ap( - cpu_id: usize, - bsp_table: &mut KernelMapper, -) -> usize { - init_pat(); - - { - let flush_all = map_percpu(cpu_id, bsp_table.get_mut().expect("KernelMapper locked re-entrant for AP")); - - // The flush can be ignored as this is not the active table. See later make_current(). - flush_all.ignore(); - }; - - bsp_table.make_current(); - - init_tcb(cpu_id) } /// Page @@ -191,22 +75,6 @@ impl Page { VirtualAddress::new(self.number * PAGE_SIZE) } - pub fn p4_index(self) -> usize { - (self.number >> 27) & 0o777 - } - - pub fn p3_index(self) -> usize { - (self.number >> 18) & 0o777 - } - - pub fn p2_index(self) -> usize { - (self.number >> 9) & 0o777 - } - - pub fn p1_index(self) -> usize { - self.number & 0o777 - } - pub fn containing_address(address: VirtualAddress) -> Page { //TODO assert!(address.data() < 0x0000_8000_0000_0000 || address.data() >= 0xffff_8000_0000_0000, // "invalid address: 0x{:x}", address.data()); diff --git a/src/arch/x86_64/rmm.rs b/src/arch/x86_64/rmm.rs index fd10673b18..96e7fc6a86 100644 --- a/src/arch/x86_64/rmm.rs +++ b/src/arch/x86_64/rmm.rs @@ -24,17 +24,6 @@ use spin::Mutex; use super::CurrentRmmArch as RmmA; -extern "C" { - /// The starting byte of the text (code) data segment. - static mut __text_start: u8; - /// The ending byte of the text (code) data segment. - static mut __text_end: u8; - /// The starting byte of the _.rodata_ (read-only data) segment. - static mut __rodata_start: u8; - /// The ending byte of the _.rodata_ (read-only data) segment. - static mut __rodata_end: u8; -} - // Keep synced with OsMemoryKind in bootloader #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u64)] @@ -55,20 +44,13 @@ pub struct BootloaderMemoryEntry { } unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { + use crate::kernel_executable_offsets::*; let virt_addr = virt.data(); - // Test for being inside a region - macro_rules! in_section { - ($n: ident) => { - virt_addr >= &concat_idents!(__, $n, _start) as *const u8 as usize - && virt_addr < &concat_idents!(__, $n, _end) as *const u8 as usize - }; - } - - if in_section!(text) { + if virt_addr >= __text_start() && virt_addr < __text_end() { // Remap text read-only, execute PageFlags::new().execute(true) - } else if in_section!(rodata) { + } else if virt_addr >= __rodata_start() && virt_addr < __rodata_end() { // Remap rodata read-only, no execute PageFlags::new() } else { diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index b96c15935f..71e58fd9c7 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -20,7 +20,7 @@ use crate::idt; use crate::interrupt; use crate::misc; use crate::log::{self, info}; -use crate::paging::{self, KernelMapper, TableKind}; +use crate::paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind}; /// Test of zero values in BSS. static BSS_TEST_ZERO: usize = 0; @@ -132,11 +132,11 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { args.bootstrap_base as usize, args.bootstrap_size as usize, ); - // Initialize paging - let tcb_offset = paging::init(0); + // Initialize PAT + paging::init(); // Set up GDT after paging with TLS - gdt::init_paging(0, tcb_offset, args.stack_base as usize + args.stack_size as usize); + gdt::init_paging(args.stack_base as usize + args.stack_size as usize); // Set up IDT idt::init_paging_bsp(); @@ -235,16 +235,11 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { idt::init(); // Initialize paging - let tcb_offset = { - use crate::paging::{PageMapper, PhysicalAddress}; - use crate::rmm::FRAME_ALLOCATOR; - - let mut mapper = KernelMapper::lock_for_manual_mapper(cpu_id, PageMapper::new(TableKind::Kernel, PhysicalAddress::new(bsp_table), FRAME_ALLOCATOR)); - paging::init_ap(cpu_id, &mut mapper) - }; + RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table)); + paging::init(); // Set up GDT with TLS - gdt::init_paging(cpu_id as u32, tcb_offset, stack_end); + gdt::init_paging(stack_end); // Set up IDT for AP idt::init_paging_post_heap(false, cpu_id); diff --git a/src/context/arch/x86.rs b/src/context/arch/x86.rs index 3d24a336ad..7789c36100 100644 --- a/src/context/arch/x86.rs +++ b/src/context/arch/x86.rs @@ -4,7 +4,7 @@ use core::sync::atomic::AtomicBool; use alloc::sync::Arc; use crate::{push_scratch, pop_scratch}; -use crate::gdt::{GDT, GDT_USER_FS, GDT_USER_GS}; +use crate::gdt::{pcr, GDT_USER_FS, GDT_USER_GS}; use crate::interrupt::handler::ScratchRegisters; use crate::paging::{RmmA, RmmArch, TableKind}; use crate::syscall::FloatRegisters; @@ -137,10 +137,12 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { ); { - prev.arch.fsbase = GDT[GDT_USER_FS].offset() as usize; - GDT[GDT_USER_FS].set_offset(next.arch.fsbase as u32); - prev.arch.gsbase = GDT[GDT_USER_GS].offset() as usize; - GDT[GDT_USER_GS].set_offset(next.arch.gsbase as u32); + let gdt = &mut (&mut *pcr()).gdt; + + prev.arch.fsbase = gdt[GDT_USER_FS].offset() as usize; + gdt[GDT_USER_FS].set_offset(next.arch.fsbase as u32); + prev.arch.gsbase = gdt[GDT_USER_GS].offset() as usize; + gdt[GDT_USER_GS].set_offset(next.arch.gsbase as u32); } match next.addr_space { diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index b648674f1d..2f9900d147 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -154,17 +154,15 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { wrgsbase(next.arch.gsbase as u64); swapgs(); } else { - prev.arch.fsbase = msr::rdmsr(msr::IA32_FS_BASE) as usize; msr::wrmsr(msr::IA32_FS_BASE, next.arch.fsbase as u64); - prev.arch.gsbase = msr::rdmsr(msr::IA32_KERNEL_GSBASE) as usize; msr::wrmsr(msr::IA32_KERNEL_GSBASE, next.arch.gsbase as u64); } } match next.addr_space { - // Since Arc is essentially just wraps a pointer, in this case a regular pointer (as - // opposed to dyn or slice fat pointers), and NonNull optimization exists, map_or will - // hopefully be optimized down to checking prev and next pointers, as next cannot be null. + // Since Arc essentially just wraps a pointer, in this case a regular pointer (as opposed + // to dyn or slice fat pointers), and NonNull optimization exists, map_or will hopefully be + // optimized down to checking prev and next pointers, as next cannot be null. Some(ref next_space) => if prev.addr_space.as_ref().map_or(true, |prev_space| !Arc::ptr_eq(prev_space, next_space)) { // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A // recently called yield and is now here about to switch back. Meanwhile, B is diff --git a/src/context/memory.rs b/src/context/memory.rs index e56f97eedb..37b0ec4d79 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -960,9 +960,6 @@ pub fn setup_new_utable() -> Result { // Copy physmap mapping copy_mapping(crate::PHYS_PML4); - - // Copy kernel percpu (similar to TLS) mapping. - copy_mapping(crate::KERNEL_PERCPU_PML4); } Ok(Table { diff --git a/src/lib.rs b/src/lib.rs index 3d3db8ef96..c3611bbaa1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -279,3 +279,23 @@ pub extern fn ksignal(signal: usize) { syscall::exit(signal & 0x7F); }); } + +// TODO: Use this macro on aarch64 too. + +macro_rules! linker_offsets( + ($($name:ident),*) => { + $( + #[inline] + pub fn $name() -> usize { + extern "C" { + // TODO: UnsafeCell? + static $name: u8; + } + unsafe { &$name as *const u8 as usize } + } + )* + } +); +pub mod kernel_executable_offsets { + linker_offsets!(__text_start, __text_end, __rodata_start, __rodata_end, __data_start, __data_end, __bss_start, __bss_end, __tdata_start, __tdata_end, __tbss_start, __tbss_end); +} diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 3384e37d9c..df2d2633c5 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -430,8 +430,8 @@ impl ProcScheme { let (fsbase, gsbase) = if info.pid == context::context_id() { unsafe { ( - crate::gdt::GDT[crate::gdt::GDT_USER_FS].offset() as u64, - crate::gdt::GDT[crate::gdt::GDT_USER_GS].offset() as u64 + (&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_FS].offset() as u64, + (&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].offset() as u64 ) } } else { @@ -501,8 +501,8 @@ impl ProcScheme { if info.pid == context::context_id() { unsafe { - crate::gdt::GDT[crate::gdt::GDT_USER_FS].set_offset(regs.fsbase); - crate::gdt::GDT[crate::gdt::GDT_USER_GS].set_offset(regs.gsbase); + (&mut *crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_FS].set_offset(regs.fsbase); + (&mut *crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].set_offset(regs.gsbase); match context::contexts().current().ok_or(Error::new(ESRCH))?.write().arch { ref mut arch => { diff --git a/targets/i686-unknown-kernel.json b/targets/i686-unknown-kernel.json index 3964322bf8..328a72f35a 100644 --- a/targets/i686-unknown-kernel.json +++ b/targets/i686-unknown-kernel.json @@ -3,7 +3,7 @@ "target-endian": "little", "target-pointer-width": "32", "target-c-int-width": "32", - "data-layout": "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128", + "data-layout": "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S64", "arch": "x86", "os": "none", "env": "", diff --git a/targets/x86_64-unknown-kernel.json b/targets/x86_64-unknown-kernel.json index 1437a35cf7..1ab209b0a3 100644 --- a/targets/x86_64-unknown-kernel.json +++ b/targets/x86_64-unknown-kernel.json @@ -3,7 +3,7 @@ "target-endian": "little", "target-pointer-width": "64", "target-c-int-width": "32", - "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", + "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S64", "arch": "x86_64", "os": "none", "env": "",