vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify x86_64 percpu and GSBASE calculation

Browse Source
This commit is contained in:
4lDO2
2023-07-10 14:13:12 +00:00
committed by Jeremy Soller
parent 5c99d73bbe
commit 3ded84c945
19 changed files with 334 additions and 635 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linkers/i686.ld
+11 -14
View File
@@ -7,45 +7,42 @@ SECTIONS {
. = KERNEL_OFFSET;
. += SIZEOF_HEADERS;
. = ALIGN(4096);
.text : AT(ADDR(.text) - KERNEL_OFFSET) {
.text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) {
__text_start = .;
*(.text*)
__usercopy_start = .;
*(.usercopy-fns)
__usercopy_end = .;
. = ALIGN(4096);
__text_end = .;
}
.rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) {
.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) {
__text_end = .;
__rodata_start = .;
*(.rodata*)
. = ALIGN(4096);
__rodata_end = .;
}
.data : AT(ADDR(.data) - KERNEL_OFFSET) {
.data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) {
__rodata_end = .;
__data_start = .;
*(.data*)
. = ALIGN(4096);
. = ALIGN(4K);
__data_end = .;
__bss_start = .;
*(.bss*)
. = ALIGN(4096);
__bss_end = .;
. = ALIGN(4K);
}
.tdata : AT(ADDR(.tdata) - KERNEL_OFFSET) {
.tdata ALIGN(4K) : AT(ADDR(.tdata) - KERNEL_OFFSET) {
__bss_end = .;
__tdata_start = .;
*(.tdata*)
. = ALIGN(4096);
. = ALIGN(4K);
__tdata_end = .;
__tbss_start = .;
*(.tbss*)
. += 8;
. = ALIGN(4096);
. = ALIGN(4K);
__tbss_end = .;
}
linkers/x86_64.ld
+9 -15
View File
@@ -7,45 +7,39 @@ SECTIONS {
. = KERNEL_OFFSET;
. += SIZEOF_HEADERS;
. = ALIGN(4096);
.text : AT(ADDR(.text) - KERNEL_OFFSET) {
.text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) {
__text_start = .;
*(.text*)
__usercopy_start = .;
*(.usercopy-fns)
__usercopy_end = .;
. = ALIGN(4096);
__text_end = .;
}
.rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) {
.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) {
__text_end = .;
__rodata_start = .;
*(.rodata*)
. = ALIGN(4096);
__rodata_end = .;
}
.data : AT(ADDR(.data) - KERNEL_OFFSET) {
.data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) {
__rodata_end = .;
__data_start = .;
*(.data*)
. = ALIGN(4096);
. = ALIGN(4K);
__data_end = .;
__bss_start = .;
*(.bss*)
. = ALIGN(4096);
__bss_end = .;
}
.tdata : AT(ADDR(.tdata) - KERNEL_OFFSET) {
.tdata ALIGN(4K) : AT(ADDR(.tdata) - KERNEL_OFFSET) {
__bss_end = .;
__tdata_start = .;
*(.tdata*)
. = ALIGN(4096);
__tdata_end = .;
__tbss_start = .;
*(.tbss*)
. += 8;
. = ALIGN(4096);
. = ALIGN(4K);
__tbss_end = .;
}
src/arch/x86/gdt.rs
+69 -78
View File
@@ -2,26 +2,27 @@
use core::convert::TryInto;
use core::mem;
use core::ptr::addr_of_mut;
use x86::segmentation::load_cs;
use x86::bits32::task::TaskStateSegment;
use x86::Ring;
use x86::dtables::{self, DescriptorTablePointer};
use x86::segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector};
use x86::task;
use crate::paging::{RmmA, RmmArch, PAGE_SIZE};
use super::cpuid::cpuid;
pub const GDT_NULL: usize = 0;
pub const GDT_KERNEL_CODE: usize = 1;
pub const GDT_KERNEL_DATA: usize = 2;
pub const GDT_KERNEL_KPCR: usize = 3;
pub const GDT_KERNEL_PERCPU: usize = 3;
pub const GDT_USER_CODE: usize = 4;
pub const GDT_USER_DATA: usize = 5;
pub const GDT_USER_FS: usize = 6;
pub const GDT_USER_GS: usize = 7;
pub const GDT_TSS: usize = 8;
pub const GDT_CPU_ID_CONTAINER: usize = 9;
pub const GDT_A_PRESENT: u8 = 1 << 7;
pub const GDT_A_RING_0: u8 = 0 << 5;
@@ -41,19 +42,16 @@ pub const GDT_F_PAGE_SIZE: u8 = 1 << 7;
pub const GDT_F_PROTECTED_MODE: u8 = 1 << 6;
pub const GDT_F_LONG_MODE: u8 = 1 << 5;
static mut INIT_GDT: [GdtEntry; 4] = [
static INIT_GDT: [GdtEntry; 3] = [
// Null
GdtEntry::new(0, 0, 0, 0),
// Kernel code
GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE),
// Kernel data
GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE),
// Kernel TLS
GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE),
];
#[thread_local]
pub static mut GDT: [GdtEntry; 10] = [
const BASE_GDT: [GdtEntry; 9] = [
// Null
GdtEntry::new(0, 0, 0, 0),
// Kernel code
@@ -72,16 +70,15 @@ pub static mut GDT: [GdtEntry; 10] = [
GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE),
// TSS
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0),
// Unused entry which stores the CPU ID. This is necessary for paranoid interrupts as they have
// no other way of determining it.
GdtEntry::new(0, 0, 0, 0),
];
#[repr(C, align(16))]
#[repr(C, align(4096))]
pub struct ProcessorControlRegion {
pub tcb_end: usize,
pub user_rsp_tmp: usize,
pub tss: TssWrapper,
pub self_ref: usize,
pub gdt: [GdtEntry; 9],
}
// NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding
@@ -89,114 +86,108 @@ pub struct ProcessorControlRegion {
#[repr(C, align(16))]
pub struct TssWrapper(pub TaskStateSegment);
#[thread_local]
pub static mut KPCR: ProcessorControlRegion = ProcessorControlRegion {
tcb_end: 0,
user_rsp_tmp: 0,
tss: TssWrapper(TaskStateSegment::new()),
};
pub unsafe fn pcr() -> *mut ProcessorControlRegion {
let mut ret: *mut ProcessorControlRegion;
core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(memoffset::offset_of!(ProcessorControlRegion, self_ref)));
ret
}
#[cfg(feature = "pti")]
pub unsafe fn set_tss_stack(stack: usize) {
use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK};
KPCR.tss.0.ss0 = (GDT_KERNEL_DATA << 3) as u16;
KPCR.tss.0.esp0 = (PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32;
addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16);
addr_of_mut!((*pcr()).tss.0.esp0).write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32);
PTI_CONTEXT_STACK = stack;
}
#[cfg(not(feature = "pti"))]
pub unsafe fn set_tss_stack(stack: usize) {
KPCR.tss.0.ss0 = (GDT_KERNEL_DATA << 3) as u16;
KPCR.tss.0.esp0 = stack as u32;
addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16);
addr_of_mut!((*pcr()).tss.0.esp0).write(stack as u32);
}
// Initialize GDT
/// Initialize a minimal GDT without configuring percpu.
pub unsafe fn init() {
{
// Setup the initial GDT with TLS, so we can setup the TLS GDT (a little confusing)
// This means that each CPU will have its own GDT, but we only need to define it once as a thread local
let limit = (INIT_GDT.len() * mem::size_of::<GdtEntry>() - 1)
.try_into()
.expect("initial GDT way too large");
let base = INIT_GDT.as_ptr() as *const SegmentDescriptor;
let init_gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
limit,
base,
};
// Load the initial GDT, before we have access to thread locals
dtables::lgdt(&init_gdtr);
}
// Load the initial GDT, before the kernel remaps itself.
dtables::lgdt(&DescriptorTablePointer {
limit: (INIT_GDT.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: INIT_GDT.as_ptr() as *const SegmentDescriptor,
});
// Load the segment descriptors
load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
}
/// Initialize GDT with TLS
pub unsafe fn init_paging(cpu_id: u32, tcb_offset: usize, stack_offset: usize) {
//TODO: will this work with multicore?
{
INIT_GDT[GDT_KERNEL_KPCR].set_offset(tcb_offset as u32);
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0));
}
/// Initialize GDT and configure percpu.
pub unsafe fn init_paging(stack_offset: usize) {
let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR frame");
let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion);
// Now that we have access to thread locals, begin by getting a pointer to the Processor
// Control Region.
let kpcr = &mut KPCR;
// Then, setup the AP's individual GDT
let limit = (GDT.len() * mem::size_of::<GdtEntry>() - 1)
.try_into()
.expect("main GDT way too large");
let base = GDT.as_ptr() as *const SegmentDescriptor;
pcr.self_ref = pcr as *const _ as usize;
pcr.gdt = BASE_GDT;
pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32);
let gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
limit,
base,
limit: (pcr.gdt.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: pcr.gdt.as_ptr() as *const SegmentDescriptor,
};
// Once we have fetched the real KPCR address, set the TLS segment to the TCB pointer there.
kpcr.tcb_end = (tcb_offset as *const usize).read();
GDT[GDT_KERNEL_KPCR].set_offset(tcb_offset as u32);
pcr.tcb_end = init_percpu();
{
// We can now access our TSS, via the KPCR, which is a thread local
let tss = &kpcr.tss.0 as *const _ as usize as u32;
let tss = &pcr.tss.0 as *const _ as usize as u32;
GDT[GDT_TSS].set_offset(tss);
GDT[GDT_TSS].set_limit(mem::size_of::<TaskStateSegment>() as u32);
pcr.gdt[GDT_TSS].set_offset(tss);
pcr.gdt[GDT_TSS].set_limit(mem::size_of::<TaskStateSegment>() as u32);
}
// And finally, populate the last GDT entry with the current CPU ID, to allow paranoid
// interrupt handlers to safely use TLS.
(&mut GDT[GDT_CPU_ID_CONTAINER] as *mut GdtEntry).cast::<u32>().write(cpu_id);
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(stack_offset);
// Load the new GDT, which is correctly located in thread local storage.
dtables::lgdt(&gdtr);
// Reload the segment descriptors
load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
// TODO: Use FS for kernel TLS on i686?
segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0));
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(stack_offset);
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
}
// TODO: Share code with x86. Maybe even with aarch64?
/// Copy tdata, clear tbss, calculate TCB end pointer
#[cold]
unsafe fn init_percpu() -> usize {
use crate::kernel_executable_offsets::*;
let size = __tbss_end() - __tdata_start();
assert_eq!(size % PAGE_SIZE, 0);
let tbss_offset = __tbss_start() - __tdata_start();
let base_frame = crate::memory::allocate_frames(size / PAGE_SIZE).expect("failed to allocate percpu memory");
let base = RmmA::phys_to_virt(base_frame.start_address());
let tls_end = base.data() + size;
core::ptr::copy_nonoverlapping(__tdata_start() as *const u8, base.data() as *mut u8, tbss_offset);
core::ptr::write_bytes((base.data() + tbss_offset) as *mut u8, 0, size - tbss_offset);
tls_end
}
#[derive(Copy, Clone, Debug)]
#[repr(packed)]
pub struct GdtEntry {
src/arch/x86/paging/mod.rs
+3 -117
View File
@@ -29,6 +29,7 @@ pub const ENTRY_COUNT: usize = RmmA::PAGE_ENTRIES;
pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE;
/// Setup page attribute table
#[cold]
unsafe fn init_pat() {
let uncacheable = 0;
let write_combining = 1;
@@ -60,124 +61,9 @@ unsafe fn init_pat() {
);
}
/// Map percpu
unsafe fn map_percpu(cpu_id: usize, mapper: &mut PageMapper) -> PageFlushAll<RmmA> {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
}
let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
let end = start + size;
let mut flush_all = PageFlushAll::new();
let start_page = Page::containing_address(VirtualAddress::new(start));
let end_page = Page::containing_address(VirtualAddress::new(end - 1));
for page in Page::range_inclusive(start_page, end_page) {
let result = mapper.map(
page.start_address(),
PageFlags::new().write(true).global(cfg!(not(feature = "pti"))),
)
.expect("failed to allocate page table frames while mapping percpu");
flush_all.consume(result);
}
flush_all
}
/// Copy tdata, clear tbss, set TCB self pointer
unsafe fn init_tcb(cpu_id: usize) -> usize {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
}
let tcb_offset;
{
let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
let tbss_offset = &__tbss_start as *const _ as usize - &__tdata_start as *const _ as usize;
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
let end = start + size;
tcb_offset = end - mem::size_of::<usize>();
ptr::copy(&__tdata_start as *const u8, start as *mut u8, tbss_offset);
ptr::write_bytes((start + tbss_offset) as *mut u8, 0, size - tbss_offset);
*(tcb_offset as *mut usize) = end;
}
tcb_offset
}
/// Initialize paging
///
/// Returns page table and thread control block offset
pub unsafe fn init(
cpu_id: usize,
) -> usize {
extern "C" {
/// The starting byte of the text (code) data segment.
static mut __text_start: u8;
/// The ending byte of the text (code) data segment.
static mut __text_end: u8;
/// The starting byte of the _.rodata_ (read-only data) segment.
static mut __rodata_start: u8;
/// The ending byte of the _.rodata_ (read-only data) segment.
static mut __rodata_end: u8;
/// The starting byte of the _.data_ segment.
static mut __data_start: u8;
/// The ending byte of the _.data_ segment.
static mut __data_end: u8;
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
/// The starting byte of the _.bss_ (uninitialized data) segment.
static mut __bss_start: u8;
/// The ending byte of the _.bss_ (uninitialized data) segment.
static mut __bss_end: u8;
}
#[cold]
pub unsafe fn init() {
init_pat();
let flush_all = map_percpu(cpu_id, KernelMapper::lock_manually(cpu_id).get_mut().expect("expected KernelMapper not to be locked re-entrant in paging::init"));
flush_all.flush();
init_tcb(cpu_id)
}
pub unsafe fn init_ap(
cpu_id: usize,
bsp_table: &mut KernelMapper,
) -> usize {
init_pat();
{
let flush_all = map_percpu(cpu_id, bsp_table.get_mut().expect("KernelMapper locked re-entrant for AP"));
// The flush can be ignored as this is not the active table. See later make_current().
flush_all.ignore();
};
bsp_table.make_current();
init_tcb(cpu_id)
}
/// Page
src/arch/x86/start.rs
+6 -12
View File
@@ -18,7 +18,7 @@ use crate::gdt;
use crate::idt;
use crate::interrupt;
use crate::log::{self, info};
use crate::paging::{self, KernelMapper, TableKind};
use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKind};
/// Test of zero values in BSS.
static BSS_TEST_ZERO: usize = 0;
@@ -129,12 +129,11 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
args.areas_base as usize, args.areas_size as usize,
args.bootstrap_base as usize, args.bootstrap_size as usize,
);
// Initialize paging
let tcb_offset = paging::init(0);
paging::init();
// Set up GDT after paging with TLS
gdt::init_paging(0, tcb_offset, args.stack_base as usize + args.stack_size as usize);
gdt::init_paging(args.stack_base as usize + args.stack_size as usize);
// Set up IDT
idt::init_paging_bsp();
@@ -230,16 +229,11 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
idt::init();
// Initialize paging
let tcb_offset = {
use crate::paging::{PageMapper, PhysicalAddress};
use crate::rmm::FRAME_ALLOCATOR;
let mut mapper = KernelMapper::lock_for_manual_mapper(cpu_id, PageMapper::new(TableKind::Kernel, PhysicalAddress::new(bsp_table), FRAME_ALLOCATOR));
paging::init_ap(cpu_id, &mut mapper)
};
RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table));
paging::init();
// Set up GDT with TLS
gdt::init_paging(cpu_id as u32, tcb_offset, stack_end);
gdt::init_paging(stack_end);
// Set up IDT for AP
idt::init_paging_post_heap(false, cpu_id);
src/arch/x86_64/consts.rs
-10
View File
@@ -20,16 +20,6 @@
/// Size of kernel heap
pub const KERNEL_HEAP_SIZE: usize = 1 * 1024 * 1024; // 1 MB
/// Offset of temporary mapping for misc kernel bring-up actions
pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
/// Offset to kernel percpu variables
pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_TMP_MISC_OFFSET - PML4_SIZE;
pub const KERNEL_PERCPU_PML4: usize = (KERNEL_PERCPU_OFFSET & PML4_MASK)/PML4_SIZE;
/// Size of kernel percpu variables
pub const KERNEL_PERCPU_SHIFT: u8 = 16; // 2^16 = 64 KiB
pub const KERNEL_PERCPU_SIZE: usize = 1_usize << KERNEL_PERCPU_SHIFT;
/// Offset of physmap
// This needs to match RMM's PHYS_OFFSET
pub const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000;
src/arch/x86_64/gdt.rs
+117 -113
View File
@@ -1,10 +1,10 @@
//! Global descriptor table
use core::cell::UnsafeCell;
use core::convert::TryInto;
use core::mem;
use x86::segmentation::load_cs;
use crate::paging::{PAGE_SIZE, RmmA, RmmArch};
use x86::bits64::task::TaskStateSegment;
use x86::Ring;
use x86::dtables::{self, DescriptorTablePointer};
@@ -16,13 +16,11 @@ use super::cpuid::cpuid;
pub const GDT_NULL: usize = 0;
pub const GDT_KERNEL_CODE: usize = 1;
pub const GDT_KERNEL_DATA: usize = 2;
pub const GDT_KERNEL_KPCR: usize = 3;
pub const GDT_USER_CODE32_UNUSED: usize = 4;
pub const GDT_USER_DATA: usize = 5;
pub const GDT_USER_CODE: usize = 6;
pub const GDT_TSS: usize = 7;
pub const GDT_TSS_HIGH: usize = 8;
pub const GDT_CPU_ID_CONTAINER: usize = 9;
pub const GDT_USER_CODE32_UNUSED: usize = 3;
pub const GDT_USER_DATA: usize = 4;
pub const GDT_USER_CODE: usize = 5;
pub const GDT_TSS: usize = 6;
pub const GDT_TSS_HIGH: usize = 7;
pub const GDT_A_PRESENT: u8 = 1 << 7;
pub const GDT_A_RING_0: u8 = 0 << 5;
@@ -42,28 +40,24 @@ pub const GDT_F_PAGE_SIZE: u8 = 1 << 7;
pub const GDT_F_PROTECTED_MODE: u8 = 1 << 6;
pub const GDT_F_LONG_MODE: u8 = 1 << 5;
static mut INIT_GDT: [GdtEntry; 4] = [
static mut INIT_GDT: [GdtEntry; 3] = [
// Null
GdtEntry::new(0, 0, 0, 0),
// Kernel code
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE),
// Kernel data
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE),
// Kernel TLS
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE)
];
#[thread_local]
pub static GDT: UnsafeCell<[GdtEntry; 10]> = UnsafeCell::new([
// Later copied into the actual GDT with various fields set.
const BASE_GDT: [GdtEntry; 8] = [
// Null
GdtEntry::new(0, 0, 0, 0),
// Kernel code
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE),
// Kernel data
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE),
// Kernel TLS
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE),
// Dummy 32-bit user code - apparently necessary for SYSEXIT. We restrict it to ring 0 anyway.
// Dummy 32-bit user code - apparently necessary for SYSRET. We restrict it to ring 0 anyway.
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PROTECTED_MODE),
// User data
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE),
@@ -73,165 +67,175 @@ pub static GDT: UnsafeCell<[GdtEntry; 10]> = UnsafeCell::new([
GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0),
// TSS must be 16 bytes long, twice the normal size
GdtEntry::new(0, 0, 0, 0),
// Unused entry which stores the CPU ID. This is necessary for paranoid interrupts as they have
// no other way of determining it.
GdtEntry::new(0, 0, 0, 0),
]);
];
#[repr(C, align(16))]
#[repr(C, align(4096))]
pub struct ProcessorControlRegion {
// TODO: When both KASLR and KPTI are implemented, the PCR may need to be split into two pages,
// such that "secret" kernel addresses are only stored in the protected half.
pub tcb_end: usize,
pub user_rsp_tmp: usize,
pub tss: TssWrapper,
// TODO: The I/O permissions bitmap can require more than 8192 bytes of space.
pub tss: TaskStateSegment,
pub self_ref: usize,
// The GDT *must* be stored in the PCR! The paranoid interrupt handler, lacking a reliable way
// to correctly obtain GSBASE, uses SGDT to calculate the PCR offset.
pub gdt: [GdtEntry; 8],
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
}
// NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding
// inserted before and after the TSS, the main TSS structure will remain intact.
#[repr(C, align(16))]
pub struct TssWrapper(pub TaskStateSegment);
const _: () = {
if memoffset::offset_of!(ProcessorControlRegion, tss) % 16 != 0 {
panic!("PCR is incorrectly defined, TSS alignment is too small");
}
if memoffset::offset_of!(ProcessorControlRegion, gdt) % 8 != 0 {
panic!("PCR is incorrectly defined, GDT alignment is too small");
}
};
#[thread_local]
pub static KPCR: UnsafeCell<ProcessorControlRegion> = UnsafeCell::new(ProcessorControlRegion {
tcb_end: 0,
user_rsp_tmp: 0,
tss: TssWrapper(TaskStateSegment {
reserved: 0,
rsp: [0; 3],
reserved2: 0,
ist: [0; 7],
reserved3: 0,
reserved4: 0,
iomap_base: 0xFFFF
}),
});
pub unsafe fn pcr() -> *mut ProcessorControlRegion {
// Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that
// obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably
// microcoded) instructions.
let mut ret: *mut ProcessorControlRegion;
core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(memoffset::offset_of!(ProcessorControlRegion, self_ref)));
ret
}
#[cfg(feature = "pti")]
pub unsafe fn set_tss_stack(stack: usize) {
use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK};
(*KPCR.get()).tss.0.rsp[0] = (PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64;
core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64);
PTI_CONTEXT_STACK = stack;
}
#[cfg(not(feature = "pti"))]
pub unsafe fn set_tss_stack(stack: usize) {
(*KPCR.get()).tss.0.rsp[0] = stack as u64;
// TODO: If this increases performance, read gs:[offset] directly
core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write(stack as u64);
}
// Initialize GDT
// Initialize startup GDT
#[cold]
pub unsafe fn init() {
{
// Setup the initial GDT with TLS, so we can setup the TLS GDT (a little confusing)
// This means that each CPU will have its own GDT, but we only need to define it once as a thread local
// Before the kernel can remap itself, it needs to switch to a GDT it controls. Start with a
// minimal kernel-only GDT.
dtables::lgdt(&DescriptorTablePointer {
limit: (INIT_GDT.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: INIT_GDT.as_ptr() as *const SegmentDescriptor,
});
let limit = (INIT_GDT.len() * mem::size_of::<GdtEntry>() - 1)
.try_into()
.expect("initial GDT way too large");
let base = INIT_GDT.as_ptr() as *const SegmentDescriptor;
let init_gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
limit,
base,
};
// Load the initial GDT, before we have access to thread locals
dtables::lgdt(&init_gdtr);
}
// Load the segment descriptors
load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_KPCR as u16, Ring::Ring0));
load_segments();
}
#[cold]
unsafe fn load_segments() {
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::from_raw(0));
segmentation::load_es(SegmentSelector::from_raw(0));
segmentation::load_fs(SegmentSelector::from_raw(0));
// What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However,
// GSBASE is set later, and percpu is not used until gdt::init_paging().
segmentation::load_gs(SegmentSelector::from_raw(0));
}
/// Initialize GDT with TLS
pub unsafe fn init_paging(cpu_id: u32, tcb_offset: usize, stack_offset: usize) {
// Set temporary TLS segment to the self-pointer of the Thread Control Block.
x86::msr::wrmsr(x86::msr::IA32_GS_BASE, tcb_offset as u64);
/// Initialize GDT and PCR.
#[cold]
pub unsafe fn init_paging(stack_offset: usize) {
let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR");
let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion);
// Now that we have access to thread locals, begin by getting a pointer to the Processor
// Control Region.
let kpcr = KPCR.get();
pcr.self_ref = pcr as *mut ProcessorControlRegion as usize;
// Then, setup the AP's individual GDT
let limit = ((*GDT.get()).len() * mem::size_of::<GdtEntry>() - 1)
// Setup the GDT.
pcr.gdt = BASE_GDT;
let limit = (pcr.gdt.len() * mem::size_of::<GdtEntry>() - 1)
.try_into()
.expect("main GDT way too large");
let base = GDT.get() as *const SegmentDescriptor;
let base = pcr.gdt.as_ptr() as *const SegmentDescriptor;
let gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
limit,
base,
};
// Once we have fetched the real KPCR address, set the TLS segment to the TCB pointer there.
(*kpcr).tcb_end = (tcb_offset as *const usize).read();
pcr.tcb_end = init_percpu();
{
// We can now access our TSS, via the KPCR, which is a thread local
let tss = &(*kpcr).tss.0 as *const _ as usize as u64;
pcr.tss.iomap_base = 0xFFFF;
let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64;
let tss_lo = (tss & 0xFFFF_FFFF) as u32;
let tss_hi = (tss >> 32) as u32;
(*GDT.get())[GDT_TSS].set_offset(tss_lo);
(*GDT.get())[GDT_TSS].set_limit(mem::size_of::<TaskStateSegment>() as u32);
pcr.gdt[GDT_TSS].set_offset(tss_lo);
pcr.gdt[GDT_TSS].set_limit(mem::size_of::<TaskStateSegment>() as u32);
(&mut (*GDT.get())[GDT_TSS_HIGH] as *mut GdtEntry)
.cast::<u32>()
.write(tss_hi);
(&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry).cast::<u32>().write(tss_hi);
}
// And finally, populate the last GDT entry with the current CPU ID, to allow paranoid
// interrupt handlers to safely use TLS.
(&mut (*GDT.get())[GDT_CPU_ID_CONTAINER] as *mut GdtEntry)
.cast::<u32>()
.write(cpu_id);
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(stack_offset);
// Load the new GDT, which is correctly located in thread local storage.
dtables::lgdt(&gdtr);
// Ensure that GS always points to the KPCR in kernel space.
x86::msr::wrmsr(x86::msr::IA32_GS_BASE, kpcr as *mut _ as usize as u64);
// Inside kernel space, GS should _always_ point to the TSS. When leaving userspace, `swapgs`
// is called again, making the userspace GS always point to user data.
// Load segments again, possibly resetting FSBASE and GSBASE.
load_segments();
// Ensure that GSBASE always points to the PCR in kernel space.
x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64);
// While GSBASE points to the PCR in kernel space, userspace is free to set it to other values.
// Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is
// because entering usermode will entail executing the SWAPGS instruction.
x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0);
// Set the User TLS segment to zero, before we create any contexts and start scheduling.
// Set the userspace FSBASE to zero.
x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0);
// Reload the segment descriptors
load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
// NOTE: FS has already been updated while calling set_tcb.
// NOTE: We do not want to load GS again, since it has already been loaded into
// GDT_KERNEL_KPCR. Instead, we use the base MSR to allow for a 64-bit offset.
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(stack_offset);
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
let has_fsgsbase = cpuid().map_or(false, |cpuid| {
let cpu_supports_fsgsbase = cpuid().map_or(false, |cpuid| {
cpuid.get_extended_feature_info().map_or(false, |extended_features| {
extended_features.has_fsgsbase()
})
});
if cfg!(feature = "x86_fsgsbase") {
assert!(has_fsgsbase, "running kernel with features not supported by the current CPU");
}
assert!(cpu_supports_fsgsbase, "running kernel with features not supported by the current CPU");
if has_fsgsbase {
x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE);
}
}
/// Copy tdata, clear tbss, calculate TCB end pointer
#[cold]
unsafe fn init_percpu() -> usize {
use crate::kernel_executable_offsets::*;
let size = __tbss_end() - __tdata_start();
assert_eq!(size % PAGE_SIZE, 0);
let tbss_offset = __tbss_start() - __tdata_start();
let base_frame = crate::memory::allocate_frames(size / PAGE_SIZE).expect("failed to allocate percpu memory");
let base = RmmA::phys_to_virt(base_frame.start_address());
let tls_end = base.data() + size;
core::ptr::copy_nonoverlapping(__tdata_start() as *const u8, base.data() as *mut u8, tbss_offset);
core::ptr::write_bytes((base.data() + tbss_offset) as *mut u8, 0, size - tbss_offset);
tls_end
}
#[derive(Copy, Clone, Debug)]
#[repr(packed)]
pub struct GdtEntry {
src/arch/x86_64/idt.rs
+1 -1
View File
@@ -176,7 +176,7 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) {
let address = base_address.data() + BACKUP_STACK_SIZE;
// Put them in the 1st entry of the IST.
(*crate::gdt::KPCR.get()).tss.0.ist[usize::from(index - 1)] = address as u64;
(*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64;
index
};
src/arch/x86_64/interrupt/handler.rs
+68 -87
View File
@@ -258,6 +258,7 @@ macro_rules! pop_preserved {
" };
}
macro_rules! swapgs_iff_ring3_fast {
// TODO: Spectre V1: LFENCE?
() => { "
// Check whether the last two bits RSP+8 (code segment) are equal to zero.
test QWORD PTR [rsp + 8], 0x3
@@ -268,6 +269,7 @@ macro_rules! swapgs_iff_ring3_fast {
" };
}
macro_rules! swapgs_iff_ring3_fast_errorcode {
// TODO: Spectre V1: LFENCE?
() => { "
test QWORD PTR [rsp + 16], 0x3
jz 1f
@@ -276,109 +278,87 @@ macro_rules! swapgs_iff_ring3_fast_errorcode {
" };
}
#[cfg(feature = "x86_fsgsbase")]
macro_rules! save_gsbase_paranoid {
() => { "
// Unused: {IA32_GS_BASE}
rdgsbase rax
push rax
" }
#[cfg(feature = "x86_fsbase")]
macro_rules! read_gsbase_into_rdx {
() => { "rdgsbase rdx;" }
}
#[cfg(feature = "x86_fsgsbase")]
macro_rules! restore_gsbase_paranoid {
() => { "
// Unused: {IA32_GS_BASE}
pop rax
wrgsbase rax
" }
}
#[cfg(not(feature = "x86_fsgsbase"))]
macro_rules! save_gsbase_paranoid {
#[cfg(not(feature = "x86_fsbase"))]
macro_rules! read_gsbase_into_rdx {
() => { "
mov ecx, {IA32_GS_BASE}
rdmsr
shl rdx, 32
or rax, rdx
push rax
" }
}
#[cfg(not(feature = "x86_fsgsbase"))]
macro_rules! restore_gsbase_paranoid {
() => { "
pop rdx
mov ecx, {IA32_GS_BASE}
mov eax, edx
shr rdx, 32
wrmsr
or rdx, rax
" }
}
#[cfg(feature = "x86_fsgsbase")]
macro_rules! set_gsbase_paranoid {
() => { "
// Unused: {IA32_GS_BASE}
wrgsbase rdx
" }
}
#[cfg(not(feature = "x86_fsgsbase"))]
macro_rules! set_gsbase_paranoid {
() => { "
mov ecx, {IA32_GS_BASE}
mov eax, edx
shr rdx, 32
wrmsr
" }
}
macro_rules! save_and_set_gsbase_paranoid {
// For paranoid interrupt entries, we have to be extremely careful with how we use IA32_GS_BASE
// and IA32_KERNEL_GS_BASE. If FSGSBASE is enabled, then we have no way to differentiate these
// two, as paranoid interrupts (e.g. NMIs) can occur even in kernel mode. In fact, they can
// even occur within another IRQ, so we cannot check the the privilege level via the stack.
macro_rules! conditional_swapgs_paranoid {
// For regular interrupt handlers and the syscall handler, managing IA32_GS_BASE and
// IA32_KERNEL_GS_BASE (the "GSBASE registers") is more or less trivial when using the SWAPGS
// instruction.
//
// What we do instead, is using a special entry in the GDT, since we know that the GDT will
// always be thread local, as it contains the TSS. This gives us more than 32 bits to work
// with, which already is the largest x2APIC ID that an x86 CPU can handle. Luckily we can also
// use the stack, even though there might be interrupts in between.
// The syscall handler simply runs SWAPGS, as syscalls can only originate from usermode,
// whereas interrupt handlers conditionally SWAPGS unless the interrupt was triggered from
// kernel mode, in which case the "swap state" is already valid, and there is no need to
// SWAPGS.
//
// TODO: Linux uses the Interrupt Stack Table to figure out which NMIs were nested. Perhaps
// this could be done here, because if nested (sp > initial_sp), that means the NMI could not
// have come from userspace. But then, knowing the initial sp would somehow have to involve
// percpu, which brings us back to square one. But it might be useful if we would allow faults
// in NMIs. If we do detect a nested interrupt, then we can perform the iretq procedure
// ourselves, so that the newly nested NMI still blocks additional interrupts while still
// returning to the previously (faulting) NMI. See https://lwn.net/Articles/484932/, although I
// think the solution becomes a bit simpler when we cannot longer rely on GSBASE anymore.
// Handling GSBASE correctly for paranoid interrupts however, is not as simple. NMIs can occur
// between the check of whether an interrupt came from usermode, and the actual SWAPGS
// instruction. #DB can also be triggered inside of a kernel interrupt handler, due to
// breakpoints, even though setting up such breakpoints in the first place, is not yet
// supported by the kernel.
//
// Luckily, the GDT always resides in the PCR (at least after init_paging, but there are no
// interrupt handlers set up before that), allowing GSBASE to be calculated relatively cheaply.
// Out of the two GSBASE registers, at least one must be *the* kernel GSBASE, allowing for a
// simple conditional SWAPGS.
//
// (An alternative to conditionally executing SWAPGS, would be to save and restore GSBASE via
// e.g. the stack. That would nonetheless require saving and restoring both GSBASE registers,
// if the interrupt handler should be allowed to context switch, which the current #DB handler
// may do.)
//
// TODO: Handle nested NMIs like Linux does (https://lwn.net/Articles/484932/)?.
() => { concat!(
save_gsbase_paranoid!(),
// Allocate stack space for 8 bytes GDT base and 2 bytes size (ignored).
"sub rsp, 16\n",
// Set it to the GDT base.
"sgdt [rsp + 6]\n",
// Get the base pointer
// Put the GDT base pointer in RDI.
"
mov rax, [rsp + 8]
sub rsp, 16
sgdt [rsp + 6]
mov rdi, [rsp + 8]
add rsp, 16
",
// Load the lower 32 bits of that GDT entry.
"mov edx, [rax + {gdt_cpu_id_offset}]\n",
// Calculate the percpu offset.
// Calculate the PCR address by subtracting the offset of the GDT in the PCR struct.
"sub rdi, {PCR_GDT_OFFSET};",
// Read the current IA32_GS_BASE value into RDX.
read_gsbase_into_rdx!(),
// If they were not equal, the PCR address must instead be in IA32_KERNEL_GS_BASE,
// requiring a SWAPGS. GSBASE needs to be swapped back, so store the same flag in RBX.
// TODO: Spectre V1: LFENCE?
"
mov rbx, {KERNEL_PERCPU_OFFSET}
shl rdx, {KERNEL_PERCPU_SHIFT}
add rdx, rbx
cmp rdx, rdi
sete bl
je 1f
swapgs
1:
",
// Set GSBASE to RAX accordingly
set_gsbase_paranoid!(),
) }
}
macro_rules! conditional_swapgs_back_paranoid {
() => { "
test ebx, ebx
jnz 1f
swapgs
1:
" }
}
macro_rules! nop {
() => { "
// Unused: {IA32_GS_BASE} {KERNEL_PERCPU_OFFSET} {KERNEL_PERCPU_SHIFT} {gdt_cpu_id_offset}
// Unused: {IA32_GS_BASE} {PCR_GDT_OFFSET}
" }
}
@@ -399,7 +379,10 @@ macro_rules! interrupt_stack {
_guard = $crate::ptrace::set_process_regs($stack);
}
// TODO: Force the declarations to specify unsafe?
// TODO: Force the declarations to specify unsafe? For example, accessing a global
// core::cell::Cell is safe when running at the "bottom level" (interrupt from
// userspace or prior to entering userspace), but UB if simultaneously accessed
// from an interrupt.
#[allow(unused_unsafe)]
unsafe {
@@ -442,10 +425,8 @@ macro_rules! interrupt_stack {
inner = sym inner,
IA32_GS_BASE = const(x86::msr::IA32_GS_BASE),
KERNEL_PERCPU_SHIFT = const(crate::KERNEL_PERCPU_SHIFT),
KERNEL_PERCPU_OFFSET = const(crate::KERNEL_PERCPU_OFFSET),
gdt_cpu_id_offset = const(crate::gdt::GDT_CPU_ID_CONTAINER * core::mem::size_of::<crate::gdt::GdtEntry>()),
PCR_GDT_OFFSET = const(memoffset::offset_of!(crate::gdt::ProcessorControlRegion, gdt)),
options(noreturn),
@@ -453,7 +434,7 @@ macro_rules! interrupt_stack {
}
};
($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, swapgs_iff_ring3_fast!, nop!, nop!, swapgs_iff_ring3_fast!, is_paranoid: false, |$stack| $code); };
($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, save_and_set_gsbase_paranoid!, restore_gsbase_paranoid!, nop!, is_paranoid: true, |$stack| $code); }
($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, conditional_swapgs_paranoid!, conditional_swapgs_back_paranoid!, nop!, is_paranoid: true, |$stack| $code); }
}
#[macro_export]
src/arch/x86_64/paging/mod.rs
+4 -136
View File
@@ -1,11 +1,8 @@
//! # Paging
//! Some code was borrowed from [Phil Opp's Blog](http://os.phil-opp.com/modifying-page-tables.html)
use core::{mem, ptr};
use x86::msr;
use self::mapper::PageFlushAll;
pub use rmm::{
Arch as RmmArch,
Flusher,
@@ -29,6 +26,7 @@ pub const ENTRY_COUNT: usize = RmmA::PAGE_ENTRIES;
pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE;
/// Setup page attribute table
#[cold]
unsafe fn init_pat() {
let uncacheable = 0;
let write_combining = 1;
@@ -60,124 +58,10 @@ unsafe fn init_pat() {
);
}
/// Map percpu
unsafe fn map_percpu(cpu_id: usize, mapper: &mut PageMapper) -> PageFlushAll<RmmA> {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
}
let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
let end = start + size;
let mut flush_all = PageFlushAll::new();
let start_page = Page::containing_address(VirtualAddress::new(start));
let end_page = Page::containing_address(VirtualAddress::new(end - 1));
for page in Page::range_inclusive(start_page, end_page) {
let result = mapper.map(
page.start_address(),
PageFlags::new().write(true).global(cfg!(not(feature = "pti"))),
)
.expect("failed to allocate page table frames while mapping percpu");
flush_all.consume(result);
}
flush_all
}
/// Copy tdata, clear tbss, set TCB self pointer
unsafe fn init_tcb(cpu_id: usize) -> usize {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
}
let tcb_offset;
{
let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
let tbss_offset = &__tbss_start as *const _ as usize - &__tdata_start as *const _ as usize;
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
let end = start + size;
tcb_offset = end - mem::size_of::<usize>();
ptr::copy(&__tdata_start as *const u8, start as *mut u8, tbss_offset);
ptr::write_bytes((start + tbss_offset) as *mut u8, 0, size - tbss_offset);
*(tcb_offset as *mut usize) = end;
}
tcb_offset
}
/// Initialize paging
///
/// Returns page table and thread control block offset
pub unsafe fn init(
cpu_id: usize,
) -> usize {
extern "C" {
/// The starting byte of the text (code) data segment.
static mut __text_start: u8;
/// The ending byte of the text (code) data segment.
static mut __text_end: u8;
/// The starting byte of the _.rodata_ (read-only data) segment.
static mut __rodata_start: u8;
/// The ending byte of the _.rodata_ (read-only data) segment.
static mut __rodata_end: u8;
/// The starting byte of the _.data_ segment.
static mut __data_start: u8;
/// The ending byte of the _.data_ segment.
static mut __data_end: u8;
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
/// The starting byte of the _.bss_ (uninitialized data) segment.
static mut __bss_start: u8;
/// The ending byte of the _.bss_ (uninitialized data) segment.
static mut __bss_end: u8;
}
/// Initialize PAT
#[cold]
pub unsafe fn init() {
init_pat();
let flush_all = map_percpu(cpu_id, KernelMapper::lock_manually(cpu_id).get_mut().expect("expected KernelMapper not to be locked re-entrant in paging::init"));
flush_all.flush();
init_tcb(cpu_id)
}
pub unsafe fn init_ap(
cpu_id: usize,
bsp_table: &mut KernelMapper,
) -> usize {
init_pat();
{
let flush_all = map_percpu(cpu_id, bsp_table.get_mut().expect("KernelMapper locked re-entrant for AP"));
// The flush can be ignored as this is not the active table. See later make_current().
flush_all.ignore();
};
bsp_table.make_current();
init_tcb(cpu_id)
}
/// Page
@@ -191,22 +75,6 @@ impl Page {
VirtualAddress::new(self.number * PAGE_SIZE)
}
pub fn p4_index(self) -> usize {
(self.number >> 27) & 0o777
}
pub fn p3_index(self) -> usize {
(self.number >> 18) & 0o777
}
pub fn p2_index(self) -> usize {
(self.number >> 9) & 0o777
}
pub fn p1_index(self) -> usize {
self.number & 0o777
}
pub fn containing_address(address: VirtualAddress) -> Page {
//TODO assert!(address.data() < 0x0000_8000_0000_0000 || address.data() >= 0xffff_8000_0000_0000,
// "invalid address: 0x{:x}", address.data());
src/arch/x86_64/rmm.rs
+3 -21
View File
@@ -24,17 +24,6 @@ use spin::Mutex;
use super::CurrentRmmArch as RmmA;
extern "C" {
/// The starting byte of the text (code) data segment.
static mut __text_start: u8;
/// The ending byte of the text (code) data segment.
static mut __text_end: u8;
/// The starting byte of the _.rodata_ (read-only data) segment.
static mut __rodata_start: u8;
/// The ending byte of the _.rodata_ (read-only data) segment.
static mut __rodata_end: u8;
}
// Keep synced with OsMemoryKind in bootloader
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[repr(u64)]
@@ -55,20 +44,13 @@ pub struct BootloaderMemoryEntry {
}
unsafe fn page_flags<A: Arch>(virt: VirtualAddress) -> PageFlags<A> {
use crate::kernel_executable_offsets::*;
let virt_addr = virt.data();
// Test for being inside a region
macro_rules! in_section {
($n: ident) => {
virt_addr >= &concat_idents!(__, $n, _start) as *const u8 as usize
&& virt_addr < &concat_idents!(__, $n, _end) as *const u8 as usize
};
}
if in_section!(text) {
if virt_addr >= __text_start() && virt_addr < __text_end() {
// Remap text read-only, execute
PageFlags::new().execute(true)
} else if in_section!(rodata) {
} else if virt_addr >= __rodata_start() && virt_addr < __rodata_end() {
// Remap rodata read-only, no execute
PageFlags::new()
} else {
src/arch/x86_64/start.rs
+7 -12
View File
@@ -20,7 +20,7 @@ use crate::idt;
use crate::interrupt;
use crate::misc;
use crate::log::{self, info};
use crate::paging::{self, KernelMapper, TableKind};
use crate::paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind};
/// Test of zero values in BSS.
static BSS_TEST_ZERO: usize = 0;
@@ -132,11 +132,11 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
args.bootstrap_base as usize, args.bootstrap_size as usize,
);
// Initialize paging
let tcb_offset = paging::init(0);
// Initialize PAT
paging::init();
// Set up GDT after paging with TLS
gdt::init_paging(0, tcb_offset, args.stack_base as usize + args.stack_size as usize);
gdt::init_paging(args.stack_base as usize + args.stack_size as usize);
// Set up IDT
idt::init_paging_bsp();
@@ -235,16 +235,11 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
idt::init();
// Initialize paging
let tcb_offset = {
use crate::paging::{PageMapper, PhysicalAddress};
use crate::rmm::FRAME_ALLOCATOR;
let mut mapper = KernelMapper::lock_for_manual_mapper(cpu_id, PageMapper::new(TableKind::Kernel, PhysicalAddress::new(bsp_table), FRAME_ALLOCATOR));
paging::init_ap(cpu_id, &mut mapper)
};
RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table));
paging::init();
// Set up GDT with TLS
gdt::init_paging(cpu_id as u32, tcb_offset, stack_end);
gdt::init_paging(stack_end);
// Set up IDT for AP
idt::init_paging_post_heap(false, cpu_id);
src/context/arch/x86.rs
+7 -5
View File
@@ -4,7 +4,7 @@ use core::sync::atomic::AtomicBool;
use alloc::sync::Arc;
use crate::{push_scratch, pop_scratch};
use crate::gdt::{GDT, GDT_USER_FS, GDT_USER_GS};
use crate::gdt::{pcr, GDT_USER_FS, GDT_USER_GS};
use crate::interrupt::handler::ScratchRegisters;
use crate::paging::{RmmA, RmmArch, TableKind};
use crate::syscall::FloatRegisters;
@@ -137,10 +137,12 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
);
{
prev.arch.fsbase = GDT[GDT_USER_FS].offset() as usize;
GDT[GDT_USER_FS].set_offset(next.arch.fsbase as u32);
prev.arch.gsbase = GDT[GDT_USER_GS].offset() as usize;
GDT[GDT_USER_GS].set_offset(next.arch.gsbase as u32);
let gdt = &mut (&mut *pcr()).gdt;
prev.arch.fsbase = gdt[GDT_USER_FS].offset() as usize;
gdt[GDT_USER_FS].set_offset(next.arch.fsbase as u32);
prev.arch.gsbase = gdt[GDT_USER_GS].offset() as usize;
gdt[GDT_USER_GS].set_offset(next.arch.gsbase as u32);
}
match next.addr_space {
src/context/arch/x86_64.rs
+3 -5
View File
@@ -154,17 +154,15 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
wrgsbase(next.arch.gsbase as u64);
swapgs();
} else {
prev.arch.fsbase = msr::rdmsr(msr::IA32_FS_BASE) as usize;
msr::wrmsr(msr::IA32_FS_BASE, next.arch.fsbase as u64);
prev.arch.gsbase = msr::rdmsr(msr::IA32_KERNEL_GSBASE) as usize;
msr::wrmsr(msr::IA32_KERNEL_GSBASE, next.arch.gsbase as u64);
}
}
match next.addr_space {
// Since Arc is essentially just wraps a pointer, in this case a regular pointer (as
// opposed to dyn or slice fat pointers), and NonNull optimization exists, map_or will
// hopefully be optimized down to checking prev and next pointers, as next cannot be null.
// Since Arc essentially just wraps a pointer, in this case a regular pointer (as opposed
// to dyn or slice fat pointers), and NonNull optimization exists, map_or will hopefully be
// optimized down to checking prev and next pointers, as next cannot be null.
Some(ref next_space) => if prev.addr_space.as_ref().map_or(true, |prev_space| !Arc::ptr_eq(prev_space, next_space)) {
// Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A
// recently called yield and is now here about to switch back. Meanwhile, B is
src/context/memory.rs
-3
View File
@@ -960,9 +960,6 @@ pub fn setup_new_utable() -> Result<Table> {
// Copy physmap mapping
copy_mapping(crate::PHYS_PML4);
// Copy kernel percpu (similar to TLS) mapping.
copy_mapping(crate::KERNEL_PERCPU_PML4);
}
Ok(Table {
src/lib.rs
+20
View File
@@ -279,3 +279,23 @@ pub extern fn ksignal(signal: usize) {
syscall::exit(signal & 0x7F);
});
}
// TODO: Use this macro on aarch64 too.
macro_rules! linker_offsets(
($($name:ident),*) => {
$(
#[inline]
pub fn $name() -> usize {
extern "C" {
// TODO: UnsafeCell?
static $name: u8;
}
unsafe { &$name as *const u8 as usize }
}
)*
}
);
pub mod kernel_executable_offsets {
linker_offsets!(__text_start, __text_end, __rodata_start, __rodata_end, __data_start, __data_end, __bss_start, __bss_end, __tdata_start, __tdata_end, __tbss_start, __tbss_end);
}
src/scheme/proc.rs
+4 -4
View File
@@ -430,8 +430,8 @@ impl ProcScheme {
let (fsbase, gsbase) = if info.pid == context::context_id() {
unsafe {
(
crate::gdt::GDT[crate::gdt::GDT_USER_FS].offset() as u64,
crate::gdt::GDT[crate::gdt::GDT_USER_GS].offset() as u64
(&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_FS].offset() as u64,
(&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].offset() as u64
)
}
} else {
@@ -501,8 +501,8 @@ impl ProcScheme {
if info.pid == context::context_id() {
unsafe {
crate::gdt::GDT[crate::gdt::GDT_USER_FS].set_offset(regs.fsbase);
crate::gdt::GDT[crate::gdt::GDT_USER_GS].set_offset(regs.gsbase);
(&mut *crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_FS].set_offset(regs.fsbase);
(&mut *crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].set_offset(regs.gsbase);
match context::contexts().current().ok_or(Error::new(ESRCH))?.write().arch {
ref mut arch => {
targets/i686-unknown-kernel.json
+1 -1
View File
@@ -3,7 +3,7 @@
"target-endian": "little",
"target-pointer-width": "32",
"target-c-int-width": "32",
"data-layout": "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128",
"data-layout": "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S64",
"arch": "x86",
"os": "none",
"env": "",
targets/x86_64-unknown-kernel.json
+1 -1
View File
@@ -3,7 +3,7 @@
"target-endian": "little",
"target-pointer-width": "64",
"target-c-int-width": "32",
"data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128",
"data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S64",
"arch": "x86_64",
"os": "none",
"env": "",