From 081efd6cfa1603d8609d32986178d521d7bd71f5 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 17 Sep 2025 20:57:47 +0200 Subject: [PATCH] Allocate IDT for APs on the BSP and statically allocate BSP IDT This allows them to be immediately installed by kstart/kstart_ap without having to wait for the page tables to be set up correctly. This removes the initial IDT. --- src/acpi/madt/arch/x86.rs | 3 + src/arch/x86_shared/idt.rs | 295 ++++++++++++++++++----------------- src/arch/x86_shared/start.rs | 17 +- 3 files changed, 160 insertions(+), 155 deletions(-) diff --git a/src/acpi/madt/arch/x86.rs b/src/acpi/madt/arch/x86.rs index dd28500803..d170de94d8 100644 --- a/src/acpi/madt/arch/x86.rs +++ b/src/acpi/madt/arch/x86.rs @@ -77,10 +77,13 @@ pub(super) fn init(madt: Madt) { let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end); + let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id); + let args = KernelArgsAp { cpu_id, page_table: page_table_physaddr, pcr_ptr, + idt_ptr, }; let ap_ready = (TRAMPOLINE + 8) as *mut u64; diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 563c682f48..9be5e928eb 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -14,16 +14,15 @@ use x86::{ #[cfg(target_arch = "x86_64")] use crate::interrupt::irq::{__generic_interrupts_end, __generic_interrupts_start}; -use crate::{cpu_set::LogicalCpuId, interrupt::*, ipi::IpiKind}; +use crate::{cpu_set::LogicalCpuId, interrupt::*, ipi::IpiKind, memory::PAGE_SIZE}; use spin::RwLock; -static INIT_IDT: SyncUnsafeCell<[IdtEntry; 32]> = SyncUnsafeCell::new([IdtEntry::new(); 32]); - #[repr(C)] -struct Idt { +pub struct Idt { entries: [IdtEntry; 256], reservations: [AtomicU32; 8], + backup_stack_end: usize, } impl Idt { @@ -31,6 +30,7 @@ impl Idt { Self { entries: [IdtEntry::new(); 256], reservations: [const { AtomicU32::new(0) }; 8], + backup_stack_end: 0, } } @@ -61,6 +61,9 @@ impl Idt { } } +// Allocate 64 KiB of stack space for the backup stack. +const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4; + static INIT_BSP_IDT: SyncUnsafeCell = SyncUnsafeCell::new(Idt::new()); // TODO: VecMap? @@ -69,11 +72,20 @@ static IDTS: RwLock> = #[inline] pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool { + if cpu_id == LogicalCpuId::BSP { + return unsafe { (&*INIT_BSP_IDT.get()).is_reserved(index) }; + } + IDTS.read().get(&cpu_id).unwrap().is_reserved(index) } #[inline] pub fn set_reserved(cpu_id: LogicalCpuId, index: u8, reserved: bool) { + if cpu_id == LogicalCpuId::BSP { + unsafe { (&*INIT_BSP_IDT.get()).set_reserved(index, reserved) }; + return; + } + IDTS.read() .get(&cpu_id) .unwrap() @@ -99,14 +111,6 @@ macro_rules! use_default_irqs( }} ); -pub unsafe fn init() { - unsafe { - let idt = &mut *INIT_IDT.get(); - set_exceptions(idt); - dtables::lidt(&DescriptorTablePointer::new(&idt)); - } -} - fn set_exceptions(idt: &mut [IdtEntry]) { // Set up exceptions idt[0].set_func(exception::divide_by_zero); @@ -136,149 +140,154 @@ fn set_exceptions(idt: &mut [IdtEntry]) { // 31 reserved } -/// Initialize the IDT for a processor -pub unsafe fn init_paging_post_heap(cpu_id: LogicalCpuId) { - let mut idts_btree = IDTS.write(); - - if cpu_id == LogicalCpuId::BSP { - idts_btree.insert(cpu_id, unsafe { &mut *INIT_BSP_IDT.get() }); - } else { - let idt = idts_btree - .entry(cpu_id) - .or_insert_with(|| Box::leak(Box::new(Idt::new()))); - unsafe { - init_generic(cpu_id, idt); - } - } -} - /// Initializes a fully functional IDT for use before it be moved into the map. This is ONLY called /// on the BSP, since the kernel heap is ready for the APs. -pub unsafe fn init_paging_bsp() { +pub unsafe fn init_bsp() { + #[repr(C, packed(4096))] + struct BackupStack([u8; BACKUP_STACK_SIZE]); + + static INIT_BSP_BACKUP_STACK: SyncUnsafeCell = + SyncUnsafeCell::new(BackupStack([0; BACKUP_STACK_SIZE])); + unsafe { - init_generic(LogicalCpuId::BSP, &mut *INIT_BSP_IDT.get()); + init_generic( + LogicalCpuId::BSP, + &mut *INIT_BSP_IDT.get(), + INIT_BSP_BACKUP_STACK.get().addr() + BACKUP_STACK_SIZE, + ); + + install_idt(&mut *INIT_BSP_IDT.get()); } } +pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt { + let mut idts_btree = IDTS.write(); + + let idt = idts_btree + .entry(cpu_id) + .or_insert_with(|| Box::leak(Box::new(Idt::new()))); + + use crate::paging::{RmmA, RmmArch}; + let frames = crate::memory::allocate_p2frame(4) + .expect("failed to allocate pages for backup interrupt stack"); + + // Physical pages are mapped linearly. So is the linearly mapped virtual memory. + let base_address = unsafe { RmmA::phys_to_virt(frames.base()) }; + + // Stack always grows downwards. + let backup_stack_end = base_address.data() + BACKUP_STACK_SIZE; + + init_generic(cpu_id, idt, backup_stack_end); + + *idt +} + +const BACKUP_IST: u8 = 1; + /// Initializes an IDT for any type of processor. -unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) { +fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) { + let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations); + + set_exceptions(current_idt); + + // We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate + // stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are + // considered the most fatal, especially Double Faults which are caused by errors __when + // accessing the system IDT__. If that goes wrong, then kernel memory may be partially + // corrupt, and we want a separate stack. + // + // Note that each CPU has its own "backup interrupt stack". + idt.backup_stack_end = backup_stack_end; + current_idt[2].set_ist(BACKUP_IST); + current_idt[8].set_ist(BACKUP_IST); + current_idt[18].set_ist(BACKUP_IST); + + #[cfg(target_arch = "x86_64")] + assert_eq!( + __generic_interrupts_end as usize - __generic_interrupts_start as usize, + 224 * 8 + ); + + #[cfg(target_arch = "x86_64")] + for i in 0..224 { + current_idt[i + 32] + .set_func(unsafe { mem::transmute(__generic_interrupts_start as usize + i * 8) }); + } + + // reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions + *current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF; + + if cpu_id == LogicalCpuId::BSP { + // Set up IRQs + current_idt[32].set_func(irq::pit_stack); + current_idt[33].set_func(irq::keyboard); + current_idt[34].set_func(irq::cascade); + current_idt[35].set_func(irq::com2); + current_idt[36].set_func(irq::com1); + current_idt[37].set_func(irq::lpt2); + current_idt[38].set_func(irq::floppy); + current_idt[39].set_func(irq::lpt1); + current_idt[40].set_func(irq::rtc); + current_idt[41].set_func(irq::pci1); + current_idt[42].set_func(irq::pci2); + current_idt[43].set_func(irq::pci3); + current_idt[44].set_func(irq::mouse); + current_idt[45].set_func(irq::fpu); + current_idt[46].set_func(irq::ata1); + current_idt[47].set_func(irq::ata2); + current_idt[48].set_func(irq::lapic_timer); + current_idt[49].set_func(irq::lapic_error); + + // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error. + *current_reservations[1].get_mut() |= 0x0003_FFFF; + } else { + // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU + current_idt[49].set_func(irq::lapic_error); + + // reserve bit 49 + *current_reservations[1].get_mut() |= 1 << 17; + } + + #[cfg(target_arch = "x86")] + use_default_irqs!(current_idt); + + // Set IPI handlers + current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup); + current_idt[IpiKind::Switch as usize].set_func(ipi::switch); + current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb); + current_idt[IpiKind::Pit as usize].set_func(ipi::pit); + idt.set_reserved_mut(IpiKind::Wakeup as u8, true); + idt.set_reserved_mut(IpiKind::Switch as u8, true); + idt.set_reserved_mut(IpiKind::Tlb as u8, true); + idt.set_reserved_mut(IpiKind::Pit as u8, true); + + #[cfg(target_arch = "x86")] + { + let current_idt = &mut idt.entries; + // Set syscall function + current_idt[0x80].set_func(syscall::syscall); + current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); + idt.set_reserved_mut(0x80, true); + } + + #[cfg(feature = "profiling")] + crate::profiling::maybe_setup_timer(idt, cpu_id); +} + +pub unsafe fn install_idt(idt_ptr: *mut Idt) { unsafe { - let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations); + let idt = &mut *idt_ptr; + + #[cfg(target_arch = "x86_64")] // TODO: x86 + { + (*crate::gdt::pcr()).tss.ist[usize::from(BACKUP_IST - 1)] = idt.backup_stack_end as u64; + } let idtr: DescriptorTablePointer = DescriptorTablePointer { - limit: (current_idt.len() * mem::size_of::() - 1) as u16, - base: current_idt.as_ptr() as *const X86IdtEntry, + limit: (idt.entries.len() * mem::size_of::() - 1) as u16, + base: idt.entries.as_ptr() as *const X86IdtEntry, }; - let backup_ist = { - // We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate - // stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are - // considered the most fatal, especially Double Faults which are caused by errors __when - // accessing the system IDT__. If that goes wrong, then kernel memory may be partially - // corrupt, and we want a separate stack. - // - // Note that each CPU has its own "backup interrupt stack". - let index = 1_u8; - - // Put them in the 1st entry of the IST. - #[cfg(target_arch = "x86_64")] // TODO: x86 - { - use crate::paging::PAGE_SIZE; - // Allocate 64 KiB of stack space for the backup stack. - const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4; - let frames = crate::memory::allocate_p2frame(4) - .expect("failed to allocate pages for backup interrupt stack"); - - use crate::paging::{RmmA, RmmArch}; - - // Physical pages are mapped linearly. So is the linearly mapped virtual memory. - let base_address = RmmA::phys_to_virt(frames.base()); - - // Stack always grows downwards. - let address = base_address.data() + BACKUP_STACK_SIZE; - - (*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64; - } - - index - }; - - set_exceptions(current_idt); - current_idt[2].set_ist(backup_ist); - current_idt[8].set_ist(backup_ist); - current_idt[18].set_ist(backup_ist); - - #[cfg(target_arch = "x86_64")] - assert_eq!( - __generic_interrupts_end as usize - __generic_interrupts_start as usize, - 224 * 8 - ); - - #[cfg(target_arch = "x86_64")] - for i in 0..224 { - current_idt[i + 32] - .set_func(mem::transmute(__generic_interrupts_start as usize + i * 8)); - } - - // reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions - *current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF; - - if cpu_id == LogicalCpuId::BSP { - // Set up IRQs - current_idt[32].set_func(irq::pit_stack); - current_idt[33].set_func(irq::keyboard); - current_idt[34].set_func(irq::cascade); - current_idt[35].set_func(irq::com2); - current_idt[36].set_func(irq::com1); - current_idt[37].set_func(irq::lpt2); - current_idt[38].set_func(irq::floppy); - current_idt[39].set_func(irq::lpt1); - current_idt[40].set_func(irq::rtc); - current_idt[41].set_func(irq::pci1); - current_idt[42].set_func(irq::pci2); - current_idt[43].set_func(irq::pci3); - current_idt[44].set_func(irq::mouse); - current_idt[45].set_func(irq::fpu); - current_idt[46].set_func(irq::ata1); - current_idt[47].set_func(irq::ata2); - current_idt[48].set_func(irq::lapic_timer); - current_idt[49].set_func(irq::lapic_error); - - // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error. - *current_reservations[1].get_mut() |= 0x0003_FFFF; - } else { - // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU - current_idt[49].set_func(irq::lapic_error); - - // reserve bit 49 - *current_reservations[1].get_mut() |= 1 << 17; - } - - #[cfg(target_arch = "x86")] - use_default_irqs!(current_idt); - - // Set IPI handlers - current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup); - current_idt[IpiKind::Switch as usize].set_func(ipi::switch); - current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb); - current_idt[IpiKind::Pit as usize].set_func(ipi::pit); - idt.set_reserved_mut(IpiKind::Wakeup as u8, true); - idt.set_reserved_mut(IpiKind::Switch as u8, true); - idt.set_reserved_mut(IpiKind::Tlb as u8, true); - idt.set_reserved_mut(IpiKind::Pit as u8, true); - - #[cfg(target_arch = "x86")] - { - let current_idt = &mut idt.entries; - // Set syscall function - current_idt[0x80].set_func(syscall::syscall); - current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT); - idt.set_reserved_mut(0x80, true); - } - - #[cfg(feature = "profiling")] - crate::profiling::maybe_setup_timer(idt, cpu_id); - dtables::lidt(&idtr); } } diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 30cfe73f14..53a6c91962 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -119,8 +119,8 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { // Set up GDT gdt::init_bsp(args.stack_base as usize + args.stack_size as usize); - // Set up IDT before paging - idt::init(); + // Set up IDT + idt::init_bsp(); // Initialize RMM register_bootloader_areas(args.areas_base as usize, args.areas_size as usize); @@ -157,9 +157,6 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { // Initialize paging paging::init(); - // Set up IDT - idt::init_paging_bsp(); - #[cfg(target_arch = "x86_64")] crate::alternative::early_init(true); @@ -180,8 +177,6 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { // Set up double buffer for graphical debug now that heap is available graphical_debug::init_heap(); - idt::init_paging_post_heap(LogicalCpuId::BSP); - // Activate memory logging crate::log::init(); @@ -225,6 +220,7 @@ pub struct KernelArgsAp { pub cpu_id: LogicalCpuId, pub page_table: usize, pub pcr_ptr: *mut gdt::ProcessorControlRegion, + pub idt_ptr: *mut idt::Idt, } /// Entry to rust for an AP @@ -238,8 +234,8 @@ pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { // Set up GDT gdt::install_pcr(args.pcr_ptr); - // Set up IDT before paging - idt::init(); + // Set up IDT + idt::install_idt(args.idt_ptr); // Initialize paging RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(args.page_table)); @@ -248,9 +244,6 @@ pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { #[cfg(all(target_arch = "x86_64", feature = "profiling"))] crate::profiling::init(); - // Set up IDT for AP - idt::init_paging_post_heap(args.cpu_id); - #[cfg(target_arch = "x86_64")] crate::alternative::early_init(false);