diff --git a/src/arch/x86/gdt.rs b/src/arch/x86/gdt.rs index d6c5cc0e1e..b3dc9d5ee8 100644 --- a/src/arch/x86/gdt.rs +++ b/src/arch/x86/gdt.rs @@ -79,6 +79,7 @@ pub struct ProcessorControlRegion { pub tss: TssWrapper, pub self_ref: usize, pub gdt: [GdtEntry; 9], + percpu: crate::percpu::PercpuBlock, } // NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding @@ -124,7 +125,7 @@ pub unsafe fn init() { } /// Initialize GDT and configure percpu. -pub unsafe fn init_paging(stack_offset: usize) { +pub unsafe fn init_paging(stack_offset: usize, cpu_id: usize) { let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR frame"); let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); @@ -164,6 +165,11 @@ pub unsafe fn init_paging(stack_offset: usize) { // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); + + pcr.percpu = crate::percpu::PercpuBlock { + cpu_id, + switch_internals: Default::default(), + }; } // TODO: Share code with x86. Maybe even with aarch64? @@ -228,3 +234,9 @@ impl GdtEntry { self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F; } } + +impl crate::percpu::PercpuBlock { + pub fn current() -> &'static Self { + unsafe { &*core::ptr::addr_of!((*pcr()).percpu) } + } +} diff --git a/src/arch/x86/start.rs b/src/arch/x86/start.rs index 94170d16eb..8f29c8380a 100644 --- a/src/arch/x86/start.rs +++ b/src/arch/x86/start.rs @@ -25,12 +25,6 @@ use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKin static BSS_TEST_ZERO: usize = 0; /// Test of non-zero values in data. static DATA_TEST_NONZERO: usize = usize::max_value(); -/// Test of zero values in thread BSS -#[thread_local] -static mut TBSS_TEST_ZERO: usize = 0; -/// Test of non-zero values in thread data. -#[thread_local] -static mut TDATA_TEST_NONZERO: usize = usize::max_value(); pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0); pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0); @@ -134,7 +128,7 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { paging::init(); // Set up GDT after paging with TLS - gdt::init_paging(args.stack_base as usize + args.stack_size as usize); + gdt::init_paging(args.stack_base as usize + args.stack_size as usize, 0); // Set up IDT idt::init_paging_bsp(); @@ -142,16 +136,6 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Set up syscall instruction interrupt::syscall::init(); - // Test tdata and tbss - { - assert_eq!(TBSS_TEST_ZERO, 0); - TBSS_TEST_ZERO += 1; - assert_eq!(TBSS_TEST_ZERO, 1); - assert_eq!(TDATA_TEST_NONZERO, usize::max_value()); - TDATA_TEST_NONZERO -= 1; - assert_eq!(TDATA_TEST_NONZERO, usize::max_value() - 1); - } - // Reset AP variables CPU_COUNT.store(1, Ordering::SeqCst); AP_READY.store(false, Ordering::SeqCst); @@ -237,7 +221,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { paging::init(); // Set up GDT with TLS - gdt::init_paging(stack_end); + gdt::init_paging(stack_end, cpu_id); // Set up IDT for AP idt::init_paging_post_heap(false, cpu_id); @@ -245,16 +229,6 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { // Set up syscall instruction interrupt::syscall::init(); - // Test tdata and tbss - { - assert_eq!(TBSS_TEST_ZERO, 0); - TBSS_TEST_ZERO += 1; - assert_eq!(TBSS_TEST_ZERO, 1); - assert_eq!(TDATA_TEST_NONZERO, usize::max_value()); - TDATA_TEST_NONZERO -= 1; - assert_eq!(TDATA_TEST_NONZERO, usize::max_value() - 1); - } - // Initialize devices (for AP) device::init_ap(); diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index ceaa9aac42..411386e8b4 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -4,6 +4,7 @@ use core::convert::TryInto; use core::mem; use crate::paging::{PAGE_SIZE, RmmA, RmmArch}; +use crate::percpu::PercpuBlock; use x86::bits64::task::TaskStateSegment; use x86::Ring; @@ -82,8 +83,7 @@ pub struct ProcessorControlRegion { // The GDT *must* be stored in the PCR! The paranoid interrupt handler, lacking a reliable way // to correctly obtain GSBASE, uses SGDT to calculate the PCR offset. pub gdt: [GdtEntry; 8], - // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it - // first to avoid cache invalidation. + pub percpu: PercpuBlock, } const _: () = { @@ -145,7 +145,7 @@ unsafe fn load_segments() { /// Initialize GDT and PCR. #[cold] -pub unsafe fn init_paging(stack_offset: usize) { +pub unsafe fn init_paging(stack_offset: usize, cpu_id: usize) { let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR"); let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); @@ -213,6 +213,11 @@ pub unsafe fn init_paging(stack_offset: usize) { x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE); } + + pcr.percpu = PercpuBlock { + cpu_id, + switch_internals: Default::default(), + }; } /// Copy tdata, clear tbss, calculate TCB end pointer @@ -270,3 +275,9 @@ impl GdtEntry { self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F; } } + +impl PercpuBlock { + pub fn current() -> &'static Self { + unsafe { &*core::ptr::addr_of!((*pcr()).percpu) } + } +} diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index 0036c2d625..9accbc53fc 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -3,7 +3,6 @@ /// It must create the IDT with the correct entries, those entries are /// defined in other files inside of the `arch` module -use core::cell::Cell; use core::slice; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -26,12 +25,6 @@ use crate::paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind}; static BSS_TEST_ZERO: usize = 0; /// Test of non-zero values in data. static DATA_TEST_NONZERO: usize = usize::max_value(); -/// Test of zero values in thread BSS -#[thread_local] -static TBSS_TEST_ZERO: Cell = Cell::new(0); -/// Test of non-zero values in thread data. -#[thread_local] -static TDATA_TEST_NONZERO: Cell = Cell::new(usize::max_value()); pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0); pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0); @@ -136,7 +129,7 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { paging::init(); // Set up GDT after paging with TLS - gdt::init_paging(args.stack_base as usize + args.stack_size as usize); + gdt::init_paging(args.stack_base as usize + args.stack_size as usize, 0); // Set up IDT idt::init_paging_bsp(); @@ -144,16 +137,6 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Set up syscall instruction interrupt::syscall::init(); - // Test tdata and tbss - { - assert_eq!(TBSS_TEST_ZERO.get(), 0); - TBSS_TEST_ZERO.set(TBSS_TEST_ZERO.get() + 1); - assert_eq!(TBSS_TEST_ZERO.get(), 1); - assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value()); - TDATA_TEST_NONZERO.set(TDATA_TEST_NONZERO.get() - 1); - assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value() - 1); - } - // Reset AP variables CPU_COUNT.store(1, Ordering::SeqCst); AP_READY.store(false, Ordering::SeqCst); @@ -242,7 +225,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { paging::init(); // Set up GDT with TLS - gdt::init_paging(stack_end); + gdt::init_paging(stack_end, cpu_id); // Set up IDT for AP idt::init_paging_post_heap(false, cpu_id); @@ -253,16 +236,6 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { // Initialize miscellaneous processor features misc::init(); - // Test tdata and tbss - { - assert_eq!(TBSS_TEST_ZERO.get(), 0); - TBSS_TEST_ZERO.set(TBSS_TEST_ZERO.get() + 1); - assert_eq!(TBSS_TEST_ZERO.get(), 1); - assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value()); - TDATA_TEST_NONZERO.set(TDATA_TEST_NONZERO.get() - 1); - assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value() - 1); - } - // Initialize devices (for AP) device::init_ap(); diff --git a/src/common/int_like.rs b/src/common/int_like.rs index 925d9b7321..873d3c87cc 100644 --- a/src/common/int_like.rs +++ b/src/common/int_like.rs @@ -25,7 +25,7 @@ #[macro_export] macro_rules! int_like { ($new_type_name:ident, $backing_type: ident) => { - #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] + #[derive(Default, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)] pub struct $new_type_name($backing_type); impl $new_type_name { diff --git a/src/context/list.rs b/src/context/list.rs index 2944d0e914..43755339a7 100644 --- a/src/context/list.rs +++ b/src/context/list.rs @@ -1,7 +1,6 @@ use alloc::sync::Arc; use alloc::collections::BTreeMap; use core::{iter, mem}; -use core::sync::atomic::Ordering; use spin::RwLock; @@ -39,7 +38,7 @@ impl ContextList { /// Get the current context. pub fn current(&self) -> Option<&Arc>> { - self.map.get(&super::CONTEXT_ID.load(Ordering::SeqCst)) + self.map.get(&super::context_id()) } pub fn iter(&self) -> ::alloc::collections::btree_map::Iter>> { diff --git a/src/context/mod.rs b/src/context/mod.rs index 6b742357d6..868621e57a 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -1,7 +1,6 @@ //! # Context management //! //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching) -use core::sync::atomic::Ordering; use alloc::borrow::Cow; use alloc::sync::Arc; @@ -9,6 +8,7 @@ use alloc::sync::Arc; use spin::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use crate::paging::{RmmA, RmmArch, TableKind}; +use crate::percpu::PercpuBlock; use crate::syscall::error::{Error, ESRCH, Result}; pub use self::context::{BorrowedHtBuf, Context, ContextId, ContextSnapshot, Status, WaitpidKey}; @@ -59,9 +59,6 @@ pub const CONTEXT_MAX_FILES: usize = 65_536; /// Contexts list static CONTEXTS: RwLock = RwLock::new(ContextList::new()); -#[thread_local] -static CONTEXT_ID: context::AtomicContextId = context::AtomicContextId::default(); - pub use self::arch::empty_cr3; pub fn init() { @@ -77,7 +74,10 @@ pub fn init() { context.status = Status::Runnable; context.running = true; context.cpu_id = Some(crate::cpu_id()); - CONTEXT_ID.store(context.id, Ordering::SeqCst); + + unsafe { + PercpuBlock::current().switch_internals.set_context_id(context.id); + } } /// Get the global schemes list, const @@ -91,13 +91,7 @@ pub fn contexts_mut() -> RwLockWriteGuard<'static, ContextList> { } pub fn context_id() -> ContextId { - // Thread local variables can and should only be modified using Relaxed. This is to prevent a - // hardware thread from racing with itself, for example if there is an interrupt. Orderings - // stronger than Relaxed are only necessary for inter-processor synchronization. - let id = CONTEXT_ID.load(Ordering::Relaxed); - // Prevent the compiler from reordering subsequent loads and stores to before this load. - core::sync::atomic::compiler_fence(Ordering::Acquire); - id + PercpuBlock::current().switch_internals.context_id() } pub fn current() -> Result>> { diff --git a/src/context/switch.rs b/src/context/switch.rs index e49a2b2166..457ea5938b 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -7,13 +7,15 @@ use alloc::sync::Arc; use spin::{RwLock, RwLockWriteGuard}; use crate::context::signal::signal_handler; -use crate::context::{arch, contexts, Context, CONTEXT_ID}; -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +use crate::context::{arch, contexts, Context}; use crate::gdt; use crate::interrupt; +use crate::percpu::PercpuBlock; use crate::ptrace; use crate::time; +use super::ContextId; + unsafe fn update_runnable(context: &mut Context, cpu_id: usize) -> bool { // Ignore already running contexts if context.running { @@ -87,16 +89,12 @@ struct SwitchResult { next_lock: Arc>, } -#[thread_local] -static SWITCH_RESULT: Cell> = Cell::new(None); - -//resets to 0 in context::switch() -#[thread_local] -pub static PIT_TICKS: Cell = Cell::new(0); pub fn tick() { - let new_ticks = PIT_TICKS.get() + 1; - PIT_TICKS.set(new_ticks); + let ticks_cell = &PercpuBlock::current().switch_internals.pit_ticks; + + let new_ticks = ticks_cell.get() + 1; + ticks_cell.set(new_ticks); // Switch after 3 ticks (about 6.75 ms) if new_ticks >= 3 { @@ -105,7 +103,7 @@ pub fn tick() { } pub unsafe extern "C" fn switch_finish_hook() { - if let Some(SwitchResult { prev_lock, next_lock }) = SWITCH_RESULT.take() { + if let Some(SwitchResult { prev_lock, next_lock }) = PercpuBlock::current().switch_internals.switch_result.take() { prev_lock.force_write_unlock(); next_lock.force_write_unlock(); } else { @@ -121,11 +119,13 @@ pub unsafe extern "C" fn switch_finish_hook() { /// /// Do not call this while holding locks! pub unsafe fn switch() -> bool { - // TODO: Better memory orderings? + let percpu = PercpuBlock::current(); + //set PIT Interrupt counter to 0, giving each process same amount of PIT ticks - PIT_TICKS.set(0); + percpu.switch_internals.pit_ticks.set(0); // Set the global lock to avoid the unsafe operations below from causing issues + // TODO: Better memory orderings? while arch::CONTEXT_SWITCH_LOCK.compare_exchange_weak(false, true, Ordering::SeqCst, Ordering::Relaxed).is_err() { interrupt::pause(); } @@ -190,7 +190,8 @@ pub unsafe fn switch() -> bool { gdt::set_tss_stack(stack.as_ptr() as usize + stack.len()); } } - CONTEXT_ID.store(next_context.id, Ordering::SeqCst); + let percpu = PercpuBlock::current(); + percpu.switch_internals.context_id.set(next_context.id); if next_context.ksig.is_none() { //TODO: Allow nested signals @@ -204,7 +205,7 @@ pub unsafe fn switch() -> bool { } } - SWITCH_RESULT.set(Some(SwitchResult { + percpu.switch_internals.switch_result.set(Some(SwitchResult { prev_lock: prev_context_lock, next_lock: next_context_lock, })); @@ -223,3 +224,20 @@ pub unsafe fn switch() -> bool { false } } + +#[derive(Default)] +pub struct ContextSwitchPercpu { + switch_result: Cell>, + pit_ticks: Cell, + + /// Unique ID of the currently running context. + context_id: Cell, +} +impl ContextSwitchPercpu { + pub fn context_id(&self) -> ContextId { + self.context_id.get() + } + pub unsafe fn set_context_id(&self, new: ContextId) { + self.context_id.set(new) + } +} diff --git a/src/lib.rs b/src/lib.rs index 78007aae2e..e4fcae0305 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,12 +48,12 @@ #![feature(iter_array_chunks)] #![feature(asm_const)] // TODO: Relax requirements of most asm invocations #![feature(const_option)] +#![feature(const_refs_to_cell)] #![feature(int_roundings)] #![feature(let_chains)] #![feature(naked_functions)] #![feature(slice_ptr_get, slice_ptr_len)] #![feature(sync_unsafe_cell)] -#![feature(thread_local)] #![no_std] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] @@ -64,13 +64,6 @@ extern crate alloc; #[macro_use] extern crate bitflags; -extern crate bitfield; -extern crate goblin; -extern crate linked_list_allocator; -extern crate rustc_demangle; -extern crate spin; -#[cfg(feature = "slab")] -extern crate slab_allocator; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -125,6 +118,8 @@ pub mod memory; #[cfg(not(any(feature="doc", test)))] pub mod panic; +pub mod percpu; + /// Process tracing pub mod ptrace; @@ -147,14 +142,10 @@ pub mod tests; #[global_allocator] static ALLOCATOR: allocator::Allocator = allocator::Allocator; -/// A unique number that identifies the current CPU - used for scheduling -#[thread_local] -static CPU_ID: AtomicUsize = AtomicUsize::new(0); - /// Get the current CPU's scheduling ID #[inline(always)] pub fn cpu_id() -> usize { - CPU_ID.load(Ordering::Relaxed) + crate::percpu::PercpuBlock::current().cpu_id } /// The count of all CPUs that can have work scheduled @@ -185,7 +176,6 @@ static BOOTSTRAP: spin::Once = spin::Once::new(); /// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this pub fn kmain(cpus: usize, bootstrap: Bootstrap) -> ! { - CPU_ID.store(0, Ordering::SeqCst); CPU_COUNT.store(cpus, Ordering::SeqCst); //Initialize the first context, stored in kernel/src/context/mod.rs @@ -226,8 +216,6 @@ pub fn kmain(cpus: usize, bootstrap: Bootstrap) -> ! { /// This is the main kernel entry point for secondary CPUs #[allow(unreachable_code, unused_variables)] pub fn kmain_ap(id: usize) -> ! { - CPU_ID.store(id, Ordering::SeqCst); - if cfg!(feature = "multi_core") { context::init(); diff --git a/src/percpu.rs b/src/percpu.rs new file mode 100644 index 0000000000..28c018c0df --- /dev/null +++ b/src/percpu.rs @@ -0,0 +1,16 @@ +use crate::context::switch::ContextSwitchPercpu; + +/// The percpu block, that stored all percpu variables. +pub struct PercpuBlock { + /// A unique immutable number that identifies the current CPU - used for scheduling + // TODO: Differentiate between logical CPU IDs and hardware CPU IDs (e.g. APIC IDs) + pub cpu_id: usize, + + /// Context management + pub switch_internals: ContextSwitchPercpu, + + // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it + // first to avoid cache invalidation. +} + +// PercpuBlock::current() is implemented somewhere in the arch-specific modules