Stop using #[thread_local] on x86_*.

src/arch/x86/gdt.rs

@@ -79,6 +79,7 @@ pub struct ProcessorControlRegion {
     pub tss: TssWrapper,
     pub self_ref: usize,
     pub gdt: [GdtEntry; 9],
+    percpu: crate::percpu::PercpuBlock,
 }
 
 // NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding
@@ -124,7 +125,7 @@ pub unsafe fn init() {
 }
 
 /// Initialize GDT and configure percpu.
-pub unsafe fn init_paging(stack_offset: usize) {
+pub unsafe fn init_paging(stack_offset: usize, cpu_id: usize) {
     let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR frame");
     let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion);
 
@@ -164,6 +165,11 @@ pub unsafe fn init_paging(stack_offset: usize) {
 
     // Load the task register
     task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
+
+    pcr.percpu = crate::percpu::PercpuBlock {
+        cpu_id,
+        switch_internals: Default::default(),
+    };
 }
 
 // TODO: Share code with x86. Maybe even with aarch64?
@@ -228,3 +234,9 @@ impl GdtEntry {
         self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F;
     }
 }
+
+impl crate::percpu::PercpuBlock {
+    pub fn current() -> &'static Self {
+        unsafe { &*core::ptr::addr_of!((*pcr()).percpu) }
+    }
+}

src/arch/x86/start.rs

@@ -25,12 +25,6 @@ use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKin
 static BSS_TEST_ZERO: usize = 0;
 /// Test of non-zero values in data.
 static DATA_TEST_NONZERO: usize = usize::max_value();
-/// Test of zero values in thread BSS
-#[thread_local]
-static mut TBSS_TEST_ZERO: usize = 0;
-/// Test of non-zero values in thread data.
-#[thread_local]
-static mut TDATA_TEST_NONZERO: usize = usize::max_value();
 
 pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0);
 pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0);
@@ -134,7 +128,7 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
         paging::init();
 
         // Set up GDT after paging with TLS
-        gdt::init_paging(args.stack_base as usize + args.stack_size as usize);
+        gdt::init_paging(args.stack_base as usize + args.stack_size as usize, 0);
 
         // Set up IDT
         idt::init_paging_bsp();
@@ -142,16 +136,6 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
         // Set up syscall instruction
         interrupt::syscall::init();
 
-        // Test tdata and tbss
-        {
-            assert_eq!(TBSS_TEST_ZERO, 0);
-            TBSS_TEST_ZERO += 1;
-            assert_eq!(TBSS_TEST_ZERO, 1);
-            assert_eq!(TDATA_TEST_NONZERO, usize::max_value());
-            TDATA_TEST_NONZERO -= 1;
-            assert_eq!(TDATA_TEST_NONZERO, usize::max_value() - 1);
-        }
-
         // Reset AP variables
         CPU_COUNT.store(1, Ordering::SeqCst);
         AP_READY.store(false, Ordering::SeqCst);
@@ -237,7 +221,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
         paging::init();
 
         // Set up GDT with TLS
-        gdt::init_paging(stack_end);
+        gdt::init_paging(stack_end, cpu_id);
 
         // Set up IDT for AP
         idt::init_paging_post_heap(false, cpu_id);
@@ -245,16 +229,6 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
         // Set up syscall instruction
         interrupt::syscall::init();
 
-        // Test tdata and tbss
-        {
-            assert_eq!(TBSS_TEST_ZERO, 0);
-            TBSS_TEST_ZERO += 1;
-            assert_eq!(TBSS_TEST_ZERO, 1);
-            assert_eq!(TDATA_TEST_NONZERO, usize::max_value());
-            TDATA_TEST_NONZERO -= 1;
-            assert_eq!(TDATA_TEST_NONZERO, usize::max_value() - 1);
-        }
-
         // Initialize devices (for AP)
         device::init_ap();
 

src/arch/x86_64/gdt.rs

@@ -4,6 +4,7 @@ use core::convert::TryInto;
 use core::mem;
 
 use crate::paging::{PAGE_SIZE, RmmA, RmmArch};
+use crate::percpu::PercpuBlock;
 
 use x86::bits64::task::TaskStateSegment;
 use x86::Ring;
@@ -82,8 +83,7 @@ pub struct ProcessorControlRegion {
     // The GDT *must* be stored in the PCR! The paranoid interrupt handler, lacking a reliable way
     // to correctly obtain GSBASE, uses SGDT to calculate the PCR offset.
     pub gdt: [GdtEntry; 8],
-    // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
-    // first to avoid cache invalidation.
+    pub percpu: PercpuBlock,
 }
 
 const _: () = {
@@ -145,7 +145,7 @@ unsafe fn load_segments() {
 
 /// Initialize GDT and PCR.
 #[cold]
-pub unsafe fn init_paging(stack_offset: usize) {
+pub unsafe fn init_paging(stack_offset: usize, cpu_id: usize) {
     let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR");
     let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion);
 
@@ -213,6 +213,11 @@ pub unsafe fn init_paging(stack_offset: usize) {
 
         x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE);
     }
+
+    pcr.percpu = PercpuBlock {
+        cpu_id,
+        switch_internals: Default::default(),
+    };
 }
 
 /// Copy tdata, clear tbss, calculate TCB end pointer
@@ -270,3 +275,9 @@ impl GdtEntry {
         self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F;
     }
 }
+
+impl PercpuBlock {
+    pub fn current() -> &'static Self {
+        unsafe { &*core::ptr::addr_of!((*pcr()).percpu) }
+    }
+}

src/arch/x86_64/start.rs

@@ -3,7 +3,6 @@
 /// It must create the IDT with the correct entries, those entries are
 /// defined in other files inside of the `arch` module
 
-use core::cell::Cell;
 use core::slice;
 use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 
@@ -26,12 +25,6 @@ use crate::paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind};
 static BSS_TEST_ZERO: usize = 0;
 /// Test of non-zero values in data.
 static DATA_TEST_NONZERO: usize = usize::max_value();
-/// Test of zero values in thread BSS
-#[thread_local]
-static TBSS_TEST_ZERO: Cell<usize> = Cell::new(0);
-/// Test of non-zero values in thread data.
-#[thread_local]
-static TDATA_TEST_NONZERO: Cell<usize> = Cell::new(usize::max_value());
 
 pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0);
 pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0);
@@ -136,7 +129,7 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
         paging::init();
 
         // Set up GDT after paging with TLS
-        gdt::init_paging(args.stack_base as usize + args.stack_size as usize);
+        gdt::init_paging(args.stack_base as usize + args.stack_size as usize, 0);
 
         // Set up IDT
         idt::init_paging_bsp();
@@ -144,16 +137,6 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
         // Set up syscall instruction
         interrupt::syscall::init();
 
-        // Test tdata and tbss
-        {
-            assert_eq!(TBSS_TEST_ZERO.get(), 0);
-            TBSS_TEST_ZERO.set(TBSS_TEST_ZERO.get() + 1);
-            assert_eq!(TBSS_TEST_ZERO.get(), 1);
-            assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value());
-            TDATA_TEST_NONZERO.set(TDATA_TEST_NONZERO.get() - 1);
-            assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value() - 1);
-        }
-
         // Reset AP variables
         CPU_COUNT.store(1, Ordering::SeqCst);
         AP_READY.store(false, Ordering::SeqCst);
@@ -242,7 +225,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
         paging::init();
 
         // Set up GDT with TLS
-        gdt::init_paging(stack_end);
+        gdt::init_paging(stack_end, cpu_id);
 
         // Set up IDT for AP
         idt::init_paging_post_heap(false, cpu_id);
@@ -253,16 +236,6 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
         // Initialize miscellaneous processor features
         misc::init();
 
-        // Test tdata and tbss
-        {
-            assert_eq!(TBSS_TEST_ZERO.get(), 0);
-            TBSS_TEST_ZERO.set(TBSS_TEST_ZERO.get() + 1);
-            assert_eq!(TBSS_TEST_ZERO.get(), 1);
-            assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value());
-            TDATA_TEST_NONZERO.set(TDATA_TEST_NONZERO.get() - 1);
-            assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value() - 1);
-        }
-
         // Initialize devices (for AP)
         device::init_ap();
 

src/common/int_like.rs

@@ -25,7 +25,7 @@
 #[macro_export]
 macro_rules! int_like {
     ($new_type_name:ident, $backing_type: ident) => {
-        #[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
+        #[derive(Default, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
         pub struct $new_type_name($backing_type);
 
         impl $new_type_name {

src/context/list.rs

@@ -1,7 +1,6 @@
 use alloc::sync::Arc;
 use alloc::collections::BTreeMap;
 use core::{iter, mem};
-use core::sync::atomic::Ordering;
 
 use spin::RwLock;
 
@@ -39,7 +38,7 @@ impl ContextList {
 
     /// Get the current context.
     pub fn current(&self) -> Option<&Arc<RwLock<Context>>> {
-        self.map.get(&super::CONTEXT_ID.load(Ordering::SeqCst))
+        self.map.get(&super::context_id())
     }
 
     pub fn iter(&self) -> ::alloc::collections::btree_map::Iter<ContextId, Arc<RwLock<Context>>> {

src/context/mod.rs

@@ -1,7 +1,6 @@
 //! # Context management
 //!
 //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and  [osdev](https://wiki.osdev.org/Context_Switching)
-use core::sync::atomic::Ordering;
 
 use alloc::borrow::Cow;
 use alloc::sync::Arc;
@@ -9,6 +8,7 @@ use alloc::sync::Arc;
 use spin::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 use crate::paging::{RmmA, RmmArch, TableKind};
+use crate::percpu::PercpuBlock;
 use crate::syscall::error::{Error, ESRCH, Result};
 
 pub use self::context::{BorrowedHtBuf, Context, ContextId, ContextSnapshot, Status, WaitpidKey};
@@ -59,9 +59,6 @@ pub const CONTEXT_MAX_FILES: usize = 65_536;
 /// Contexts list
 static CONTEXTS: RwLock<ContextList> = RwLock::new(ContextList::new());
 
-#[thread_local]
-static CONTEXT_ID: context::AtomicContextId = context::AtomicContextId::default();
-
 pub use self::arch::empty_cr3;
 
 pub fn init() {
@@ -77,7 +74,10 @@ pub fn init() {
     context.status = Status::Runnable;
     context.running = true;
     context.cpu_id = Some(crate::cpu_id());
-    CONTEXT_ID.store(context.id, Ordering::SeqCst);
+
+    unsafe {
+        PercpuBlock::current().switch_internals.set_context_id(context.id);
+    }
 }
 
 /// Get the global schemes list, const
@@ -91,13 +91,7 @@ pub fn contexts_mut() -> RwLockWriteGuard<'static, ContextList> {
 }
 
 pub fn context_id() -> ContextId {
-    // Thread local variables can and should only be modified using Relaxed. This is to prevent a
-    // hardware thread from racing with itself, for example if there is an interrupt. Orderings
-    // stronger than Relaxed are only necessary for inter-processor synchronization.
-    let id = CONTEXT_ID.load(Ordering::Relaxed);
-    // Prevent the compiler from reordering subsequent loads and stores to before this load.
-    core::sync::atomic::compiler_fence(Ordering::Acquire);
-    id
+    PercpuBlock::current().switch_internals.context_id()
 }
 
 pub fn current() -> Result<Arc<RwLock<Context>>> {

src/context/switch.rs

@@ -7,13 +7,15 @@ use alloc::sync::Arc;
 use spin::{RwLock, RwLockWriteGuard};
 
 use crate::context::signal::signal_handler;
-use crate::context::{arch, contexts, Context, CONTEXT_ID};
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+use crate::context::{arch, contexts, Context};
 use crate::gdt;
 use crate::interrupt;
+use crate::percpu::PercpuBlock;
 use crate::ptrace;
 use crate::time;
 
+use super::ContextId;
+
 unsafe fn update_runnable(context: &mut Context, cpu_id: usize) -> bool {
     // Ignore already running contexts
     if context.running {
@@ -87,16 +89,12 @@ struct SwitchResult {
     next_lock: Arc<RwLock<Context>>,
 }
 
-#[thread_local]
-static SWITCH_RESULT: Cell<Option<SwitchResult>> = Cell::new(None);
-
-//resets to 0 in context::switch()
-#[thread_local]
-pub static PIT_TICKS: Cell<usize> = Cell::new(0);
 
 pub fn tick() {
-    let new_ticks = PIT_TICKS.get() + 1;
-    PIT_TICKS.set(new_ticks);
+    let ticks_cell = &PercpuBlock::current().switch_internals.pit_ticks;
+
+    let new_ticks = ticks_cell.get() + 1;
+    ticks_cell.set(new_ticks);
 
     // Switch after 3 ticks (about 6.75 ms)
     if new_ticks >= 3 {
@@ -105,7 +103,7 @@ pub fn tick() {
 }
 
 pub unsafe extern "C" fn switch_finish_hook() {
-    if let Some(SwitchResult { prev_lock, next_lock }) = SWITCH_RESULT.take() {
+    if let Some(SwitchResult { prev_lock, next_lock }) = PercpuBlock::current().switch_internals.switch_result.take() {
         prev_lock.force_write_unlock();
         next_lock.force_write_unlock();
     } else {
@@ -121,11 +119,13 @@ pub unsafe extern "C" fn switch_finish_hook() {
 ///
 /// Do not call this while holding locks!
 pub unsafe fn switch() -> bool {
-    // TODO: Better memory orderings?
+    let percpu = PercpuBlock::current();
+
     //set PIT Interrupt counter to 0, giving each process same amount of PIT ticks
-    PIT_TICKS.set(0);
+    percpu.switch_internals.pit_ticks.set(0);
 
     // Set the global lock to avoid the unsafe operations below from causing issues
+    // TODO: Better memory orderings?
     while arch::CONTEXT_SWITCH_LOCK.compare_exchange_weak(false, true, Ordering::SeqCst, Ordering::Relaxed).is_err() {
         interrupt::pause();
     }
@@ -190,7 +190,8 @@ pub unsafe fn switch() -> bool {
                 gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
             }
         }
-        CONTEXT_ID.store(next_context.id, Ordering::SeqCst);
+        let percpu = PercpuBlock::current();
+        percpu.switch_internals.context_id.set(next_context.id);
 
         if next_context.ksig.is_none() {
             //TODO: Allow nested signals
@@ -204,7 +205,7 @@ pub unsafe fn switch() -> bool {
             }
         }
 
-        SWITCH_RESULT.set(Some(SwitchResult {
+        percpu.switch_internals.switch_result.set(Some(SwitchResult {
             prev_lock: prev_context_lock,
             next_lock: next_context_lock,
         }));
@@ -223,3 +224,20 @@ pub unsafe fn switch() -> bool {
         false
     }
 }
+
+#[derive(Default)]
+pub struct ContextSwitchPercpu {
+    switch_result: Cell<Option<SwitchResult>>,
+    pit_ticks: Cell<usize>,
+
+    /// Unique ID of the currently running context.
+    context_id: Cell<ContextId>,
+}
+impl ContextSwitchPercpu {
+    pub fn context_id(&self) -> ContextId {
+        self.context_id.get()
+    }
+    pub unsafe fn set_context_id(&self, new: ContextId) {
+        self.context_id.set(new)
+    }
+}

src/lib.rs

@@ -48,12 +48,12 @@
 #![feature(iter_array_chunks)]
 #![feature(asm_const)] // TODO: Relax requirements of most asm invocations
 #![feature(const_option)]
+#![feature(const_refs_to_cell)]
 #![feature(int_roundings)]
 #![feature(let_chains)]
 #![feature(naked_functions)]
 #![feature(slice_ptr_get, slice_ptr_len)]
 #![feature(sync_unsafe_cell)]
-#![feature(thread_local)]
 #![no_std]
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
@@ -64,13 +64,6 @@ extern crate alloc;
 
 #[macro_use]
 extern crate bitflags;
-extern crate bitfield;
-extern crate goblin;
-extern crate linked_list_allocator;
-extern crate rustc_demangle;
-extern crate spin;
-#[cfg(feature = "slab")]
-extern crate slab_allocator;
 
 use core::sync::atomic::{AtomicUsize, Ordering};
 
@@ -125,6 +118,8 @@ pub mod memory;
 #[cfg(not(any(feature="doc", test)))]
 pub mod panic;
 
+pub mod percpu;
+
 /// Process tracing
 pub mod ptrace;
 
@@ -147,14 +142,10 @@ pub mod tests;
 #[global_allocator]
 static ALLOCATOR: allocator::Allocator = allocator::Allocator;
 
-/// A unique number that identifies the current CPU - used for scheduling
-#[thread_local]
-static CPU_ID: AtomicUsize = AtomicUsize::new(0);
-
 /// Get the current CPU's scheduling ID
 #[inline(always)]
 pub fn cpu_id() -> usize {
-    CPU_ID.load(Ordering::Relaxed)
+    crate::percpu::PercpuBlock::current().cpu_id
 }
 
 /// The count of all CPUs that can have work scheduled
@@ -185,7 +176,6 @@ static BOOTSTRAP: spin::Once<Bootstrap> = spin::Once::new();
 
 /// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this
 pub fn kmain(cpus: usize, bootstrap: Bootstrap) -> ! {
-    CPU_ID.store(0, Ordering::SeqCst);
     CPU_COUNT.store(cpus, Ordering::SeqCst);
 
     //Initialize the first context, stored in kernel/src/context/mod.rs
@@ -226,8 +216,6 @@ pub fn kmain(cpus: usize, bootstrap: Bootstrap) -> ! {
 /// This is the main kernel entry point for secondary CPUs
 #[allow(unreachable_code, unused_variables)]
 pub fn kmain_ap(id: usize) -> ! {
-    CPU_ID.store(id, Ordering::SeqCst);
-
     if cfg!(feature = "multi_core") {
         context::init();
 

src/percpu.rs

@@ -0,0 +1,16 @@
+use crate::context::switch::ContextSwitchPercpu;
+
+/// The percpu block, that stored all percpu variables.
+pub struct PercpuBlock {
+    /// A unique immutable number that identifies the current CPU - used for scheduling
+    // TODO: Differentiate between logical CPU IDs and hardware CPU IDs (e.g. APIC IDs)
+    pub cpu_id: usize,
+
+    /// Context management
+    pub switch_internals: ContextSwitchPercpu,
+
+    // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
+    // first to avoid cache invalidation.
+}
+
+// PercpuBlock::current() is implemented somewhere in the arch-specific modules