vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'no_more_tls' into 'master'

Browse Source 
Replace #[thread_local]

See merge request redox-os/kernel!239
This commit is contained in:
Jeremy Soller
2023-08-09 14:18:16 +00:00
parent 0c90802ae0 de23cd447e
commit 652aa4f1ac
27 changed files with 156 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linkers/aarch64.ld
-12
View File
@@ -39,18 +39,6 @@ SECTIONS {
__bss_end = .;
}
.tdata : AT(ADDR(.tdata) - KERNEL_OFFSET) {
__tdata_start = .;
*(.tdata*)
. = ALIGN(4096);
__tdata_end = .;
__tbss_start = .;
*(.tbss*)
. += 8;
. = ALIGN(4096);
__tbss_end = .;
}
__end = .;
/DISCARD/ : {
linkers/i686.ld
-13
View File
@@ -33,19 +33,6 @@ SECTIONS {
. = ALIGN(4K);
}
.tdata ALIGN(4K) : AT(ADDR(.tdata) - KERNEL_OFFSET) {
__bss_end = .;
__tdata_start = .;
*(.tdata*)
. = ALIGN(4K);
__tdata_end = .;
__tbss_start = .;
*(.tbss*)
. += 8;
. = ALIGN(4K);
__tbss_end = .;
}
__end = .;
/DISCARD/ : {
linkers/x86_64.ld
-11
View File
@@ -32,17 +32,6 @@ SECTIONS {
*(.bss*)
}
.tdata ALIGN(4K) : AT(ADDR(.tdata) - KERNEL_OFFSET) {
__bss_end = .;
__tdata_start = .;
*(.tdata*)
__tdata_end = .;
__tbss_start = .;
*(.tbss*)
. = ALIGN(4K);
__tbss_end = .;
}
__end = .;
/DISCARD/ : {
src/arch/aarch64/device/cpu/registers/control_regs.rs
+6
View File
@@ -50,6 +50,12 @@ pub unsafe fn tpidr_el0_write(val: u64) {
asm!("msr tpidr_el0, {}", in(reg) val);
}
pub unsafe fn tpidr_el1() -> u64 {
let ret: u64;
asm!("mrs {}, tpidr_el1", out(reg) ret);
ret
}
pub unsafe fn tpidr_el1_write(val: u64) {
asm!("msr tpidr_el1, {}", in(reg) val);
}
src/arch/aarch64/interrupt/irq.rs
+2 -7
View File
@@ -9,9 +9,6 @@ use crate::time;
use crate::{exception_stack};
//resets to 0 in context::switch()
pub static PIT_TICKS: AtomicUsize = ATOMIC_USIZE_INIT;
exception_stack!(irq_at_el0, |stack| {
match gic::irq_ack() {
30 => irq_handler_gentimer(30),
@@ -56,10 +53,8 @@ pub unsafe fn irq_handler_gentimer(irq: u32) {
timeout::trigger();
// Switch after 3 ticks (about 6.75 ms)
if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 2 {
let _ = context::switch();
}
context::switch::tick();
trigger(irq);
GENTIMER.reload_count();
}
src/arch/aarch64/misc.rs
+21
View File
@@ -0,0 +1,21 @@
use crate::paging::{RmmA, RmmArch};
use crate::percpu::PercpuBlock;
impl PercpuBlock {
pub fn current() -> &'static Self {
unsafe { &*(crate::device::cpu::registers::control_regs::tpidr_el1() as *const Self) }
}
}
#[cold]
pub unsafe fn init(cpu_id: usize) {
let frame = crate::memory::allocate_frames(1).expect("failed to allocate percpu memory");
let virt = RmmA::phys_to_virt(frame.start_address()).data() as *mut PercpuBlock;
virt.write(PercpuBlock {
cpu_id,
switch_internals: crate::context::switch::ContextSwitchPercpu::default(),
});
crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64);
}
src/arch/aarch64/mod.rs
+3
View File
@@ -16,6 +16,9 @@ pub mod interrupt;
/// Inter-processor interrupts
pub mod ipi;
/// Miscellaneous
pub mod misc;
/// Paging
pub mod paging;
src/arch/aarch64/paging/mod.rs
+4 -123
View File
@@ -29,6 +29,7 @@ pub const ENTRY_COUNT: usize = RmmA::PAGE_ENTRIES;
pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE;
/// Setup Memory Access Indirection Register
#[cold]
unsafe fn init_mair() {
let mut val: control_regs::MairEl1 = control_regs::mair_el1();
@@ -39,130 +40,10 @@ unsafe fn init_mair() {
control_regs::mair_el1_write(val);
}
/// Map percpu
unsafe fn map_percpu(cpu_id: usize, mapper: &mut PageMapper) -> PageFlushAll<RmmA> {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
}
let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
let end = start + size;
let mut flush_all = PageFlushAll::new();
let start_page = Page::containing_address(VirtualAddress::new(start));
let end_page = Page::containing_address(VirtualAddress::new(end - 1));
for page in Page::range_inclusive(start_page, end_page) {
let result = mapper.map(
page.start_address(),
PageFlags::new().write(true).global(cfg!(not(feature = "pti"))),
)
.expect("failed to allocate page table frames while mapping percpu");
flush_all.consume(result);
}
flush_all
}
/// Copy tdata, clear tbss, set TCB self pointer
unsafe fn init_tcb(cpu_id: usize) -> usize {
extern "C" {
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
}
let tcb_offset;
{
let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
let tbss_offset = &__tbss_start as *const _ as usize - &__tdata_start as *const _ as usize;
let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
println!("SET TPIDR_EL1 TO {:X}", start - 0x10);
// FIXME: Empirically initializing tpidr to 16 bytes below start works. I do not know
// whether this is the correct way to handle TLS. Will need to revisit.
control_regs::tpidr_el1_write((start - 0x10) as u64);
println!("SET TPIDR_EL1 DONE");
let end = start + size;
tcb_offset = end - mem::size_of::<usize>();
ptr::copy(&__tdata_start as *const u8, start as *mut u8, tbss_offset);
ptr::write_bytes((start + tbss_offset) as *mut u8, 0, size - tbss_offset);
*(tcb_offset as *mut usize) = end;
}
tcb_offset
}
/// Initialize paging
///
/// Returns page table and thread control block offset
pub unsafe fn init(
cpu_id: usize,
) -> usize {
extern "C" {
/// The starting byte of the text (code) data segment.
static mut __text_start: u8;
/// The ending byte of the text (code) data segment.
static mut __text_end: u8;
/// The starting byte of the _.rodata_ (read-only data) segment.
static mut __rodata_start: u8;
/// The ending byte of the _.rodata_ (read-only data) segment.
static mut __rodata_end: u8;
/// The starting byte of the _.data_ segment.
static mut __data_start: u8;
/// The ending byte of the _.data_ segment.
static mut __data_end: u8;
/// The starting byte of the thread data segment
static mut __tdata_start: u8;
/// The ending byte of the thread data segment
static mut __tdata_end: u8;
/// The starting byte of the thread BSS segment
static mut __tbss_start: u8;
/// The ending byte of the thread BSS segment
static mut __tbss_end: u8;
/// The starting byte of the _.bss_ (uninitialized data) segment.
static mut __bss_start: u8;
/// The ending byte of the _.bss_ (uninitialized data) segment.
static mut __bss_end: u8;
}
/// Initialize MAIR
#[cold]
pub unsafe fn init() {
init_mair();
let flush_all = map_percpu(cpu_id, KernelMapper::lock_manually(cpu_id).get_mut().expect("expected KernelMapper not to be locked re-entrant in paging::init"));
flush_all.flush();
return init_tcb(cpu_id);
}
pub unsafe fn init_ap(
cpu_id: usize,
bsp_table: &mut KernelMapper,
) -> usize {
init_mair();
{
let flush_all = map_percpu(cpu_id, bsp_table.get_mut().expect("KernelMapper locked re-entrant for AP"));
// The flush can be ignored as this is not the active table. See later make_current().
flush_all.ignore();
};
bsp_table.make_current();
init_tcb(cpu_id)
}
/// Page
src/arch/aarch64/start.rs
+2 -16
View File
@@ -22,12 +22,6 @@ use crate::paging::{self, KernelMapper};
static BSS_TEST_ZERO: usize = 0;
/// Test of non-zero values in data.
static DATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
/// Test of zero values in thread BSS
#[thread_local]
static mut TBSS_TEST_ZERO: usize = 0;
/// Test of non-zero values in thread data.
#[thread_local]
static mut TDATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0);
pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0);
@@ -138,17 +132,9 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! {
);
// Initialize paging
let tcb_offset = paging::init(0);
paging::init();
// Test tdata and tbss
{
assert_eq!(TBSS_TEST_ZERO, 0);
TBSS_TEST_ZERO += 1;
assert_eq!(TBSS_TEST_ZERO, 1);
assert_eq!(TDATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
TDATA_TEST_NONZERO -= 1;
assert_eq!(TDATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFE);
}
crate::misc::init(0);
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
src/arch/x86/gdt.rs
+13 -26
View File
@@ -74,11 +74,11 @@ const BASE_GDT: [GdtEntry; 9] = [
#[repr(C, align(4096))]
pub struct ProcessorControlRegion {
pub tcb_end: usize,
pub self_ref: usize,
pub user_rsp_tmp: usize,
pub tss: TssWrapper,
pub self_ref: usize,
pub gdt: [GdtEntry; 9],
percpu: crate::percpu::PercpuBlock,
}
// NOTE: Despite not using #[repr(packed)], we do know that while there may be some padding
@@ -124,7 +124,7 @@ pub unsafe fn init() {
}
/// Initialize GDT and configure percpu.
pub unsafe fn init_paging(stack_offset: usize) {
pub unsafe fn init_paging(stack_offset: usize, cpu_id: usize) {
let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR frame");
let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion);
@@ -137,8 +137,6 @@ pub unsafe fn init_paging(stack_offset: usize) {
base: pcr.gdt.as_ptr() as *const SegmentDescriptor,
};
pcr.tcb_end = init_percpu();
{
let tss = &pcr.tss.0 as *const _ as usize as u32;
@@ -164,28 +162,11 @@ pub unsafe fn init_paging(stack_offset: usize) {
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
}
// TODO: Share code with x86. Maybe even with aarch64?
/// Copy tdata, clear tbss, calculate TCB end pointer
#[cold]
unsafe fn init_percpu() -> usize {
use crate::kernel_executable_offsets::*;
let size = __tbss_end() - __tdata_start();
assert_eq!(size % PAGE_SIZE, 0);
let tbss_offset = __tbss_start() - __tdata_start();
let base_frame = crate::memory::allocate_frames(size / PAGE_SIZE).expect("failed to allocate percpu memory");
let base = RmmA::phys_to_virt(base_frame.start_address());
let tls_end = base.data() + size;
core::ptr::copy_nonoverlapping(__tdata_start() as *const u8, base.data() as *mut u8, tbss_offset);
core::ptr::write_bytes((base.data() + tbss_offset) as *mut u8, 0, size - tbss_offset);
tls_end
pcr.percpu = crate::percpu::PercpuBlock {
cpu_id,
switch_internals: Default::default(),
};
}
#[derive(Copy, Clone, Debug)]
@@ -228,3 +209,9 @@ impl GdtEntry {
self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F;
}
}
impl crate::percpu::PercpuBlock {
pub fn current() -> &'static Self {
unsafe { &*core::ptr::addr_of!((*pcr()).percpu) }
}
}
src/arch/x86/interrupt/ipi.rs
+2 -6
View File
@@ -1,9 +1,7 @@
use core::sync::atomic::Ordering;
use x86::tlb;
use crate::context;
use crate::device::local_apic::LOCAL_APIC;
use super::irq::PIT_TICKS;
interrupt!(wakeup, || {
LOCAL_APIC.eoi();
@@ -24,8 +22,6 @@ interrupt!(switch, || {
interrupt!(pit, || {
LOCAL_APIC.eoi();
// Switch after 3 ticks (about 6.75 ms)
if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 2 {
let _ = context::switch();
}
// Switch after a sufficent amount of time since the last switch.
context::switch::tick();
});
src/arch/x86/interrupt/irq.rs
+2 -8
View File
@@ -11,10 +11,6 @@ use crate::scheme::debug::{debug_input, debug_notify};
use crate::scheme::serio::serio_input;
use crate::{context, time};
//resets to 0 in context::switch()
#[thread_local]
pub static PIT_TICKS: AtomicUsize = AtomicUsize::new(0);
#[repr(u8)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum IrqMethod {
@@ -149,10 +145,8 @@ interrupt_stack!(pit_stack, |_stack| {
// Any better way of doing this?
timeout::trigger();
// Switch after 3 ticks (about 6.75 ms)
if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 2 {
let _ = context::switch();
}
// Switch after a sufficent amount of time since the last switch.
context::switch::tick();
});
interrupt!(keyboard, || {
src/arch/x86/start.rs
+2 -28
View File
@@ -25,12 +25,6 @@ use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKin
static BSS_TEST_ZERO: usize = 0;
/// Test of non-zero values in data.
static DATA_TEST_NONZERO: usize = usize::max_value();
/// Test of zero values in thread BSS
#[thread_local]
static mut TBSS_TEST_ZERO: usize = 0;
/// Test of non-zero values in thread data.
#[thread_local]
static mut TDATA_TEST_NONZERO: usize = usize::max_value();
pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0);
pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0);
@@ -134,7 +128,7 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
paging::init();
// Set up GDT after paging with TLS
gdt::init_paging(args.stack_base as usize + args.stack_size as usize);
gdt::init_paging(args.stack_base as usize + args.stack_size as usize, 0);
// Set up IDT
idt::init_paging_bsp();
@@ -142,16 +136,6 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
// Set up syscall instruction
interrupt::syscall::init();
// Test tdata and tbss
{
assert_eq!(TBSS_TEST_ZERO, 0);
TBSS_TEST_ZERO += 1;
assert_eq!(TBSS_TEST_ZERO, 1);
assert_eq!(TDATA_TEST_NONZERO, usize::max_value());
TDATA_TEST_NONZERO -= 1;
assert_eq!(TDATA_TEST_NONZERO, usize::max_value() - 1);
}
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
AP_READY.store(false, Ordering::SeqCst);
@@ -237,7 +221,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
paging::init();
// Set up GDT with TLS
gdt::init_paging(stack_end);
gdt::init_paging(stack_end, cpu_id);
// Set up IDT for AP
idt::init_paging_post_heap(false, cpu_id);
@@ -245,16 +229,6 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
// Set up syscall instruction
interrupt::syscall::init();
// Test tdata and tbss
{
assert_eq!(TBSS_TEST_ZERO, 0);
TBSS_TEST_ZERO += 1;
assert_eq!(TBSS_TEST_ZERO, 1);
assert_eq!(TDATA_TEST_NONZERO, usize::max_value());
TDATA_TEST_NONZERO -= 1;
assert_eq!(TDATA_TEST_NONZERO, usize::max_value() - 1);
}
// Initialize devices (for AP)
device::init_ap();
src/arch/x86_64/gdt.rs
+15 -28
View File
@@ -3,7 +3,8 @@
use core::convert::TryInto;
use core::mem;
use crate::paging::{PAGE_SIZE, RmmA, RmmArch};
use crate::paging::{RmmA, RmmArch};
use crate::percpu::PercpuBlock;
use x86::bits64::task::TaskStateSegment;
use x86::Ring;
@@ -73,17 +74,15 @@ const BASE_GDT: [GdtEntry; 8] = [
pub struct ProcessorControlRegion {
// TODO: When both KASLR and KPTI are implemented, the PCR may need to be split into two pages,
// such that "secret" kernel addresses are only stored in the protected half.
pub self_ref: usize,
pub tcb_end: usize,
pub user_rsp_tmp: usize,
// TODO: The I/O permissions bitmap can require more than 8192 bytes of space.
pub tss: TaskStateSegment,
pub self_ref: usize,
// The GDT *must* be stored in the PCR! The paranoid interrupt handler, lacking a reliable way
// to correctly obtain GSBASE, uses SGDT to calculate the PCR offset.
pub gdt: [GdtEntry; 8],
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
pub percpu: PercpuBlock,
}
const _: () = {
@@ -145,7 +144,7 @@ unsafe fn load_segments() {
/// Initialize GDT and PCR.
#[cold]
pub unsafe fn init_paging(stack_offset: usize) {
pub unsafe fn init_paging(stack_offset: usize, cpu_id: usize) {
let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR");
let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion);
@@ -164,8 +163,6 @@ pub unsafe fn init_paging(stack_offset: usize) {
base,
};
pcr.tcb_end = init_percpu();
{
pcr.tss.iomap_base = 0xFFFF;
@@ -213,27 +210,11 @@ pub unsafe fn init_paging(stack_offset: usize) {
x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE);
}
}
/// Copy tdata, clear tbss, calculate TCB end pointer
#[cold]
unsafe fn init_percpu() -> usize {
use crate::kernel_executable_offsets::*;
let size = __tbss_end() - __tdata_start();
assert_eq!(size % PAGE_SIZE, 0);
let tbss_offset = __tbss_start() - __tdata_start();
let base_frame = crate::memory::allocate_frames(size / PAGE_SIZE).expect("failed to allocate percpu memory");
let base = RmmA::phys_to_virt(base_frame.start_address());
let tls_end = base.data() + size;
core::ptr::copy_nonoverlapping(__tdata_start() as *const u8, base.data() as *mut u8, tbss_offset);
core::ptr::write_bytes((base.data() + tbss_offset) as *mut u8, 0, size - tbss_offset);
tls_end
pcr.percpu = PercpuBlock {
cpu_id,
switch_internals: Default::default(),
};
}
#[derive(Copy, Clone, Debug)]
@@ -270,3 +251,9 @@ impl GdtEntry {
self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F;
}
}
impl PercpuBlock {
pub fn current() -> &'static Self {
unsafe { &*core::ptr::addr_of!((*pcr()).percpu) }
}
}
src/arch/x86_64/interrupt/ipi.rs
+2 -6
View File
@@ -1,9 +1,7 @@
use core::sync::atomic::Ordering;
use x86::tlb;
use crate::context;
use crate::device::local_apic::LOCAL_APIC;
use super::irq::PIT_TICKS;
interrupt!(wakeup, || {
LOCAL_APIC.eoi();
@@ -24,8 +22,6 @@ interrupt!(switch, || {
interrupt!(pit, || {
LOCAL_APIC.eoi();
// Switch after 3 ticks (about 6.75 ms)
if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 2 {
let _ = context::switch();
}
// Switch after a sufficient amount of time since the last switch.
context::switch::tick();
});
src/arch/x86_64/interrupt/irq.rs
+2 -8
View File
@@ -11,10 +11,6 @@ use crate::scheme::debug::{debug_input, debug_notify};
use crate::scheme::serio::serio_input;
use crate::{context, time};
//resets to 0 in context::switch()
#[thread_local]
pub static PIT_TICKS: AtomicUsize = AtomicUsize::new(0);
#[repr(u8)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum IrqMethod {
@@ -149,10 +145,8 @@ interrupt_stack!(pit_stack, |_stack| {
// Any better way of doing this?
timeout::trigger();
// Switch after 3 ticks (about 6.75 ms)
if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 2 {
let _ = context::switch();
}
// Switch after a sufficient amount of time since the last switch.
context::switch::tick();
});
interrupt!(keyboard, || {
src/arch/x86_64/start.rs
+2 -29
View File
@@ -3,7 +3,6 @@
/// It must create the IDT with the correct entries, those entries are
/// defined in other files inside of the `arch` module
use core::cell::Cell;
use core::slice;
use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -26,12 +25,6 @@ use crate::paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind};
static BSS_TEST_ZERO: usize = 0;
/// Test of non-zero values in data.
static DATA_TEST_NONZERO: usize = usize::max_value();
/// Test of zero values in thread BSS
#[thread_local]
static TBSS_TEST_ZERO: Cell<usize> = Cell::new(0);
/// Test of non-zero values in thread data.
#[thread_local]
static TDATA_TEST_NONZERO: Cell<usize> = Cell::new(usize::max_value());
pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0);
pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0);
@@ -136,7 +129,7 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
paging::init();
// Set up GDT after paging with TLS
gdt::init_paging(args.stack_base as usize + args.stack_size as usize);
gdt::init_paging(args.stack_base as usize + args.stack_size as usize, 0);
// Set up IDT
idt::init_paging_bsp();
@@ -144,16 +137,6 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
// Set up syscall instruction
interrupt::syscall::init();
// Test tdata and tbss
{
assert_eq!(TBSS_TEST_ZERO.get(), 0);
TBSS_TEST_ZERO.set(TBSS_TEST_ZERO.get() + 1);
assert_eq!(TBSS_TEST_ZERO.get(), 1);
assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value());
TDATA_TEST_NONZERO.set(TDATA_TEST_NONZERO.get() - 1);
assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value() - 1);
}
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
AP_READY.store(false, Ordering::SeqCst);
@@ -242,7 +225,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
paging::init();
// Set up GDT with TLS
gdt::init_paging(stack_end);
gdt::init_paging(stack_end, cpu_id);
// Set up IDT for AP
idt::init_paging_post_heap(false, cpu_id);
@@ -253,16 +236,6 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
// Initialize miscellaneous processor features
misc::init();
// Test tdata and tbss
{
assert_eq!(TBSS_TEST_ZERO.get(), 0);
TBSS_TEST_ZERO.set(TBSS_TEST_ZERO.get() + 1);
assert_eq!(TBSS_TEST_ZERO.get(), 1);
assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value());
TDATA_TEST_NONZERO.set(TDATA_TEST_NONZERO.get() - 1);
assert_eq!(TDATA_TEST_NONZERO.get(), usize::max_value() - 1);
}
// Initialize devices (for AP)
device::init_ap();
src/common/int_like.rs
+1 -1
View File
@@ -25,7 +25,7 @@
#[macro_export]
macro_rules! int_like {
($new_type_name:ident, $backing_type: ident) => {
#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
pub struct $new_type_name($backing_type);
impl $new_type_name {
src/context/list.rs
+1 -2
View File
@@ -1,7 +1,6 @@
use alloc::sync::Arc;
use alloc::collections::BTreeMap;
use core::{iter, mem};
use core::sync::atomic::Ordering;
use spin::RwLock;
@@ -39,7 +38,7 @@ impl ContextList {
/// Get the current context.
pub fn current(&self) -> Option<&Arc<RwLock<Context>>> {
self.map.get(&super::CONTEXT_ID.load(Ordering::SeqCst))
self.map.get(&super::context_id())
}
pub fn iter(&self) -> ::alloc::collections::btree_map::Iter<ContextId, Arc<RwLock<Context>>> {
src/context/mod.rs
+7 -13
View File
@@ -1,7 +1,6 @@
//! # Context management
//!
//! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching)
use core::sync::atomic::Ordering;
use alloc::borrow::Cow;
use alloc::sync::Arc;
@@ -9,6 +8,7 @@ use alloc::sync::Arc;
use spin::{RwLock, RwLockReadGuard, RwLockWriteGuard};
use crate::paging::{RmmA, RmmArch, TableKind};
use crate::percpu::PercpuBlock;
use crate::syscall::error::{Error, ESRCH, Result};
pub use self::context::{BorrowedHtBuf, Context, ContextId, ContextSnapshot, Status, WaitpidKey};
@@ -34,7 +34,7 @@ pub mod context;
mod list;
/// Context switch function
mod switch;
pub mod switch;
/// File struct - defines a scheme and a file number
pub mod file;
@@ -59,9 +59,6 @@ pub const CONTEXT_MAX_FILES: usize = 65_536;
/// Contexts list
static CONTEXTS: RwLock<ContextList> = RwLock::new(ContextList::new());
#[thread_local]
static CONTEXT_ID: context::AtomicContextId = context::AtomicContextId::default();
pub use self::arch::empty_cr3;
pub fn init() {
@@ -77,7 +74,10 @@ pub fn init() {
context.status = Status::Runnable;
context.running = true;
context.cpu_id = Some(crate::cpu_id());
CONTEXT_ID.store(context.id, Ordering::SeqCst);
unsafe {
PercpuBlock::current().switch_internals.set_context_id(context.id);
}
}
/// Get the global schemes list, const
@@ -91,13 +91,7 @@ pub fn contexts_mut() -> RwLockWriteGuard<'static, ContextList> {
}
pub fn context_id() -> ContextId {
// Thread local variables can and should only be modified using Relaxed. This is to prevent a
// hardware thread from racing with itself, for example if there is an interrupt. Orderings
// stronger than Relaxed are only necessary for inter-processor synchronization.
let id = CONTEXT_ID.load(Ordering::Relaxed);
// Prevent the compiler from reordering subsequent loads and stores to before this load.
core::sync::atomic::compiler_fence(Ordering::Acquire);
id
PercpuBlock::current().switch_internals.context_id()
}
pub fn current() -> Result<Arc<RwLock<Context>>> {
src/context/switch.rs
+41 -10
View File
@@ -7,14 +7,15 @@ use alloc::sync::Arc;
use spin::{RwLock, RwLockWriteGuard};
use crate::context::signal::signal_handler;
use crate::context::{arch, contexts, Context, CONTEXT_ID};
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
use crate::context::{arch, contexts, Context};
use crate::gdt;
use crate::interrupt::irq::PIT_TICKS;
use crate::interrupt;
use crate::percpu::PercpuBlock;
use crate::ptrace;
use crate::time;
use super::ContextId;
unsafe fn update_runnable(context: &mut Context, cpu_id: usize) -> bool {
// Ignore already running contexts
if context.running {
@@ -88,11 +89,21 @@ struct SwitchResult {
next_lock: Arc<RwLock<Context>>,
}
#[thread_local]
static SWITCH_RESULT: Cell<Option<SwitchResult>> = Cell::new(None);
pub fn tick() {
let ticks_cell = &PercpuBlock::current().switch_internals.pit_ticks;
let new_ticks = ticks_cell.get() + 1;
ticks_cell.set(new_ticks);
// Switch after 3 ticks (about 6.75 ms)
if new_ticks >= 3 {
let _ = unsafe { switch() };
}
}
pub unsafe extern "C" fn switch_finish_hook() {
if let Some(SwitchResult { prev_lock, next_lock }) = SWITCH_RESULT.take() {
if let Some(SwitchResult { prev_lock, next_lock }) = PercpuBlock::current().switch_internals.switch_result.take() {
prev_lock.force_write_unlock();
next_lock.force_write_unlock();
} else {
@@ -108,11 +119,13 @@ pub unsafe extern "C" fn switch_finish_hook() {
///
/// Do not call this while holding locks!
pub unsafe fn switch() -> bool {
// TODO: Better memory orderings?
let percpu = PercpuBlock::current();
//set PIT Interrupt counter to 0, giving each process same amount of PIT ticks
let _ticks = PIT_TICKS.swap(0, Ordering::SeqCst);
percpu.switch_internals.pit_ticks.set(0);
// Set the global lock to avoid the unsafe operations below from causing issues
// TODO: Better memory orderings?
while arch::CONTEXT_SWITCH_LOCK.compare_exchange_weak(false, true, Ordering::SeqCst, Ordering::Relaxed).is_err() {
interrupt::pause();
}
@@ -177,7 +190,8 @@ pub unsafe fn switch() -> bool {
gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
}
}
CONTEXT_ID.store(next_context.id, Ordering::SeqCst);
let percpu = PercpuBlock::current();
percpu.switch_internals.context_id.set(next_context.id);
if next_context.ksig.is_none() {
//TODO: Allow nested signals
@@ -191,7 +205,7 @@ pub unsafe fn switch() -> bool {
}
}
SWITCH_RESULT.set(Some(SwitchResult {
percpu.switch_internals.switch_result.set(Some(SwitchResult {
prev_lock: prev_context_lock,
next_lock: next_context_lock,
}));
@@ -210,3 +224,20 @@ pub unsafe fn switch() -> bool {
false
}
}
#[derive(Default)]
pub struct ContextSwitchPercpu {
switch_result: Cell<Option<SwitchResult>>,
pit_ticks: Cell<usize>,
/// Unique ID of the currently running context.
context_id: Cell<ContextId>,
}
impl ContextSwitchPercpu {
pub fn context_id(&self) -> ContextId {
self.context_id.get()
}
pub unsafe fn set_context_id(&self, new: ContextId) {
self.context_id.set(new)
}
}
src/lib.rs
+5 -17
View File
@@ -48,12 +48,12 @@
#![feature(iter_array_chunks)]
#![feature(asm_const)] // TODO: Relax requirements of most asm invocations
#![feature(const_option)]
#![feature(const_refs_to_cell)]
#![feature(int_roundings)]
#![feature(let_chains)]
#![feature(naked_functions)]
#![feature(slice_ptr_get, slice_ptr_len)]
#![feature(sync_unsafe_cell)]
#![feature(thread_local)]
#![no_std]
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
@@ -64,13 +64,6 @@ extern crate alloc;
#[macro_use]
extern crate bitflags;
extern crate bitfield;
extern crate goblin;
extern crate linked_list_allocator;
extern crate rustc_demangle;
extern crate spin;
#[cfg(feature = "slab")]
extern crate slab_allocator;
use core::sync::atomic::{AtomicUsize, Ordering};
@@ -125,6 +118,8 @@ pub mod memory;
#[cfg(not(any(feature="doc", test)))]
pub mod panic;
pub mod percpu;
/// Process tracing
pub mod ptrace;
@@ -147,14 +142,10 @@ pub mod tests;
#[global_allocator]
static ALLOCATOR: allocator::Allocator = allocator::Allocator;
/// A unique number that identifies the current CPU - used for scheduling
#[thread_local]
static CPU_ID: AtomicUsize = AtomicUsize::new(0);
/// Get the current CPU's scheduling ID
#[inline(always)]
pub fn cpu_id() -> usize {
CPU_ID.load(Ordering::Relaxed)
crate::percpu::PercpuBlock::current().cpu_id
}
/// The count of all CPUs that can have work scheduled
@@ -185,7 +176,6 @@ static BOOTSTRAP: spin::Once<Bootstrap> = spin::Once::new();
/// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this
pub fn kmain(cpus: usize, bootstrap: Bootstrap) -> ! {
CPU_ID.store(0, Ordering::SeqCst);
CPU_COUNT.store(cpus, Ordering::SeqCst);
//Initialize the first context, stored in kernel/src/context/mod.rs
@@ -226,8 +216,6 @@ pub fn kmain(cpus: usize, bootstrap: Bootstrap) -> ! {
/// This is the main kernel entry point for secondary CPUs
#[allow(unreachable_code, unused_variables)]
pub fn kmain_ap(id: usize) -> ! {
CPU_ID.store(id, Ordering::SeqCst);
if cfg!(feature = "multi_core") {
context::init();
@@ -294,5 +282,5 @@ macro_rules! linker_offsets(
}
);
pub mod kernel_executable_offsets {
linker_offsets!(__text_start, __text_end, __rodata_start, __rodata_end, __data_start, __data_end, __bss_start, __bss_end, __tdata_start, __tdata_end, __tbss_start, __tbss_end, __usercopy_start, __usercopy_end);
linker_offsets!(__text_start, __text_end, __rodata_start, __rodata_end, __data_start, __data_end, __bss_start, __bss_end, __usercopy_start, __usercopy_end);
}
src/percpu.rs
+16
View File
@@ -0,0 +1,16 @@
use crate::context::switch::ContextSwitchPercpu;
/// The percpu block, that stored all percpu variables.
pub struct PercpuBlock {
/// A unique immutable number that identifies the current CPU - used for scheduling
// TODO: Differentiate between logical CPU IDs and hardware CPU IDs (e.g. APIC IDs)
pub cpu_id: usize,
/// Context management
pub switch_internals: ContextSwitchPercpu,
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
}
// PercpuBlock::current() is implemented somewhere in the arch-specific modules
src/syscall/driver.rs
+3 -44
View File
@@ -1,3 +1,5 @@
use alloc::sync::Arc;
use crate::interrupt::InterruptStack;
use crate::memory::{allocate_frames_complex, deallocate_frames, Frame, PAGE_SIZE};
use crate::paging::{PhysicalAddress, VirtualAddress};
@@ -5,10 +7,7 @@ use crate::context;
use crate::scheme::memory::{MemoryScheme, MemoryType};
use crate::syscall::error::{Error, EFAULT, EINVAL, ENOMEM, EPERM, ESRCH, Result};
use crate::syscall::flag::{MapFlags, PhysallocFlags, PartialAllocStrategy, PhysmapFlags};
use alloc::sync::Arc;
use super::usercopy::UserSliceRw;
use crate::syscall::usercopy::UserSliceRw;
fn enforce_root() -> Result<()> {
let contexts = context::contexts();
@@ -101,46 +100,6 @@ pub fn inner_physmap(physical_address: usize, size: usize, flags: PhysmapFlags)
MemoryType::Writeback
};
/*
let end = 1 << 52;
if (physical_address.saturating_add(size) as u64) > end || physical_address % PAGE_SIZE != 0 {
return Err(Error::new(EINVAL));
}
if size % PAGE_SIZE != 0 {
log::warn!("physmap size {} is not multiple of PAGE_SIZE {}", size, PAGE_SIZE);
}
let pages = NonZeroUsize::new(size.div_ceil(PAGE_SIZE)).ok_or(Error::new(EINVAL))?;
let addr_space = Arc::clone(context::current()?.read().addr_space()?);
let mut guard = addr_space.write();
guard.mmap_anywhere(pages, Default::default(), |dst_page, _, dst_mapper, dst_flusher| {
let mut page_flags = PageFlags::new().user(true);
if flags.contains(PHYSMAP_WRITE) {
page_flags = page_flags.write(true);
}
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] // TODO: AARCH64
if flags.contains(PHYSMAP_WRITE_COMBINE) {
page_flags = page_flags.custom_flag(EntryFlags::HUGE_PAGE.bits(), true);
}
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] // TODO: AARCH64
if flags.contains(PHYSMAP_NO_CACHE) {
page_flags = page_flags.custom_flag(EntryFlags::NO_CACHE.bits(), true);
}
Grant::physmap(
Frame::containing_address(PhysicalAddress::new(physical_address)),
PageSpan::new(
dst_page,
pages.get(),
),
page_flags,
dst_mapper,
dst_flusher,
)
}).map(|page| page.start_address().data())
*/
MemoryScheme::physmap(physical_address, size, map_flags, memory_type)
}
pub fn physmap(physical_address: usize, size: usize, flags: PhysmapFlags) -> Result<usize> {
targets/aarch64-unknown-kernel.json
+1 -2
View File
@@ -22,6 +22,5 @@
"exe-suffix": "",
"has-rpath": false,
"no-default-libraries": true,
"position-independent-executables": false,
"tls-model": "global-dynamic"
"position-independent-executables": false
}
targets/i686-unknown-kernel.json
+2 -3
View File
@@ -16,13 +16,12 @@
"features": "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float",
"dynamic-linking": false,
"executables": false,
"relocation-model": "pic",
"relocation-model": "static",
"code-model": "kernel",
"disable-redzone": true,
"frame-pointer": "always",
"exe-suffix": "",
"has-rpath": false,
"no-default-libraries": true,
"position-independent-executables": false,
"tls-model": "global-dynamic"
"position-independent-executables": false
}
targets/x86_64-unknown-kernel.json
+1 -2
View File
@@ -23,6 +23,5 @@
"exe-suffix": "",
"has-rpath": false,
"no-default-libraries": true,
"position-independent-executables": false,
"tls-model": "global-dynamic"
"position-independent-executables": false
}