diff --git a/Cargo.lock b/Cargo.lock index dc61ff05aa..1b0aef1d11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,7 +137,6 @@ dependencies = [ "sbi-rt", "slab", "spin", - "spinning_top 0.3.0", "toml", "x86", ] @@ -148,7 +147,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549ce1740e46b291953c4340adcd74c59bcf4308f4cac050fd33ba91b7168f4a" dependencies = [ - "spinning_top 0.2.5", + "spinning_top", ] [[package]] @@ -265,27 +264,27 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", ] [[package]] name = "serde_core" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", @@ -331,15 +330,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spinning_top" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" -dependencies = [ - "lock_api", -] - [[package]] name = "syn" version = "2.0.106" diff --git a/Cargo.toml b/Cargo.toml index a00db082e9..925e84cbbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ redox_syscall = { git = "https://gitlab.redox-os.org/redox-os/syscall.git", bran rmm = { path = "rmm", default-features = false } slab = { version = "0.4", default-features = false } spin = { version = "0.9.8" } -spinning_top = { version = "0.3", features = ["arc_lock"] } # TODO: Remove indexmap = { version = "2.5.0", default-features = false } diff --git a/src/arch/aarch64/device/generic_timer.rs b/src/arch/aarch64/device/generic_timer.rs index 1cfd63ce62..7d3b27d0b3 100644 --- a/src/arch/aarch64/device/generic_timer.rs +++ b/src/arch/aarch64/device/generic_timer.rs @@ -10,6 +10,7 @@ use crate::{ irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, }, interrupt::irq::trigger, + sync::CleanLockToken, time, }; use fdt::Fdt; @@ -124,18 +125,17 @@ impl GenericTimer { } impl InterruptHandler for GenericTimer { - fn irq_handler(&mut self, irq: u32) { + fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) { self.clear_irq(); { *time::OFFSET.lock() += self.clk_freq as u128; } - timeout::trigger(); - - context::switch::tick(); + timeout::trigger(token); + context::switch::tick(token); unsafe { - trigger(irq); + trigger(irq, token); } self.reload_count(); } diff --git a/src/arch/aarch64/device/irqchip/gic.rs b/src/arch/aarch64/device/irqchip/gic.rs index f4dd47c748..d5aef88c95 100644 --- a/src/arch/aarch64/device/irqchip/gic.rs +++ b/src/arch/aarch64/device/irqchip/gic.rs @@ -1,7 +1,10 @@ use super::InterruptController; -use crate::dtb::{ - get_mmio_address, - irqchip::{InterruptHandler, IrqCell, IrqDesc}, +use crate::{ + dtb::{ + get_mmio_address, + irqchip::{InterruptHandler, IrqCell, IrqDesc}, + }, + sync::CleanLockToken, }; use core::ptr::{read_volatile, write_volatile}; use fdt::{node::FdtNode, Fdt}; @@ -75,7 +78,7 @@ impl GenericInterruptController { } impl InterruptHandler for GenericInterruptController { - fn irq_handler(&mut self, _irq: u32) {} + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {} } impl InterruptController for GenericInterruptController { diff --git a/src/arch/aarch64/device/irqchip/gicv3.rs b/src/arch/aarch64/device/irqchip/gicv3.rs index 7a1837c861..9d8a0d5211 100644 --- a/src/arch/aarch64/device/irqchip/gicv3.rs +++ b/src/arch/aarch64/device/irqchip/gicv3.rs @@ -3,9 +3,12 @@ use core::arch::asm; use fdt::{node::NodeProperty, Fdt}; use super::{gic::GicDistIf, InterruptController}; -use crate::dtb::{ - get_mmio_address, - irqchip::{InterruptHandler, IrqCell, IrqDesc}, +use crate::{ + dtb::{ + get_mmio_address, + irqchip::{InterruptHandler, IrqCell, IrqDesc}, + }, + sync::CleanLockToken, }; use syscall::{ error::{Error, EINVAL}, @@ -74,7 +77,7 @@ impl GicV3 { } impl InterruptHandler for GicV3 { - fn irq_handler(&mut self, _irq: u32) {} + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {} } impl InterruptController for GicV3 { diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs index 2efdcab25a..59e69ec741 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs @@ -2,9 +2,12 @@ use core::ptr::{read_volatile, write_volatile}; use fdt::{node::FdtNode, Fdt}; use super::InterruptController; -use crate::dtb::{ - get_interrupt, get_mmio_address, - irqchip::{InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP}, +use crate::{ + dtb::{ + get_interrupt, get_mmio_address, + irqchip::{InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP}, + }, + sync::CleanLockToken, }; use syscall::{ error::{Error, EINVAL}, @@ -278,14 +281,14 @@ impl InterruptController for Bcm2835ArmInterruptController { } impl InterruptHandler for Bcm2835ArmInterruptController { - fn irq_handler(&mut self, _irq: u32) { + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) { unsafe { let irq = self.irq_ack(); if let Some(virq) = self.irq_to_virq(irq) && virq < 1024 { if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler { - handler.irq_handler(virq as u32); + handler.irq_handler(virq as u32, token); } } else { error!("unexpected irq num {}", irq); diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs index 3c5afcd150..8714cdad83 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs @@ -5,6 +5,7 @@ use crate::{ get_mmio_address, irqchip::{InterruptHandler, IrqCell, IrqDesc}, }, + sync::CleanLockToken, }; use core::{ arch::asm, @@ -116,7 +117,7 @@ impl Bcm2836ArmInterruptController { } impl InterruptHandler for Bcm2836ArmInterruptController { - fn irq_handler(&mut self, _irq: u32) {} + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {} } impl InterruptController for Bcm2836ArmInterruptController { diff --git a/src/arch/aarch64/device/irqchip/null.rs b/src/arch/aarch64/device/irqchip/null.rs index 706e18752a..84fb9c53c4 100644 --- a/src/arch/aarch64/device/irqchip/null.rs +++ b/src/arch/aarch64/device/irqchip/null.rs @@ -5,12 +5,15 @@ use syscall::{ }; use super::InterruptController; -use crate::dtb::irqchip::{InterruptHandler, IrqCell, IrqDesc}; +use crate::{ + dtb::irqchip::{InterruptHandler, IrqCell, IrqDesc}, + sync::CleanLockToken, +}; pub struct Null; impl InterruptHandler for Null { - fn irq_handler(&mut self, _irq: u32) {} + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {} } impl InterruptController for Null { diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs index ff4087474b..3eaa0d3774 100644 --- a/src/arch/aarch64/device/serial.rs +++ b/src/arch/aarch64/device/serial.rs @@ -9,6 +9,7 @@ use crate::{ irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, }, interrupt::irq::trigger, + sync::CleanLockToken, }; use fdt::Fdt; use syscall::Mmio; @@ -18,10 +19,10 @@ pub static COM1: Mutex = Mutex::new(SerialKind::NotPresent); pub struct Com1Irq {} impl InterruptHandler for Com1Irq { - fn irq_handler(&mut self, irq: u32) { - COM1.lock().receive(); + fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) { + COM1.lock().receive(token); unsafe { - trigger(irq); + trigger(irq, token); } } } diff --git a/src/arch/aarch64/interrupt/exception.rs b/src/arch/aarch64/interrupt/exception.rs index da42e5c910..ef093b13d7 100644 --- a/src/arch/aarch64/interrupt/exception.rs +++ b/src/arch/aarch64/interrupt/exception.rs @@ -6,6 +6,7 @@ use crate::{ exception_stack, memory::{ArchIntCtx, GenericPfFlags}, panic::stack_trace, + sync::CleanLockToken, syscall::{self, flag::*}, }; @@ -189,8 +190,9 @@ exception_stack!(synchronous_exception_at_el0, |stack| { match exception_code(stack.iret.esr_el1) { 0b010101 => { let scratch = &stack.scratch; + let mut token = unsafe { CleanLockToken::new() }; let ret = syscall::syscall( - scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, + scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, &mut token, ); stack.scratch.x0 = ret; } diff --git a/src/arch/aarch64/interrupt/irq.rs b/src/arch/aarch64/interrupt/irq.rs index 4ee05c02e8..e5851e1229 100644 --- a/src/arch/aarch64/interrupt/irq.rs +++ b/src/arch/aarch64/interrupt/irq.rs @@ -1,4 +1,7 @@ -use crate::{arch::device::ROOT_IC_IDX, dtb::irqchip::IRQ_CHIP, scheme::irq::irq_trigger}; +use crate::{ + arch::device::ROOT_IC_IDX, dtb::irqchip::IRQ_CHIP, scheme::irq::irq_trigger, + sync::CleanLockToken, +}; use core::sync::atomic::Ordering; // use crate::percpu::PercpuBlock; @@ -12,34 +15,36 @@ unsafe fn irq_ack() -> (u32, Option) { } exception_stack!(irq_at_el0, |_stack| { + let mut token = unsafe { CleanLockToken::new() }; let (irq, virq) = irq_ack(); if let Some(virq) = virq && virq < 1024 { - IRQ_CHIP.trigger_virq(virq as u32); + IRQ_CHIP.trigger_virq(virq as u32, &mut token); } else { println!("unexpected irq num {}", irq); } }); exception_stack!(irq_at_el1, |_stack| { + let mut token = unsafe { CleanLockToken::new() }; let (irq, virq) = irq_ack(); if let Some(virq) = virq && virq < 1024 { - IRQ_CHIP.trigger_virq(virq as u32); + IRQ_CHIP.trigger_virq(virq as u32, &mut token); } else { println!("unexpected irq num {}", irq); } }); //TODO -pub unsafe fn trigger(irq: u32) { +pub unsafe fn trigger(irq: u32, token: &mut CleanLockToken) { unsafe { // FIXME add_irq accepts a u8 as irq number // PercpuBlock::current().stats.add_irq(irq); - irq_trigger(irq.try_into().unwrap()); + irq_trigger(irq.try_into().unwrap(), token); IRQ_CHIP.irq_eoi(irq); } } diff --git a/src/arch/aarch64/stop.rs b/src/arch/aarch64/stop.rs index d477a0d4cb..cab73fbbea 100644 --- a/src/arch/aarch64/stop.rs +++ b/src/arch/aarch64/stop.rs @@ -1,3 +1,4 @@ +use crate::sync::CleanLockToken; use core::arch::asm; pub unsafe fn kreset() -> ! { @@ -20,7 +21,7 @@ pub unsafe fn emergency_reset() -> ! { } } -pub unsafe fn kstop() -> ! { +pub unsafe fn kstop(token: &mut CleanLockToken) -> ! { unsafe { println!("kstop"); diff --git a/src/arch/riscv64/device/irqchip/clint_sbi.rs b/src/arch/riscv64/device/irqchip/clint_sbi.rs index 57aab226e5..ee2b2f39bb 100644 --- a/src/arch/riscv64/device/irqchip/clint_sbi.rs +++ b/src/arch/riscv64/device/irqchip/clint_sbi.rs @@ -2,6 +2,7 @@ use crate::{ context, context::timeout, dtb::irqchip::{register_irq, InterruptHandler, IrqCell, IRQ_CHIP}, + sync::CleanLockToken, }; use alloc::{boxed::Box, vec::Vec}; use byteorder::{ByteOrder, BE}; @@ -34,7 +35,7 @@ struct ClintConnector { } impl InterruptHandler for ClintConnector { - fn irq_handler(&mut self, _irq: u32) { + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) { CLINT .lock() .as_mut() @@ -43,8 +44,8 @@ impl InterruptHandler for ClintConnector { if self.irq == IRQ_TIMER { // a bit of hack, but it is a really bad idea to call scheduler // from inside clint irq handler - timeout::trigger(); - context::switch::tick(); + timeout::trigger(token); + context::switch::tick(token); } } } diff --git a/src/arch/riscv64/device/irqchip/hlic.rs b/src/arch/riscv64/device/irqchip/hlic.rs index dde60f982c..28b04dc8c5 100644 --- a/src/arch/riscv64/device/irqchip/hlic.rs +++ b/src/arch/riscv64/device/irqchip/hlic.rs @@ -1,4 +1,7 @@ -use crate::dtb::irqchip::{InterruptController, InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP}; +use crate::{ + dtb::irqchip::{InterruptController, InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP}, + sync::CleanLockToken, +}; use alloc::vec::Vec; use core::arch::asm; use fdt::{node::NodeProperty, Fdt}; @@ -24,7 +27,7 @@ fn acknowledge(interrupt: usize) { } } -pub unsafe fn interrupt(hart: usize, interrupt: usize) { +pub unsafe fn interrupt(hart: usize, interrupt: usize, token: &mut CleanLockToken) { unsafe { assert!( hart < CPU_INTERRUPT_HANDLERS.len(), @@ -42,13 +45,13 @@ pub unsafe fn interrupt(hart: usize, interrupt: usize) { .unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt)); match &mut IRQ_CHIP.irq_desc[virq].handler { Some(handler) => { - handler.irq_handler(virq as u32); + handler.irq_handler(virq as u32, token); } _ => match IRQ_CHIP.irq_desc[virq].basic.child_ic_idx { Some(ic_idx) => { IRQ_CHIP.irq_chip_list.chips[ic_idx] .ic - .irq_handler(virq as u32); + .irq_handler(virq as u32, token); } _ => { panic!( @@ -83,10 +86,10 @@ impl Hlic { } } impl InterruptHandler for Hlic { - fn irq_handler(&mut self, irq: u32) { + fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) { assert!(irq < 16, "Unsupported HLIC interrupt raised!"); unsafe { - IRQ_CHIP.trigger_virq(self.virq_base as u32 + irq); + IRQ_CHIP.trigger_virq(self.virq_base as u32 + irq, token); } } } diff --git a/src/arch/riscv64/device/irqchip/plic.rs b/src/arch/riscv64/device/irqchip/plic.rs index 67886b991a..4d274bfdc3 100644 --- a/src/arch/riscv64/device/irqchip/plic.rs +++ b/src/arch/riscv64/device/irqchip/plic.rs @@ -4,6 +4,7 @@ use crate::{ get_mmio_address, irqchip::{InterruptController, InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP}, }, + sync::CleanLockToken, }; use core::{mem, num::NonZero, sync::atomic::Ordering}; use fdt::Fdt; @@ -93,12 +94,12 @@ impl Plic { } } impl InterruptHandler for Plic { - fn irq_handler(&mut self, _irq: u32) { + fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) { unsafe { let irq = self.irq_ack(); //println!("PLIC interrupt {}", irq); if let Some(virq) = self.irq_to_virq(irq) { - IRQ_CHIP.trigger_virq(virq as u32); + IRQ_CHIP.trigger_virq(virq as u32, token); } else { error!("unexpected irq num {}", irq); self.irq_eoi(irq); diff --git a/src/arch/riscv64/device/serial.rs b/src/arch/riscv64/device/serial.rs index ca84a3257b..0aa358f535 100644 --- a/src/arch/riscv64/device/serial.rs +++ b/src/arch/riscv64/device/serial.rs @@ -10,6 +10,7 @@ use crate::{ irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, }, scheme::irq::irq_trigger, + sync::CleanLockToken, }; pub static COM1: Mutex = Mutex::new(SerialKind::NotPresent); @@ -17,10 +18,10 @@ pub static COM1: Mutex = Mutex::new(SerialKind::NotPresent); pub struct Com1Irq {} impl InterruptHandler for Com1Irq { - fn irq_handler(&mut self, irq: u32) { - COM1.lock().receive(); + fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) { + COM1.lock().receive(token); unsafe { - irq_trigger(irq as u8); + irq_trigger(irq as u8, token); IRQ_CHIP.irq_eoi(irq); } } diff --git a/src/arch/riscv64/interrupt/exception.rs b/src/arch/riscv64/interrupt/exception.rs index 36019eb7cc..c87b158105 100644 --- a/src/arch/riscv64/interrupt/exception.rs +++ b/src/arch/riscv64/interrupt/exception.rs @@ -8,6 +8,7 @@ use crate::{ memory::GenericPfFlags, panic::stack_trace, ptrace, + sync::CleanLockToken, syscall::{self, flag::*}, }; @@ -146,25 +147,28 @@ unsafe fn handle_system_exception(scause: usize, regs: &InterruptStack) { unsafe fn handle_interrupt(interrupt: usize) { unsafe { + let mut token = CleanLockToken::new(); // FIXME retrieve from percpu area // For now all the interrupts go to boot hart so this suffices... let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed); - irqchip::hlic::interrupt(hart, interrupt); + irqchip::hlic::interrupt(hart, interrupt, &mut token); } } unsafe fn handle_user_exception(scause: usize, regs: &mut InterruptStack) { unsafe { + let mut token = CleanLockToken::new(); + if scause == USERMODE_ECALL { let r = &mut regs.registers; regs.iret.sepc += 4; // skip ecall - let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14); + let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14, &mut token); r.x10 = ret; return; } if scause == BREAKPOINT { - if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_some() { + if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None, &mut token).is_some() { return; } } diff --git a/src/arch/riscv64/stop.rs b/src/arch/riscv64/stop.rs index b0bd5793fd..540883cd0a 100644 --- a/src/arch/riscv64/stop.rs +++ b/src/arch/riscv64/stop.rs @@ -1,3 +1,5 @@ +use crate::sync::CleanLockToken; + pub unsafe fn kreset() -> ! { println!("kreset"); unimplemented!() @@ -7,7 +9,7 @@ pub unsafe fn emergency_reset() -> ! { unimplemented!() } -pub unsafe fn kstop() -> ! { +pub unsafe fn kstop(token: &mut CleanLockToken) -> ! { println!("kstop"); unimplemented!() } diff --git a/src/arch/x86/interrupt/syscall.rs b/src/arch/x86/interrupt/syscall.rs index d020daef61..9d23a5b640 100644 --- a/src/arch/x86/interrupt/syscall.rs +++ b/src/arch/x86/interrupt/syscall.rs @@ -1,13 +1,17 @@ use crate::{ - ptrace, syscall, + ptrace, + sync::CleanLockToken, + syscall, syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL}, }; pub unsafe fn init() {} macro_rules! with_interrupt_stack { - (|$stack:ident| $code:block) => {{ - let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None) + (|$stack:ident, $token:ident| $code:block) => {{ + let mut $token = CleanLockToken::new(); + + let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None, &mut $token) .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE))); if allowed.unwrap_or(true) { @@ -19,12 +23,12 @@ macro_rules! with_interrupt_stack { $code } - ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); + ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None, &mut $token); }}; } interrupt_stack!(syscall, |stack| { - with_interrupt_stack!(|stack| { + with_interrupt_stack!(|stack, token| { let scratch = &stack.scratch; let preserved = &stack.preserved; let ret = syscall::syscall( @@ -34,6 +38,7 @@ interrupt_stack!(syscall, |stack| { scratch.edx, preserved.esi, preserved.edi, + &mut token, ); stack.scratch.eax = ret; }) diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 39a7bc50d5..58275580f4 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -1,6 +1,8 @@ use crate::{ arch::{gdt, interrupt::InterruptStack}, - ptrace, syscall, + ptrace, + sync::CleanLockToken, + syscall, syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL}, }; use core::mem::offset_of; @@ -66,7 +68,8 @@ pub unsafe fn init() { #[unsafe(no_mangle)] pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) { unsafe { - let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None) + let mut token = CleanLockToken::new(); + let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None, &mut token) .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE))); if allowed.unwrap_or(true) { @@ -79,11 +82,12 @@ pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) scratch.rdx, scratch.r10, scratch.r8, + &mut token, ); (*stack).scratch.rax = ret; } - ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); + ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None, &mut token); } } diff --git a/src/arch/x86_shared/interrupt/exception.rs b/src/arch/x86_shared/interrupt/exception.rs index 6e69a1f021..05b0647412 100644 --- a/src/arch/x86_shared/interrupt/exception.rs +++ b/src/arch/x86_shared/interrupt/exception.rs @@ -3,7 +3,8 @@ use x86::irq::PageFaultError; use crate::{ arch::x86_shared::interrupt, context::signal::excp_handler, interrupt_error, interrupt_stack, - memory::GenericPfFlags, paging::VirtualAddress, panic::stack_trace, ptrace, syscall::flag::*, + memory::GenericPfFlags, paging::VirtualAddress, panic::stack_trace, ptrace, + sync::CleanLockToken, syscall::flag::*, }; interrupt_stack!(divide_by_zero, |stack| { @@ -28,7 +29,8 @@ interrupt_stack!(debug, @paranoid, |stack| { let had_singlestep = stack.iret.rflags & (1 << 8) == 1 << 8; stack.set_singlestep(false); - if ptrace::breakpoint_callback(PTRACE_STOP_SINGLESTEP, None).is_some() { + let mut token = unsafe { CleanLockToken::new() }; + if ptrace::breakpoint_callback(PTRACE_STOP_SINGLESTEP, None, &mut token).is_some() { handled = true; } else { // There was no breakpoint, restore original value @@ -78,7 +80,8 @@ interrupt_stack!(breakpoint, |stack| { stack.iret.rip -= 1; } - if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_none() { + let mut token = unsafe { CleanLockToken::new() }; + if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None, &mut token).is_none() { println!("Breakpoint trap"); stack.dump(); excp_handler(Exception { diff --git a/src/arch/x86_shared/interrupt/ipi.rs b/src/arch/x86_shared/interrupt/ipi.rs index f927610abd..74cd40d798 100644 --- a/src/arch/x86_shared/interrupt/ipi.rs +++ b/src/arch/x86_shared/interrupt/ipi.rs @@ -1,4 +1,6 @@ -use crate::{context, device::local_apic::the_local_apic, percpu::PercpuBlock}; +use crate::{ + context, device::local_apic::the_local_apic, percpu::PercpuBlock, sync::CleanLockToken, +}; interrupt!(wakeup, || { unsafe { the_local_apic().eoi() }; @@ -13,12 +15,14 @@ interrupt!(tlb, || { interrupt!(switch, || { unsafe { the_local_apic().eoi() }; - let _ = context::switch(); + let mut token = unsafe { CleanLockToken::new() }; + let _ = context::switch(&mut token); }); interrupt!(pit, || { unsafe { the_local_apic().eoi() }; // Switch after a sufficient amount of time since the last switch. - context::switch::tick(); + let mut token = unsafe { CleanLockToken::new() }; + context::switch::tick(&mut token); }); diff --git a/src/arch/x86_shared/interrupt/irq.rs b/src/arch/x86_shared/interrupt/irq.rs index f69172885b..6b91016983 100644 --- a/src/arch/x86_shared/interrupt/irq.rs +++ b/src/arch/x86_shared/interrupt/irq.rs @@ -12,6 +12,7 @@ use crate::{ ipi::{ipi, IpiKind, IpiTarget}, percpu::PercpuBlock, scheme::{irq::irq_trigger, serio::serio_input}, + sync::CleanLockToken, time, }; @@ -34,7 +35,7 @@ pub fn spurious_count_irq15() -> usize { pub fn spurious_count() -> usize { spurious_count_irq7() + spurious_count_irq15() } -pub fn spurious_irq_resource() -> syscall::Result> { +pub fn spurious_irq_resource(_token: &mut CleanLockToken) -> syscall::Result> { match irq_method() { IrqMethod::Apic => Ok(Vec::from(&b"(not implemented for APIC yet)"[..])), IrqMethod::Pic => Ok(format!( @@ -75,7 +76,8 @@ unsafe fn trigger(irq: u8) { } IrqMethod::Apic => ioapic_mask(irq), } - irq_trigger(irq); + let mut token = CleanLockToken::new(); + irq_trigger(irq, &mut token); } } @@ -175,11 +177,13 @@ interrupt_stack!(pit_stack, |_stack| { // Wake up other CPUs ipi(IpiKind::Pit, IpiTarget::Other); + let mut token = unsafe { CleanLockToken::new() }; + // Any better way of doing this? - timeout::trigger(); + timeout::trigger(&mut token); // Switch after a sufficient amount of time since the last switch. - context::switch::tick(); + context::switch::tick(&mut token); }); interrupt!(keyboard, || { @@ -188,7 +192,8 @@ interrupt!(keyboard, || { unsafe { eoi(1) }; - serio_input(0, data); + let mut token = unsafe { CleanLockToken::new() }; + serio_input(0, data, &mut token); }); interrupt!(cascade, || { @@ -197,12 +202,14 @@ interrupt!(cascade, || { }); interrupt!(com2, || { - COM2.lock().receive(); + let mut token = unsafe { CleanLockToken::new() }; + COM2.lock().receive(&mut token); unsafe { eoi(3) }; }); interrupt!(com1, || { - COM1.lock().receive(); + let mut token = unsafe { CleanLockToken::new() }; + COM1.lock().receive(&mut token); unsafe { eoi(4) }; }); @@ -266,7 +273,8 @@ interrupt!(mouse, || { unsafe { eoi(12) }; - serio_input(1, data); + let mut token = unsafe { CleanLockToken::new() }; + serio_input(1, data, &mut token); }); interrupt!(fpu, || { @@ -326,7 +334,8 @@ macro_rules! allocatable_irq( #[cfg(target_arch = "x86")] pub unsafe fn allocatable_irq_generic(number: u8) { unsafe { - irq_trigger(number - 32); + let mut token = unsafe { CleanLockToken::new() }; + irq_trigger(number - 32, &mut token); lapic_eoi(); } } @@ -336,10 +345,12 @@ default_irqs!((), allocatable_irq); #[cfg(target_arch = "x86_64")] interrupt_error!(generic_irq, |_stack, code| { + let mut token = unsafe { CleanLockToken::new() }; + // The reason why 128 is subtracted and added from the code, is that PUSH imm8 sign-extends the // value, and the longer PUSH imm32 would make the generic_interrupts table twice as large // (containing lots of useless NOPs). - irq_trigger((code as i32).wrapping_add(128) as u8); + irq_trigger((code as i32).wrapping_add(128) as u8, &mut token); unsafe { lapic_eoi() }; }); diff --git a/src/arch/x86_shared/stop.rs b/src/arch/x86_shared/stop.rs index e02972bab4..6187792a6a 100644 --- a/src/arch/x86_shared/stop.rs +++ b/src/arch/x86_shared/stop.rs @@ -1,7 +1,10 @@ #[cfg(feature = "acpi")] use crate::{context, scheme::acpi, time}; -use crate::syscall::io::{Io, Pio}; +use crate::{ + sync::CleanLockToken, + syscall::io::{Io, Pio}, +}; pub unsafe fn kreset() -> ! { unsafe { @@ -56,11 +59,11 @@ pub unsafe fn emergency_reset() -> ! { } #[cfg(feature = "acpi")] -fn userspace_acpi_shutdown() { +fn userspace_acpi_shutdown(token: &mut CleanLockToken) { info!("Notifying any potential ACPI driver"); // Tell whatever driver that handles ACPI, that it should enter the S5 state (i.e. // shutdown). - if !acpi::register_kstop() { + if !acpi::register_kstop(token) { // There was no context to switch to. info!("No ACPI driver was alive to handle shutdown."); return; @@ -76,7 +79,7 @@ fn userspace_acpi_shutdown() { // TODO: Switch directly to whichever process is handling the kstop pipe. We would add an // event flag like EVENT_DIRECT, which has already been suggested for IRQs. // TODO: Waitpid with timeout? Because, what if the ACPI driver would crash? - let _ = context::switch(); + let _ = context::switch(token); let current = time::monotonic(); if current - initial > time::NANOS_PER_SEC { @@ -86,12 +89,12 @@ fn userspace_acpi_shutdown() { } } -pub unsafe fn kstop() -> ! { +pub unsafe fn kstop(token: &mut CleanLockToken) -> ! { unsafe { info!("Running kstop()"); #[cfg(feature = "acpi")] - userspace_acpi_shutdown(); + userspace_acpi_shutdown(token); // Magic shutdown code for bochs and qemu (older versions). for c in "Shutdown".bytes() { diff --git a/src/context/context.rs b/src/context/context.rs index 624971b79a..ee24e14724 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -19,6 +19,7 @@ use crate::{ paging::{RmmA, RmmArch}, percpu::PercpuBlock, scheme::{CallerCtx, FileHandle, SchemeId, SchemeNamespace}, + sync::CleanLockToken, }; use crate::syscall::error::{Error, Result, EAGAIN, EBADF, EEXIST, EINVAL, EMFILE, ESRCH}; @@ -428,11 +429,11 @@ pub struct BorrowedHtBuf { head_and_not_tail: bool, } impl BorrowedHtBuf { - pub fn head() -> Result { + pub fn head(token: &mut CleanLockToken) -> Result { Ok(Self { inner: Some( context::current() - .write() + .write(token.token()) .syscall_head .take() .ok_or(Error::new(EAGAIN))?, @@ -440,11 +441,11 @@ impl BorrowedHtBuf { head_and_not_tail: true, }) } - pub fn tail() -> Result { + pub fn tail(token: &mut CleanLockToken) -> Result { Ok(Self { inner: Some( context::current() - .write() + .write(token.token()) .syscall_tail .take() .ok_or(Error::new(EAGAIN))?, @@ -495,7 +496,9 @@ impl Drop for BorrowedHtBuf { let Some(inner) = self.inner.take() else { return; }; - match context.write() { + //TODO: do not allow drop so lock token can be passed in + let mut token = unsafe { CleanLockToken::new() }; + match context.write(token.token()) { mut context => { (if self.head_and_not_tail { &mut context.syscall_head @@ -871,10 +874,10 @@ impl FdTbl { FileHandle::from(start | UPPER_FDTBL_TAG) } - pub fn force_close_all(&mut self) { + pub fn force_close_all(&mut self, token: &mut CleanLockToken) { for file_opt in self.iter_mut() { if let Some(file) = file_opt.take() { - let _ = file.close(); + let _ = file.close(token); } } self.active_count = 0; diff --git a/src/context/file.rs b/src/context/file.rs index 452e1d5526..6d947910fa 100644 --- a/src/context/file.rs +++ b/src/context/file.rs @@ -3,6 +3,7 @@ use crate::{ event, scheme::{self, SchemeId}, + sync::CleanLockToken, syscall::error::{Error, Result, EBADF}, }; use alloc::sync::Arc; @@ -70,22 +71,22 @@ pub struct FileDescriptor { impl FileDescription { /// Try closing a file, although at this point the description will be destroyed anyway, if /// doing so fails. - pub fn try_close(self) -> Result<()> { + pub fn try_close(self, token: &mut CleanLockToken) -> Result<()> { event::unregister_file(self.scheme, self.number); - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(self.scheme) .ok_or(Error::new(EBADF))? .clone(); - scheme.close(self.number) + scheme.close(self.number, token) } } impl FileDescriptor { - pub fn close(self) -> Result<()> { + pub fn close(self, token: &mut CleanLockToken) -> Result<()> { if let Ok(file) = Arc::try_unwrap(self.description).map(RwLock::into_inner) { - file.try_close()?; + file.try_close(token)?; } Ok(()) } diff --git a/src/context/memory.rs b/src/context/memory.rs index 1d1f507d1e..77bf785771 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -21,6 +21,7 @@ use crate::{ paging::{Page, PageFlags, PageMapper, RmmA, TableKind, VirtualAddress}, percpu::PercpuBlock, scheme::{self, KernelSchemes}, + sync::CleanLockToken, }; use super::{context::HardBlockedReason, file::FileDescription}; @@ -51,7 +52,7 @@ pub struct UnmapResult { pub flags: MunmapFlags, } impl UnmapResult { - pub fn unmap(mut self) -> Result<()> { + pub fn unmap(mut self, token: &mut CleanLockToken) -> Result<()> { let Some(GrantFileRef { base_offset, description, @@ -64,14 +65,13 @@ impl UnmapResult { ref desc => (desc.scheme, desc.number), }; - let funmap_result = scheme::schemes() - .get(scheme_id) - .cloned() + let scheme_opt = scheme::schemes(token.token()).get(scheme_id).cloned(); + let funmap_result = scheme_opt .ok_or(Error::new(ENODEV)) - .and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags)); + .and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags, token)); if let Ok(fd) = Arc::try_unwrap(description) { - fd.into_inner().try_close()?; + fd.into_inner().try_close(token)?; } funmap_result?; @@ -502,7 +502,11 @@ impl AddrSpaceWrapper { } /// Borrows a page from user memory, requiring that the frame be Allocated and read/write. This /// is intended to be used for user-kernel shared memory. - pub fn borrow_frame_enforce_rw_allocated(self: &Arc, page: Page) -> Result { + pub fn borrow_frame_enforce_rw_allocated( + self: &Arc, + page: Page, + token: &mut CleanLockToken, + ) -> Result { let mut guard = self.acquire_write(); let (_start_page, info) = guard.grants.contains(page).ok_or(Error::new(EINVAL))?; @@ -519,8 +523,9 @@ impl AddrSpaceWrapper { { Frame::containing(f) } else { - let (frame, flush, new_guard) = correct_inner(self, guard, page, AccessMode::Write, 0) - .map_err(|_| Error::new(ENOMEM))?; + let (frame, flush, new_guard) = + correct_inner(self, guard, page, AccessMode::Write, 0, token) + .map_err(|_| Error::new(ENOMEM))?; flush.flush(); guard = new_guard; @@ -1333,6 +1338,7 @@ impl Grant { lock: &AddrSpaceWrapper, mapper: &mut PageMapper, flusher: &mut Flusher, + token: &mut CleanLockToken, ) -> Result { if let Some(src) = src { let mut guard = src.addr_space_guard; @@ -1359,6 +1365,7 @@ impl Grant { src_page, AccessMode::Read, 0, + token, ) .map_err(|_| Error::new(EIO))?; guard = new_guard; @@ -1383,6 +1390,7 @@ impl Grant { src_page, AccessMode::Read, 0, + token, ) .map_err(|_| Error::new(EIO))?; guard = new_guard; @@ -2232,6 +2240,9 @@ pub struct Table { impl Drop for AddrSpace { fn drop(&mut self) { + //TODO: DANGER: unmap requires a CleanLockToken, this cheats! + let mut token = unsafe { CleanLockToken::new() }; + for mut grant in core::mem::take(&mut self.grants).into_iter() { // Unpinning the grant is allowed, because pinning only occurs in UserScheme calls to // prevent unmapping the mapped range twice (which would corrupt only the scheme @@ -2245,7 +2256,7 @@ impl Drop for AddrSpace { // the underlying pages (and send some funmaps). let res = grant.unmap(&mut self.table.utable, &mut NopFlusher); - let _ = res.unmap(); + let _ = res.unmap(&mut token); } } } @@ -2372,14 +2383,18 @@ pub unsafe fn copy_frame_to_frame_directly(dst: Frame, src: Frame) { } } -pub fn try_correcting_page_tables(faulting_page: Page, access: AccessMode) -> Result<(), PfError> { +pub fn try_correcting_page_tables( + faulting_page: Page, + access: AccessMode, + token: &mut CleanLockToken, +) -> Result<(), PfError> { let Ok(addr_space_lock) = AddrSpace::current() else { debug!("User page fault without address space being set."); return Err(PfError::Segv); }; let lock = &addr_space_lock; - let (_, flush, _) = correct_inner(lock, lock.acquire_write(), faulting_page, access, 0)?; + let (_, flush, _) = correct_inner(lock, lock.acquire_write(), faulting_page, access, 0, token)?; flush.flush(); @@ -2391,6 +2406,7 @@ fn correct_inner<'l>( faulting_page: Page, access: AccessMode, recursion_level: u32, + token: &mut CleanLockToken, ) -> Result<(Frame, PageFlush, RwLockWriteGuard<'l, AddrSpace>), PfError> { let mut addr_space = &mut *addr_space_guard; let mut flusher = Flusher::with_cpu_set(&mut addr_space.used_by, &addr_space_lock.tlb_ack); @@ -2533,6 +2549,7 @@ fn correct_inner<'l>( src_page, AccessMode::Read, new_recursion_level, + token, )? }; @@ -2610,7 +2627,7 @@ fn correct_inner<'l>( let (scheme_id, scheme_number) = match file_ref.description.read() { ref desc => (desc.scheme, desc.number), }; - let user_inner = scheme::schemes() + let user_inner = scheme::schemes(token.token()) .get(scheme_id) .and_then(|s| { if let KernelSchemes::User(user) = s { @@ -2623,18 +2640,18 @@ fn correct_inner<'l>( let offset = file_ref.base_offset as u64 + (pages_from_grant_start * PAGE_SIZE) as u64; user_inner - .request_fmap(scheme_number, offset, 1, flags) + .request_fmap(scheme_number, offset, 1, flags, token) .unwrap(); let context_lock = crate::context::current(); context_lock - .write() + .write(token.token()) .hard_block(HardBlockedReason::AwaitingMmap { file_ref }); - super::switch(); + super::switch(token); let frame = context_lock - .write() + .write(token.token()) .fmap_ret .take() .ok_or(PfError::NonfatalInternalError)?; @@ -2678,9 +2695,9 @@ pub struct BorrowedFmapSource<'a> { pub addr_space_guard: RwLockWriteGuard<'a, AddrSpace>, } -pub fn handle_notify_files(notify_files: Vec) { +pub fn handle_notify_files(notify_files: Vec, token: &mut CleanLockToken) { for file in notify_files { - let _ = file.unmap(); + let _ = file.unmap(token); } } diff --git a/src/context/mod.rs b/src/context/mod.rs index 9eca916fbe..e51c886037 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -2,12 +2,8 @@ //! //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching) -use core::num::NonZeroUsize; - use alloc::{collections::BTreeSet, sync::Arc}; - -use spin::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use spinning_top::RwSpinlock; +use core::num::NonZeroUsize; use syscall::ENOMEM; use crate::{ @@ -15,6 +11,10 @@ use crate::{ cpu_set::LogicalCpuSet, paging::{RmmA, RmmArch, TableKind}, percpu::PercpuBlock, + sync::{ + ArcRwLockWriteGuard, CleanLockToken, LockToken, RwLock, RwLockReadGuard, RwLockWriteGuard, + L0, L1, L2, + }, syscall::error::{Error, Result}, }; @@ -24,6 +24,9 @@ pub use self::{ switch::switch, }; +pub type ContextLock = RwLock; +pub type ArcContextLockWriteGuard = ArcRwLockWriteGuard; + #[cfg(target_arch = "aarch64")] #[path = "arch/aarch64.rs"] mod arch; @@ -67,9 +70,21 @@ pub use self::arch::empty_cr3; // Set of weak references to all contexts available for scheduling. The only strong references are // the context file descriptors. -static CONTEXTS: RwLock> = RwLock::new(BTreeSet::new()); +static CONTEXTS: RwLock> = RwLock::new(BTreeSet::new()); -pub fn init() { +/// Get the global schemes list, const +pub fn contexts<'a>(token: LockToken<'a, L0>) -> RwLockReadGuard<'a, L1, BTreeSet> { + CONTEXTS.read(token) +} + +/// Get the global schemes list, mutable +pub fn contexts_mut<'a>( + token: LockToken<'a, L0>, +) -> RwLockWriteGuard<'a, L1, BTreeSet> { + CONTEXTS.write(token) +} + +pub fn init(token: &mut CleanLockToken) { let owner = None; // kmain not owned by any fd let mut context = Context::new(owner).expect("failed to create kmain context"); context.sched_affinity = LogicalCpuSet::empty(); @@ -84,11 +99,9 @@ pub fn init() { context.running = true; context.cpu_id = Some(crate::cpu_id()); - let context_lock = Arc::new(RwSpinlock::new(context)); + let context_lock = Arc::new(ContextLock::new(context)); - CONTEXTS - .write() - .insert(ContextRef(Arc::clone(&context_lock))); + contexts_mut(token.token()).insert(ContextRef(Arc::clone(&context_lock))); unsafe { let percpu = PercpuBlock::current(); @@ -99,35 +112,25 @@ pub fn init() { } } -/// Get the global schemes list, const -pub fn contexts() -> RwLockReadGuard<'static, BTreeSet> { - CONTEXTS.read() -} - -/// Get the global schemes list, mutable -pub fn contexts_mut() -> RwLockWriteGuard<'static, BTreeSet> { - CONTEXTS.write() -} - -pub fn current() -> Arc> { +pub fn current() -> Arc { PercpuBlock::current() .switch_internals .with_context(|context| Arc::clone(context)) } -pub fn try_current() -> Option>> { +pub fn try_current() -> Option> { PercpuBlock::current() .switch_internals .try_with_context(|context| context.map(Arc::clone)) } -pub fn is_current(context: &Arc>) -> bool { +pub fn is_current(context: &Arc) -> bool { PercpuBlock::current() .switch_internals .with_context(|current| Arc::ptr_eq(context, current)) } -pub struct ContextRef(pub Arc>); +pub struct ContextRef(pub Arc); impl ContextRef { - pub fn upgrade(&self) -> Option>> { + pub fn upgrade(&self) -> Option> { Some(Arc::clone(&self.0)) } } @@ -154,18 +157,17 @@ pub fn spawn( userspace_allowed: bool, owner_proc_id: Option, func: extern "C" fn(), -) -> Result>> { + token: &mut CleanLockToken, +) -> Result> { let stack = Kstack::new()?; - let context_lock = Arc::try_new(RwSpinlock::new(Context::new(owner_proc_id)?)) + let context_lock = Arc::try_new(ContextLock::new(Context::new(owner_proc_id)?)) .map_err(|_| Error::new(ENOMEM))?; - CONTEXTS - .write() - .insert(ContextRef(Arc::clone(&context_lock))); + contexts_mut(token.token()).insert(ContextRef(Arc::clone(&context_lock))); { - let mut context = context_lock.write(); + let mut context = context_lock.write(token.token()); let _ = context.set_addr_space(Some(AddrSpaceWrapper::new()?)); context .arch diff --git a/src/context/signal.rs b/src/context/signal.rs index 0a0494984d..19f4ebc01f 100644 --- a/src/context/signal.rs +++ b/src/context/signal.rs @@ -1,10 +1,10 @@ use core::sync::atomic::Ordering; -use crate::{context, syscall::flag::SigcontrolFlags}; +use crate::{context, sync::CleanLockToken, syscall::flag::SigcontrolFlags}; -pub fn signal_handler() { +pub fn signal_handler(token: &mut CleanLockToken) { let context_lock = context::current(); - let mut context_guard = context_lock.write(); + let mut context_guard = context_lock.write(token.token()); let context = &mut *context_guard; let being_sigkilled = context.being_sigkilled; @@ -12,7 +12,7 @@ pub fn signal_handler() { if being_sigkilled { drop(context_guard); drop(context_lock); - crate::syscall::process::exit_this_context(None); + crate::syscall::process::exit_this_context(None, token); } /*let thumbs_down = ptrace::breakpoint_callback( @@ -72,9 +72,11 @@ pub fn signal_handler() { ); } pub fn excp_handler(excp: syscall::Exception) { + let mut token = unsafe { CleanLockToken::new() }; + let current = context::current(); - let mut context = current.write(); + let context = current.write(token.token()); let Some(eh) = context.sig.as_ref().and_then(|s| s.excp_handler) else { // TODO: Let procmgr print this? @@ -87,7 +89,7 @@ pub fn excp_handler(excp: syscall::Exception) { drop(context); // TODO: Allow exceptions to be caught by tracer etc, without necessarily exiting the // context (closing files, dropping AddrSpace, etc) - crate::syscall::process::exit_this_context(Some(excp)); + crate::syscall::process::exit_this_context(Some(excp), &mut token); }; // TODO /* diff --git a/src/context/switch.rs b/src/context/switch.rs index 26533a199d..581648f089 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -9,15 +9,16 @@ use core::{ }; use alloc::sync::Arc; -use spinning_top::{guard::ArcRwSpinlockWriteGuard, RwSpinlock}; use syscall::PtraceFlags; use crate::{ - context::{arch, contexts, Context}, + context::{arch, contexts, ArcContextLockWriteGuard, Context, ContextLock}, cpu_set::LogicalCpuId, cpu_stats, percpu::PercpuBlock, - ptrace, time, + ptrace, + sync::CleanLockToken, + time, }; use super::ContextRef; @@ -71,8 +72,8 @@ unsafe fn update_runnable(context: &mut Context, cpu_id: LogicalCpuId) -> Update } struct SwitchResultInner { - _prev_guard: ArcRwSpinlockWriteGuard, - _next_guard: ArcRwSpinlockWriteGuard, + _prev_guard: ArcContextLockWriteGuard, + _next_guard: ArcContextLockWriteGuard, } /// Tick function to update PIT ticks and trigger a context switch if necessary. @@ -81,7 +82,7 @@ struct SwitchResultInner { /// switch if the counter reaches a set threshold (e.g., every 3 ticks). /// /// The function also calls the signal handler after switching contexts. -pub fn tick() { +pub fn tick(token: &mut CleanLockToken) { let ticks_cell = &PercpuBlock::current().switch_internals.pit_ticks; let new_ticks = ticks_cell.get() + 1; @@ -89,8 +90,8 @@ pub fn tick() { // Trigger a context switch after every 3 ticks (approx. 6.75 ms). if new_ticks >= 3 { - switch(); - crate::context::signal::signal_handler(); + switch(token); + crate::context::signal::signal_handler(token); } } @@ -136,7 +137,7 @@ pub enum SwitchResult { /// - `SwitchResult::Switched`: Indicates a successful switch to a new context. /// - `SwitchResult::AllContextsIdle`: Indicates all contexts are idle, and the CPU will switch /// to an idle context. -pub fn switch() -> SwitchResult { +pub fn switch(token: &mut CleanLockToken) -> SwitchResult { let percpu = PercpuBlock::current(); cpu_stats::add_context_switch(); percpu @@ -161,11 +162,12 @@ pub fn switch() -> SwitchResult { let mut switch_context_opt = None; { - let contexts = contexts(); + let contexts = contexts(token.token()); // Lock the previous context. let prev_context_lock = crate::context::current(); - let prev_context_guard = prev_context_lock.write_arc(); + // We are careful not to lock this context twice + let prev_context_guard = unsafe { prev_context_lock.write_arc() }; let idle_context = percpu.switch_internals.idle_context(); @@ -201,7 +203,8 @@ pub fn switch() -> SwitchResult { { // Lock next context - let mut next_context_guard = next_context_lock.write_arc(); + // We are careful not to lock this context twice + let mut next_context_guard = unsafe { next_context_lock.write_arc() }; // Check if the context is runnable and can be switched to. if let UpdateResult::CanSwitch = @@ -234,7 +237,7 @@ pub fn switch() -> SwitchResult { let percpu = PercpuBlock::current(); unsafe { percpu.switch_internals.set_current_context(Arc::clone( - ArcRwSpinlockWriteGuard::rwlock(&next_context_guard), + ArcContextLockWriteGuard::rwlock(&next_context_guard), )); } @@ -319,10 +322,10 @@ pub struct ContextSwitchPercpu { switch_result: Cell>, pit_ticks: Cell, - current_ctxt: RefCell>>>, + current_ctxt: RefCell>>, /// The idle process. - idle_ctxt: RefCell>>>, + idle_ctxt: RefCell>>, pub(crate) being_sigkilled: Cell, } @@ -345,7 +348,7 @@ impl ContextSwitchPercpu { /// /// # Returns /// The result of applying `f` to the current context. - pub fn with_context(&self, f: impl FnOnce(&Arc>) -> T) -> T { + pub fn with_context(&self, f: impl FnOnce(&Arc) -> T) -> T { f(self .current_ctxt .borrow() @@ -360,7 +363,7 @@ impl ContextSwitchPercpu { /// /// # Returns /// The result of applying `f` to the current context if any. - pub fn try_with_context(&self, f: impl FnOnce(Option<&Arc>>) -> T) -> T { + pub fn try_with_context(&self, f: impl FnOnce(Option<&Arc>) -> T) -> T { f(self.current_ctxt.borrow().as_ref()) } @@ -371,7 +374,7 @@ impl ContextSwitchPercpu { /// /// # Parameters /// - `new`: The new context to be set as the current context. - pub unsafe fn set_current_context(&self, new: Arc>) { + pub unsafe fn set_current_context(&self, new: Arc) { *self.current_ctxt.borrow_mut() = Some(new); } @@ -382,7 +385,7 @@ impl ContextSwitchPercpu { /// /// # Parameters /// - `new`: The new context to be set as the idle context. - pub unsafe fn set_idle_context(&self, new: Arc>) { + pub unsafe fn set_idle_context(&self, new: Arc) { *self.idle_ctxt.borrow_mut() = Some(new); } @@ -390,7 +393,7 @@ impl ContextSwitchPercpu { /// /// # Returns /// A reference to the idle context. - pub fn idle_context(&self) -> Arc> { + pub fn idle_context(&self) -> Arc { Arc::clone( self.idle_ctxt .borrow() diff --git a/src/context/timeout.rs b/src/context/timeout.rs index a1c23d5b6b..afc48f32f4 100644 --- a/src/context/timeout.rs +++ b/src/context/timeout.rs @@ -1,9 +1,10 @@ use alloc::collections::VecDeque; -use spin::{Mutex, MutexGuard, Once}; +use spin::Once; use crate::{ event, scheme::SchemeId, + sync::{CleanLockToken, LockToken, Mutex, MutexGuard, L0, L1}, syscall::{ data::TimeSpec, flag::{CLOCK_MONOTONIC, CLOCK_REALTIME, EVENT_READ}, @@ -21,20 +22,26 @@ struct Timeout { type Registry = VecDeque; -static REGISTRY: Once> = Once::new(); +static REGISTRY: Once> = Once::new(); /// Initialize registry, called if needed -fn init_registry() -> Mutex { +fn init_registry() -> Mutex { Mutex::new(Registry::new()) } /// Get the global timeouts list -fn registry() -> MutexGuard<'static, Registry> { - REGISTRY.call_once(init_registry).lock() +fn registry<'a>(token: LockToken<'a, L0>) -> MutexGuard<'a, L1, Registry> { + REGISTRY.call_once(init_registry).lock(token) } -pub fn register(scheme_id: SchemeId, event_id: usize, clock: usize, time: TimeSpec) { - let mut registry = registry(); +pub fn register( + scheme_id: SchemeId, + event_id: usize, + clock: usize, + time: TimeSpec, + token: &mut CleanLockToken, +) { + let mut registry = registry(token.token()); registry.push_back(Timeout { scheme_id, event_id, @@ -43,34 +50,38 @@ pub fn register(scheme_id: SchemeId, event_id: usize, clock: usize, time: TimeSp }); } -pub fn trigger() { - let mut registry = registry(); - +pub fn trigger(token: &mut CleanLockToken) { let mono = time::monotonic(); let real = time::realtime(); let mut i = 0; - while i < registry.len() { - let trigger = match registry[i].clock { - CLOCK_MONOTONIC => { - let time = registry[i].time; - mono >= time - } - CLOCK_REALTIME => { - let time = registry[i].time; - real >= time - } - clock => { - println!("timeout::trigger: unknown clock {}", clock); - true - } - }; + loop { + let mut registry = registry(token.token()); + let timeout = if i < registry.len() { + let trigger = match registry[i].clock { + CLOCK_MONOTONIC => { + let time = registry[i].time; + mono >= time + } + CLOCK_REALTIME => { + let time = registry[i].time; + real >= time + } + clock => { + println!("timeout::trigger: unknown clock {}", clock); + true + } + }; - if trigger { - let timeout = registry.remove(i).unwrap(); - event::trigger(timeout.scheme_id, timeout.event_id, EVENT_READ); + if trigger { + registry.remove(i).unwrap() + } else { + i += 1; + continue; + } } else { - i += 1; - } + break; + }; + event::trigger(timeout.scheme_id, timeout.event_id, EVENT_READ); } } diff --git a/src/debugger.rs b/src/debugger.rs index 4ec8bf5f04..af7e39a9ae 100644 --- a/src/debugger.rs +++ b/src/debugger.rs @@ -1,14 +1,14 @@ use crate::{ - context::Context, + context::{Context, ContextLock}, memory::{get_page_info, the_zeroed_frame, Frame, RefCount}, paging::{RmmA, RmmArch, TableKind, PAGE_SIZE}, + sync::CleanLockToken, }; use alloc::sync::Arc; use hashbrown::{HashMap, HashSet}; -use spinning_top::RwSpinlock; /// Super unsafe due to page table switching and raw pointers! -pub unsafe fn debugger(target_id: Option<*const RwSpinlock>) { +pub unsafe fn debugger(target_id: Option<*const ContextLock>, token: &mut CleanLockToken) { println!("DEBUGGER START"); println!(); @@ -21,7 +21,7 @@ pub unsafe fn debugger(target_id: Option<*const RwSpinlock>) { let old_table = unsafe { RmmA::table(TableKind::User) }; - for context_lock in crate::context::contexts().iter() { + for context_lock in crate::context::contexts(token.token()).iter() { if target_id.map_or(false, |target_id| Arc::as_ptr(&context_lock.0) != target_id) { continue; } diff --git a/src/devices/serial.rs b/src/devices/serial.rs index 7121020dd8..833aeca93f 100644 --- a/src/devices/serial.rs +++ b/src/devices/serial.rs @@ -5,6 +5,7 @@ use syscall::Pio; use crate::{ devices::{uart_16550, uart_pl011}, scheme::debug::{debug_input, debug_notify}, + sync::CleanLockToken, }; #[allow(dead_code)] @@ -31,30 +32,30 @@ impl SerialKind { } } - pub fn receive(&mut self) { + pub fn receive(&mut self, token: &mut CleanLockToken) { //TODO: make PL011 receive work the same way as NS16550 match self { Self::NotPresent => {} #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] Self::Ns16550Pio(inner) => { while let Some(c) = inner.receive() { - debug_input(c); + debug_input(c, token); } - debug_notify(); + debug_notify(token); } Self::Ns16550u8(inner) => { while let Some(c) = inner.receive() { - debug_input(c); + debug_input(c, token); } - debug_notify(); + debug_notify(token); } Self::Ns16550u32(inner) => { while let Some(c) = inner.receive() { - debug_input(c); + debug_input(c, token); } - debug_notify(); + debug_notify(token); } - Self::Pl011(inner) => inner.receive(), + Self::Pl011(inner) => inner.receive(token), } } diff --git a/src/devices/uart_pl011.rs b/src/devices/uart_pl011.rs index 6cb8846f83..5923b9f044 100644 --- a/src/devices/uart_pl011.rs +++ b/src/devices/uart_pl011.rs @@ -2,7 +2,10 @@ use core::ptr; -use crate::scheme::debug::{debug_input, debug_notify}; +use crate::{ + scheme::debug::{debug_input, debug_notify}, + sync::CleanLockToken, +}; bitflags! { /// UARTFR @@ -184,7 +187,7 @@ impl SerialPort { } } - pub fn receive(&mut self) { + pub fn receive(&mut self, token: &mut CleanLockToken) { let mut flags = self.intr_stats(); let chk_flags = UartRisFlags::RTIS | UartRisFlags::RXIS; while (flags & chk_flags).bits() != 0 { @@ -203,13 +206,13 @@ impl SerialPort { } let c = self.read_reg(self.data_reg) as u8; if c != 0 { - debug_input(c); + debug_input(c, token); } } flags = self.intr_stats(); } - debug_notify(); + debug_notify(token); } pub fn send(&mut self, data: u8) { diff --git a/src/dtb/irqchip.rs b/src/dtb/irqchip.rs index 23d3b44dad..485a4bee80 100644 --- a/src/dtb/irqchip.rs +++ b/src/dtb/irqchip.rs @@ -1,12 +1,15 @@ use super::travel_interrupt_ctrl; -use crate::{arch::device::irqchip::new_irqchip, cpu_set::LogicalCpuId, scheme::irq::irq_trigger}; +use crate::{ + arch::device::irqchip::new_irqchip, cpu_set::LogicalCpuId, scheme::irq::irq_trigger, + sync::CleanLockToken, +}; use alloc::{boxed::Box, vec::Vec}; use byteorder::{ByteOrder, BE}; use fdt::{node::NodeProperty, Fdt}; use syscall::{Error, Result, EINVAL}; pub trait InterruptHandler { - fn irq_handler(&mut self, irq: u32); + fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken); } #[derive(Debug, Copy, Clone)] @@ -294,18 +297,18 @@ impl IrqChipCore { self.irq_chip_list.chips[ic_idx].ic.irq_xlate(irq_data) } - pub fn trigger_virq(&mut self, virq: u32) { + pub fn trigger_virq(&mut self, virq: u32, token: &mut CleanLockToken) { if virq < 1024 { let desc = &mut self.irq_desc[virq as usize]; match &mut desc.handler { Some(handler) => { - handler.irq_handler(virq); + handler.irq_handler(virq, token); } _ => { if let Some(ic_idx) = desc.basic.child_ic_idx { - self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq); + self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq, token); } else { - irq_trigger(virq as u8); + irq_trigger(virq as u8, token); } } } diff --git a/src/event.rs b/src/event.rs index 6727401014..6edbd2fe1d 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1,12 +1,14 @@ use alloc::sync::Arc; use core::sync::atomic::{AtomicUsize, Ordering}; use hashbrown::HashMap; -use spin::{Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use spin::Once; use crate::{ context, scheme::{self, SchemeId}, - sync::WaitQueue, + sync::{ + CleanLockToken, LockToken, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0, L1, + }, syscall::{ data::Event, error::{Error, Result, EBADF}, @@ -30,15 +32,16 @@ impl EventQueue { } } - pub fn read(&self, buf: UserSliceWo, block: bool) -> Result { - self.queue.receive_into_user(buf, block, "EventQueue::read") + pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result { + self.queue + .receive_into_user(buf, block, "EventQueue::read", token) } - pub fn write(&self, events: &[Event]) -> Result { + pub fn write(&self, events: &[Event], token: &mut CleanLockToken) -> Result { for event in events { let file = { let context_ref = context::current(); - let context = context_ref.read(); + let context = context_ref.read(token.token()); let files = context.files.read(); match files.get(event.id).ok_or(Error::new(EBADF))? { @@ -62,7 +65,7 @@ impl EventQueue { event.flags, ); - let flags = sync(RegKey { scheme, number })?; + let flags = sync(RegKey { scheme, number }, token)?; if !flags.is_empty() { trigger(scheme, number, flags); } @@ -83,21 +86,21 @@ pub fn next_queue_id() -> EventQueueId { } // Current event queues -static QUEUES: Once> = Once::new(); +static QUEUES: Once> = Once::new(); /// Initialize queues, called if needed -fn init_queues() -> RwLock { +fn init_queues() -> RwLock { RwLock::new(HashMap::new()) } /// Get the event queues list, const -pub fn queues() -> RwLockReadGuard<'static, EventQueueList> { - QUEUES.call_once(init_queues).read() +pub fn queues<'a>(token: LockToken<'a, L0>) -> RwLockReadGuard<'a, L1, EventQueueList> { + QUEUES.call_once(init_queues).read(token) } /// Get the event queues list, mutable -pub fn queues_mut() -> RwLockWriteGuard<'static, EventQueueList> { - QUEUES.call_once(init_queues).write() +pub fn queues_mut<'a>(token: LockToken<'a, L0>) -> RwLockWriteGuard<'a, L1, EventQueueList> { + QUEUES.call_once(init_queues).write(token) } #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -115,20 +118,20 @@ pub struct QueueKey { type Registry = HashMap>; -static REGISTRY: Once> = Once::new(); +static REGISTRY: Once> = Once::new(); /// Initialize registry, called if needed -fn init_registry() -> RwLock { - RwLock::new(Registry::new()) +fn init_registry() -> spin::RwLock { + spin::RwLock::new(Registry::new()) } /// Get the global schemes list, const -fn registry() -> RwLockReadGuard<'static, Registry> { +fn registry() -> spin::RwLockReadGuard<'static, Registry> { REGISTRY.call_once(init_registry).read() } /// Get the global schemes list, mutable -pub fn registry_mut() -> RwLockWriteGuard<'static, Registry> { +pub fn registry_mut() -> spin::RwLockWriteGuard<'static, Registry> { REGISTRY.call_once(init_registry).write() } @@ -144,7 +147,7 @@ pub fn register(reg_key: RegKey, queue_key: QueueKey, flags: EventFlags) { } } -pub fn sync(reg_key: RegKey) -> Result { +pub fn sync(reg_key: RegKey, token: &mut CleanLockToken) -> Result { let mut flags = EventFlags::empty(); { @@ -157,12 +160,12 @@ pub fn sync(reg_key: RegKey) -> Result { } } - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(reg_key.scheme) .ok_or(Error::new(EBADF))? .clone(); - scheme.fevent(reg_key.number, flags) + scheme.fevent(reg_key.number, flags, token) } pub fn unregister_file(scheme: SchemeId, number: usize) { @@ -177,19 +180,27 @@ pub fn unregister_file(scheme: SchemeId, number: usize) { // } pub fn trigger(scheme: SchemeId, number: usize, flags: EventFlags) { - let registry = registry(); + //TODO: propogate this lock token + let mut token = unsafe { CleanLockToken::new() }; + let registry = registry(); if let Some(queue_list) = registry.get(&RegKey { scheme, number }) { for (queue_key, &queue_flags) in queue_list.iter() { let common_flags = flags & queue_flags; if !common_flags.is_empty() { - let queues = queues(); - if let Some(queue) = queues.get(&queue_key.queue) { - queue.queue.send(Event { - id: queue_key.id, - flags: common_flags, - data: queue_key.data, - }); + let queue_opt = { + let queues = queues(token.token()); + queues.get(&queue_key.queue).cloned() + }; + if let Some(queue) = queue_opt { + queue.queue.send( + Event { + id: queue_key.id, + flags: common_flags, + data: queue_key.data, + }, + &mut token, + ); } } } diff --git a/src/main.rs b/src/main.rs index 7c8586b245..b34e1e2ce7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -137,6 +137,7 @@ mod scheme; mod startup; /// Synchronization primitives +use sync::CleanLockToken; mod sync; /// Syscall handlers @@ -168,8 +169,9 @@ fn init_env() -> &'static [u8] { } extern "C" fn userspace_init() { + let mut token = unsafe { CleanLockToken::new() }; let bootstrap = crate::BOOTSTRAP.get().expect("BOOTSTRAP was not set"); - unsafe { crate::syscall::process::usermode_bootstrap(bootstrap) } + unsafe { crate::syscall::process::usermode_bootstrap(bootstrap, &mut token) } } struct Bootstrap { @@ -181,8 +183,10 @@ static BOOTSTRAP: spin::Once = spin::Once::new(); /// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this fn kmain(bootstrap: Bootstrap) -> ! { + let mut token = unsafe { CleanLockToken::new() }; + //Initialize the first context, stored in kernel/src/context/mod.rs - context::init(); + context::init(&mut token); //Initialize global schemes, such as `acpi:`. scheme::init_globals(); @@ -196,9 +200,9 @@ fn kmain(bootstrap: Bootstrap) -> ! { profiling::ready_for_profiling(); let owner = None; // kmain not owned by any fd - match context::spawn(true, owner, userspace_init) { + match context::spawn(true, owner, userspace_init, &mut token) { Ok(context_lock) => { - let mut context = context_lock.write(); + let mut context = context_lock.write(token.token()); context.status = context::Status::Runnable; context.name.clear(); context.name.push_str("[bootstrap]"); @@ -213,12 +217,14 @@ fn kmain(bootstrap: Bootstrap) -> ! { } } - run_userspace() + run_userspace(&mut token) } /// This is the main kernel entry point for secondary CPUs #[allow(unreachable_code, unused_variables, dead_code)] fn kmain_ap(cpu_id: crate::cpu_set::LogicalCpuId) -> ! { + let mut token = unsafe { CleanLockToken::new() }; + #[cfg(feature = "profiling")] profiling::maybe_run_profiling_helper_forever(cpu_id); @@ -232,20 +238,21 @@ fn kmain_ap(cpu_id: crate::cpu_set::LogicalCpuId) -> ! { } } } - context::init(); + + context::init(&mut token); info!("AP {}", cpu_id); #[cfg(feature = "profiling")] profiling::ready_for_profiling(); - run_userspace(); + run_userspace(&mut token); } -fn run_userspace() -> ! { +fn run_userspace(token: &mut CleanLockToken) -> ! { loop { unsafe { interrupt::disable(); - match context::switch() { + match context::switch(token) { SwitchResult::Switched => { interrupt::enable_and_nop(); } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index a5d2dbd6a5..6461bd4cc2 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -21,6 +21,7 @@ use crate::{ }, kernel_executable_offsets::{__usercopy_end, __usercopy_start}, paging::{entry::EntryFlags, Page, PageFlags}, + sync::CleanLockToken, syscall::error::{Error, ENOMEM}, }; use rmm::{BumpAllocator, FrameAllocator, FrameCount, FrameUsage, TableKind, VirtualAddress}; @@ -1014,7 +1015,8 @@ pub fn page_fault_handler( } if address_is_user && (caused_by_user || is_usercopy) { - match context::memory::try_correcting_page_tables(faulting_page, mode) { + let mut token = unsafe { CleanLockToken::new() }; + match context::memory::try_correcting_page_tables(faulting_page, mode, &mut token) { Ok(()) => return Ok(()), Err(PfError::Oom) => todo!("oom"), Err(PfError::Segv | PfError::RecursionLimitExceeded) => (), diff --git a/src/panic.rs b/src/panic.rs index ff527f40f7..f74e6a6b39 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -18,6 +18,7 @@ use crate::{ arch::{consts::USER_END_OFFSET, interrupt::trace::StackTrace}, context, cpu_id, interrupt, memory::KernelMapper, + sync::CleanLockToken, syscall, }; @@ -44,9 +45,10 @@ fn rust_begin_unwind(info: &PanicInfo) -> ! { println!("CPU {}, CID {:p}", cpu_id(), context_lock); - // This could deadlock, but at this point we are going to halt anyways { - let context = context_lock.read(); + // This could deadlock, but at this point we are going to halt anyways + let mut token = unsafe { CleanLockToken::new() }; + let context = context_lock.read(token.token()); println!("NAME: {}, DEBUG ID: {}", context.name, context.debug_id); if let Some([a, b, c, d, e, f]) = context.current_syscall() { diff --git a/src/ptrace.rs b/src/ptrace.rs index d555a5c02a..187b2b374f 100644 --- a/src/ptrace.rs +++ b/src/ptrace.rs @@ -6,7 +6,7 @@ use crate::{ event, percpu::PercpuBlock, scheme::GlobalSchemes, - sync::WaitCondition, + sync::{CleanLockToken, WaitCondition}, syscall::{data::PtraceEvent, error::*, flag::*, ptrace_event}, }; @@ -104,9 +104,9 @@ impl Session { /// Remove the session from the list of open sessions and notify any /// waiting processes // TODO -pub fn close_session(session: &Session) { - session.tracer.notify(); - session.tracee.notify(); +pub fn close_session(session: &Session, token: &mut CleanLockToken) { + session.tracer.notify(token); + session.tracee.notify(token); } /// Wake up the tracer to make sure it catches on that the tracee is dead. This @@ -115,8 +115,8 @@ pub fn close_session(session: &Session) { /// session will *actually* be closed. This is partly to ensure ENOSRCH is /// returned rather than ENODEV (which occurs when there's no session - should /// never really happen). -pub fn close_tracee(session: &Session) { - session.tracer.notify(); +pub fn close_tracee(session: &Session, token: &mut CleanLockToken) { + session.tracer.notify(token); let data = session.data.lock(); proc_trigger_event(data.file_id, EVENT_READ); @@ -130,7 +130,7 @@ fn proc_trigger_event(file_id: usize, flags: EventFlags) { /// Dispatch an event to any tracer tracing `self`. This will cause /// the tracer to wake up and poll for events. Returns Some(()) if an /// event was sent. -pub fn send_event(event: PtraceEvent) -> Option<()> { +pub fn send_event(event: PtraceEvent, token: &mut CleanLockToken) -> Option<()> { let session = Session::current()?; let mut data = session.data.lock(); let breakpoint = data.breakpoint.as_ref()?; @@ -142,7 +142,7 @@ pub fn send_event(event: PtraceEvent) -> Option<()> { // Add event to queue data.add_event(event); // Notify tracer - session.tracer.notify(); + session.tracer.notify(token); Some(()) } @@ -165,7 +165,7 @@ pub(crate) struct Breakpoint { /// /// Note: Don't call while holding any locks or allocated data, this will /// switch contexts and may in fact just never terminate. -pub fn wait(session: Arc) -> Result<()> { +pub fn wait(session: Arc, token: &mut CleanLockToken) -> Result<()> { loop { // Lock the data, to make sure we're reading the final value before going // to sleep. @@ -178,7 +178,7 @@ pub fn wait(session: Arc) -> Result<()> { // Go to sleep, and drop the lock on our data, which will allow other the // tracer to wake us up. - if session.tracer.wait(data, "ptrace::wait") { + if session.tracer.wait(data, "ptrace::wait", token) { // We successfully waited, wake up! break; } @@ -196,6 +196,7 @@ pub fn wait(session: Arc) -> Result<()> { pub fn breakpoint_callback( match_flags: PtraceFlags, event: Option, + token: &mut CleanLockToken, ) -> Option { loop { let percpu = PercpuBlock::current(); @@ -221,9 +222,12 @@ pub fn breakpoint_callback( data.add_event(event.unwrap_or(ptrace_event!(match_flags))); // Wake up sleeping tracer - session.tracer.notify(); + session.tracer.notify(token); - if session.tracee.wait(data, "ptrace::breakpoint_callback") { + if session + .tracee + .wait(data, "ptrace::breakpoint_callback", token) + { // We successfully waited, wake up! break Some(breakpoint.flags); } diff --git a/src/scheme/acpi.rs b/src/scheme/acpi.rs index 7c09c88fdd..7f81cf015c 100644 --- a/src/scheme/acpi.rs +++ b/src/scheme/acpi.rs @@ -7,7 +7,7 @@ use core::{ use alloc::boxed::Box; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use spin::{Mutex, Once, RwLock}; +use spin::{Mutex, Once}; use syscall::{ dirent::{DirEntry, DirentBuf, DirentKind}, EIO, @@ -17,7 +17,7 @@ use crate::{ acpi::{RxsdtEnum, RXSDT_ENUM}, context::file::InternalFlags, event, - sync::WaitCondition, + sync::{CleanLockToken, RwLock, WaitCondition, L1}, }; use crate::syscall::{ @@ -35,18 +35,19 @@ use super::{CallerCtx, GlobalSchemes, KernelScheme, OpenResult}; /// A scheme used to access the RSDT or XSDT, which is needed for e.g. `acpid` to function. pub struct AcpiScheme; +#[derive(Clone, Copy)] struct Handle { kind: HandleKind, stat: bool, } -#[derive(Eq, PartialEq)] +#[derive(Clone, Copy, Eq, PartialEq)] enum HandleKind { TopLevel, Rxsdt, ShutdownPipe, } -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); static NEXT_FD: AtomicUsize = AtomicUsize::new(0); @@ -55,11 +56,11 @@ static DATA: Once> = Once::new(); static KSTOP_WAITCOND: WaitCondition = WaitCondition::new(); static KSTOP_FLAG: Mutex = Mutex::new(false); -pub fn register_kstop() -> bool { +pub fn register_kstop(token: &mut CleanLockToken) -> bool { *KSTOP_FLAG.lock() = true; - let mut waiters_awoken = KSTOP_WAITCOND.notify(); + let mut waiters_awoken = KSTOP_WAITCOND.notify(token); - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); for (&fd, _) in handles .iter() @@ -109,7 +110,13 @@ impl AcpiScheme { } impl KernelScheme for AcpiScheme { - fn kopen(&self, path: &str, flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let path = path.trim_start_matches('/'); if ctx.uid != 0 { @@ -148,7 +155,7 @@ impl KernelScheme for AcpiScheme { }; let fd = NEXT_FD.fetch_add(1, atomic::Ordering::Relaxed); - let mut handles_guard = HANDLES.write(); + let mut handles_guard = HANDLES.write(token.token()); let _ = handles_guard.insert( fd, @@ -161,8 +168,8 @@ impl KernelScheme for AcpiScheme { Ok(OpenResult::SchemeLocal(fd, int_flags)) } - fn fsize(&self, id: usize) -> Result { - let mut handles = HANDLES.write(); + fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { + let mut handles = HANDLES.write(token.token()); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; if handle.stat { @@ -176,8 +183,13 @@ impl KernelScheme for AcpiScheme { }) } // TODO - fn fevent(&self, id: usize, _flags: EventFlags) -> Result { - let handles = HANDLES.read(); + fn fevent( + &self, + id: usize, + _flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { + let handles = HANDLES.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; if handle.stat { @@ -186,8 +198,8 @@ impl KernelScheme for AcpiScheme { Ok(EventFlags::empty()) } - fn close(&self, id: usize) -> Result<()> { - if HANDLES.write().remove(&id).is_none() { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + if HANDLES.write(token.token()).remove(&id).is_none() { return Err(Error::new(EBADF)); } Ok(()) @@ -199,13 +211,16 @@ impl KernelScheme for AcpiScheme { offset: u64, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let Ok(offset) = usize::try_from(offset) else { return Ok(0); }; - let mut handles = HANDLES.write(); - let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; + let handle = { + let handles = HANDLES.read(token.token()); + *handles.get(&id).ok_or(Error::new(EBADF))? + }; if handle.stat { return Err(Error::new(EBADF)); @@ -222,7 +237,7 @@ impl KernelScheme for AcpiScheme { if *flag_guard { break; - } else if !KSTOP_WAITCOND.wait(flag_guard, "waiting for kstop") { + } else if !KSTOP_WAITCOND.wait(flag_guard, "waiting for kstop", token) { return Err(Error::new(EINTR)); } } @@ -246,11 +261,12 @@ impl KernelScheme for AcpiScheme { buf: UserSliceWo, header_size: u16, opaque: u64, + token: &mut CleanLockToken, ) -> Result { let Some(Handle { kind: HandleKind::TopLevel, .. - }) = HANDLES.read().get(&id) + }) = HANDLES.read(token.token()).get(&id) else { return Err(Error::new(ENOTDIR)); }; @@ -274,8 +290,8 @@ impl KernelScheme for AcpiScheme { } Ok(buf.finalize()) } - fn kfstat(&self, id: usize, buf: UserSliceWo) -> Result<()> { - let handles = HANDLES.read(); + fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let handles = HANDLES.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; buf.copy_exactly(&match handle.kind { diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index d0cefa6c72..be592498b8 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -1,12 +1,11 @@ use core::sync::atomic::{AtomicUsize, Ordering}; -use spin::RwLock; use crate::{ devices::graphical_debug, event, log::Writer, scheme::*, - sync::WaitQueue, + sync::{CleanLockToken, RwLock, WaitQueue, L1}, syscall::{ flag::{EventFlags, EVENT_READ, O_NONBLOCK}, usercopy::{UserSliceRo, UserSliceWo}, @@ -23,17 +22,17 @@ struct Handle { num: usize, } -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); /// Add to the input queue -pub fn debug_input(data: u8) { - INPUT.send(data); +pub fn debug_input(data: u8, token: &mut CleanLockToken) { + INPUT.send(data, token); } // Notify readers of input updates -pub fn debug_notify() { - for (id, _handle) in HANDLES.read().iter() { +pub fn debug_notify(token: &mut CleanLockToken) { + for (id, _handle) in HANDLES.read(token.token()).iter() { event::trigger(GlobalSchemes::Debug.scheme_id(), *id, EVENT_READ); } } @@ -51,7 +50,13 @@ enum SpecialFds { } impl KernelScheme for DebugScheme { - fn kopen(&self, path: &str, _flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { if ctx.uid != 0 { return Err(Error::new(EPERM)); } @@ -75,40 +80,52 @@ impl KernelScheme for DebugScheme { }; let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, Handle { num }); + HANDLES.write(token.token()).insert(id, Handle { num }); Ok(OpenResult::SchemeLocal(id, InternalFlags::empty())) } - fn fevent(&self, id: usize, _flags: EventFlags) -> Result { + fn fevent( + &self, + id: usize, + _flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { let _handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; Ok(EventFlags::empty()) } - fn fsync(&self, id: usize) -> Result<()> { + fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let _handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; Ok(()) } - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let _handle = { - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); handles.remove(&id).ok_or(Error::new(EBADF))? }; Ok(()) } - fn kread(&self, id: usize, buf: UserSliceWo, flags: u32, _stored_flags: u32) -> Result { + fn kread( + &self, + id: usize, + buf: UserSliceWo, + flags: u32, + _stored_flags: u32, + token: &mut CleanLockToken, + ) -> Result { let handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; @@ -129,7 +146,12 @@ impl KernelScheme for DebugScheme { ); } - INPUT.receive_into_user(buf, flags & O_NONBLOCK as u32 == 0, "DebugScheme::read") + INPUT.receive_into_user( + buf, + flags & O_NONBLOCK as u32 == 0, + "DebugScheme::read", + token, + ) } fn kwrite( @@ -138,9 +160,10 @@ impl KernelScheme for DebugScheme { buf: UserSliceRo, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; @@ -186,9 +209,9 @@ impl KernelScheme for DebugScheme { Ok(buf.len()) } - fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { + fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { let handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; if handle.num != SpecialFds::Default as usize diff --git a/src/scheme/dtb.rs b/src/scheme/dtb.rs index bbdc4ddbcd..b1adf0fc70 100644 --- a/src/scheme/dtb.rs +++ b/src/scheme/dtb.rs @@ -2,12 +2,13 @@ use core::sync::atomic::{self, AtomicUsize}; use alloc::boxed::Box; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use spin::{Once, RwLock}; +use spin::Once; use super::{CallerCtx, KernelScheme, OpenResult}; use crate::{ dtb::DTB_BINARY, scheme::InternalFlags, + sync::{CleanLockToken, RwLock, L1}, syscall::{ data::Stat, error::*, @@ -28,7 +29,7 @@ struct Handle { stat: bool, } -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); static NEXT_FD: AtomicUsize = AtomicUsize::new(0); static DATA: Once> = Once::new(); @@ -50,13 +51,19 @@ impl DtbScheme { } impl KernelScheme for DtbScheme { - fn kopen(&self, path: &str, _flags: usize, _ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let path = path.trim_matches('/'); if path.is_empty() { let id = NEXT_FD.fetch_add(1, atomic::Ordering::Relaxed); - let mut handles_guard = HANDLES.write(); + let mut handles_guard = HANDLES.write(token.token()); let _ = handles_guard.insert( id, @@ -71,8 +78,8 @@ impl KernelScheme for DtbScheme { Err(Error::new(ENOENT)) } - fn fsize(&self, id: usize) -> Result { - let mut handles = HANDLES.write(); + fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { + let mut handles = HANDLES.write(token.token()); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; if handle.stat { @@ -86,8 +93,8 @@ impl KernelScheme for DtbScheme { Ok(file_len as u64) } - fn close(&self, id: usize) -> Result<()> { - if HANDLES.write().remove(&id).is_none() { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + if HANDLES.write(token.token()).remove(&id).is_none() { return Err(Error::new(EBADF)); } Ok(()) @@ -100,8 +107,9 @@ impl KernelScheme for DtbScheme { offset: u64, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; if handle.stat { @@ -120,8 +128,8 @@ impl KernelScheme for DtbScheme { dst_buf.copy_common_bytes_from_slice(src_buf) } - fn kfstat(&self, id: usize, buf: UserSliceWo) -> Result<()> { - let handles = HANDLES.read(); + fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let handles = HANDLES.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; buf.copy_exactly(&match handle.kind { HandleKind::RawData => { diff --git a/src/scheme/event.rs b/src/scheme/event.rs index 1ebdcb4ab8..88ab9c6321 100644 --- a/src/scheme/event.rs +++ b/src/scheme/event.rs @@ -5,6 +5,7 @@ use syscall::O_NONBLOCK; use crate::{ context::file::InternalFlags, event::{next_queue_id, queues, queues_mut, EventQueue, EventQueueId}, + sync::CleanLockToken, syscall::{ data::Event, error::*, @@ -17,30 +18,43 @@ use super::{CallerCtx, KernelScheme, OpenResult}; pub struct EventScheme; impl KernelScheme for EventScheme { - fn kopen(&self, _path: &str, _flags: usize, _ctx: CallerCtx) -> Result { + fn kopen( + &self, + _path: &str, + _flags: usize, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let id = next_queue_id(); - queues_mut().insert(id, Arc::new(EventQueue::new(id))); + queues_mut(token.token()).insert(id, Arc::new(EventQueue::new(id))); Ok(OpenResult::SchemeLocal(id.get(), InternalFlags::empty())) } - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let id = EventQueueId::from(id); - queues_mut() + queues_mut(token.token()) .remove(&id) .ok_or(Error::new(EBADF)) .and(Ok(())) } - fn kread(&self, id: usize, buf: UserSliceWo, flags: u32, _stored_flags: u32) -> Result { + fn kread( + &self, + id: usize, + buf: UserSliceWo, + flags: u32, + _stored_flags: u32, + token: &mut CleanLockToken, + ) -> Result { let id = EventQueueId::from(id); let queue = { - let handles = queues(); + let handles = queues(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; handle.clone() }; - queue.read(buf, flags & O_NONBLOCK as u32 == 0) + queue.read(buf, flags & O_NONBLOCK as u32 == 0, token) } fn kwrite( @@ -49,11 +63,12 @@ impl KernelScheme for EventScheme { buf: UserSliceRo, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let id = EventQueueId::from(id); let queue = { - let handles = queues(); + let handles = queues(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -61,7 +76,7 @@ impl KernelScheme for EventScheme { for chunk in buf.in_exact_chunks(mem::size_of::()) { let event = unsafe { chunk.read_exact::()? }; - if queue.write(&[event])? == 0 { + if queue.write(&[event], token)? == 0 { break; } events_written += 1; @@ -70,7 +85,7 @@ impl KernelScheme for EventScheme { Ok(events_written * mem::size_of::()) } - fn kfpath(&self, _id: usize, buf: UserSliceWo) -> Result { + fn kfpath(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result { buf.copy_common_bytes_from_slice(b"event:") } } diff --git a/src/scheme/irq.rs b/src/scheme/irq.rs index 20cbd1d332..eada055de8 100644 --- a/src/scheme/irq.rs +++ b/src/scheme/irq.rs @@ -10,7 +10,7 @@ use core::{ use alloc::{string::String, vec::Vec}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use spin::{Mutex, Once, RwLock}; +use spin::{Mutex, Once}; use syscall::dirent::{DirEntry, DirentBuf, DirentKind}; use crate::context::file::InternalFlags; @@ -23,6 +23,7 @@ use crate::dtb::irqchip::{acknowledge, available_irqs_iter, is_reserved, set_res use crate::{ cpu_set::LogicalCpuId, event, + sync::{CleanLockToken, RwLock, L1}, syscall::{ data::Stat, error::*, @@ -34,7 +35,7 @@ use crate::{ /// /// IRQ queues pub(super) static COUNTS: Mutex<[usize; 224]> = Mutex::new([0; 224]); -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); /// These are IRQs 0..=15 (corresponding to interrupt vectors 32..=47). They are opened without the @@ -55,11 +56,11 @@ const INO_BSP: u64 = 0x8001_0000_0000_0000; const INO_PHANDLE: u64 = 0x8003_0000_0000_0000; /// Add to the input queue -pub fn irq_trigger(irq: u8) { +pub fn irq_trigger(irq: u8, token: &mut CleanLockToken) { COUNTS.lock()[irq as usize] += 1; for (fd, _) in HANDLES - .read() + .read(token.token()) .iter() .filter_map(|(fd, handle)| Some((fd, handle.as_irq_handle()?))) .filter(|&(_, (_, handle_irq))| handle_irq == irq) @@ -207,7 +208,13 @@ const fn vector_to_irq(vector: u8) -> u8 { } impl crate::scheme::KernelScheme for IrqScheme { - fn kopen(&self, path: &str, flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { if ctx.uid != 0 { return Err(Error::new(EACCES)); } @@ -297,7 +304,7 @@ impl crate::scheme::KernelScheme for IrqScheme { } }; let fd = NEXT_FD.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(fd, handle); + HANDLES.write(token.token()).insert(fd, handle); Ok(OpenResult::SchemeLocal(fd, int_flags)) } fn getdents( @@ -306,6 +313,7 @@ impl crate::scheme::KernelScheme for IrqScheme { buf: UserSliceWo, header_size: u16, opaque_id_start: u64, + token: &mut CleanLockToken, ) -> Result { let Ok(opaque) = usize::try_from(opaque_id_start) else { return Ok(0); @@ -316,7 +324,11 @@ impl crate::scheme::KernelScheme for IrqScheme { let mut buf = DirentBuf::new(buf, header_size).ok_or(Error::new(EIO))?; let mut intermediate = String::new(); - match *HANDLES.read().get(&id).ok_or(Error::new(EBADF))? { + match *HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + { Handle::TopLevel => { let cpus = CPUS.get().expect("IRQ scheme not initialized"); @@ -362,20 +374,31 @@ impl crate::scheme::KernelScheme for IrqScheme { Ok(buf.finalize()) } - fn fcntl(&self, _id: usize, _cmd: usize, _arg: usize) -> Result { + fn fcntl( + &self, + _id: usize, + _cmd: usize, + _arg: usize, + _token: &mut CleanLockToken, + ) -> Result { Ok(0) } - fn fevent(&self, _id: usize, _flags: EventFlags) -> Result { + fn fevent( + &self, + _id: usize, + _flags: EventFlags, + _token: &mut CleanLockToken, + ) -> Result { Ok(EventFlags::empty()) } - fn fsync(&self, _file: usize) -> Result<()> { + fn fsync(&self, _file: usize, _token: &mut CleanLockToken) -> Result<()> { Ok(()) } - fn close(&self, id: usize) -> Result<()> { - let handles_guard = HANDLES.read(); + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + let handles_guard = HANDLES.read(token.token()); let handle = handles_guard.get(&id).ok_or(Error::new(EBADF))?; if let &Handle::Irq { @@ -394,8 +417,9 @@ impl crate::scheme::KernelScheme for IrqScheme { buffer: UserSliceRo, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { - let handles_guard = HANDLES.read(); + let handles_guard = HANDLES.read(token.token()); let handle = handles_guard.get(&file).ok_or(Error::new(EBADF))?; match handle { @@ -422,8 +446,8 @@ impl crate::scheme::KernelScheme for IrqScheme { } } - fn kfstat(&self, id: usize, buf: UserSliceWo) -> Result<()> { - let handles_guard = HANDLES.read(); + fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let handles_guard = HANDLES.read(token.token()); let handle = handles_guard.get(&id).ok_or(Error::new(EBADF))?; buf.copy_exactly(&match *handle { @@ -472,8 +496,8 @@ impl crate::scheme::KernelScheme for IrqScheme { Ok(()) } - fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { - let handles_guard = HANDLES.read(); + fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { + let handles_guard = HANDLES.read(token.token()); let handle = handles_guard.get(&id).ok_or(Error::new(EBADF))?; let scheme_path = match handle { @@ -494,8 +518,9 @@ impl crate::scheme::KernelScheme for IrqScheme { _offset: u64, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { - let handles_guard = HANDLES.read(); + let handles_guard = HANDLES.read(token.token()); let handle = handles_guard.get(&file).ok_or(Error::new(EBADF))?; match *handle { diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs index e0bf73399d..31f18dfc14 100644 --- a/src/scheme/memory.rs +++ b/src/scheme/memory.rs @@ -10,6 +10,7 @@ use crate::{ }, memory::{free_frames, used_frames, Frame, PAGE_SIZE}, paging::VirtualAddress, + sync::CleanLockToken, syscall::usercopy::UserSliceRw, }; @@ -76,6 +77,7 @@ impl MemoryScheme { addr_space: &Arc, map: &Map, is_phys_contiguous: bool, + token: &mut CleanLockToken, ) -> Result { let span = PageSpan::validate_nonempty(VirtualAddress::new(map.address), map.size) .ok_or(Error::new(EINVAL))?; @@ -110,7 +112,7 @@ impl MemoryScheme { }, )?; - handle_notify_files(notify_files); + handle_notify_files(notify_files, token); Ok(page.start_address().data()) } @@ -179,7 +181,13 @@ impl MemoryScheme { } } impl KernelScheme for MemoryScheme { - fn kopen(&self, path: &str, _flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + ctx: CallerCtx, + _token: &mut CleanLockToken, + ) -> Result { if path.len() > 64 { return Err(Error::new(ENOENT)); } @@ -237,6 +245,7 @@ impl KernelScheme for MemoryScheme { payload: UserSliceRw, _flags: syscall::CallFlags, _metadata: &[u64], + _token: &mut CleanLockToken, ) -> Result { let (handle_ty, _, _) = u32::try_from(id) .ok() @@ -268,6 +277,7 @@ impl KernelScheme for MemoryScheme { addr_space: &Arc, map: &Map, _consume: bool, + token: &mut CleanLockToken, ) -> Result { let (handle_ty, mem_ty, flags) = u32::try_from(id) .ok() @@ -279,12 +289,13 @@ impl KernelScheme for MemoryScheme { addr_space, map, flags.contains(HandleFlags::PHYS_CONTIGUOUS), + token, ), HandleTy::PhysBorrow => Self::physmap(map.offset, map.size, map.flags, mem_ty), HandleTy::Translation => Err(Error::new(EOPNOTSUPP)), } } - fn kfstatvfs(&self, _file: usize, dst: UserSliceWo) -> Result<()> { + fn kfstatvfs(&self, _file: usize, dst: UserSliceWo, _token: &mut CleanLockToken) -> Result<()> { let used = used_frames() as u64; let free = free_frames() as u64; diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 9c778785b9..926fe59bf6 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -12,7 +12,7 @@ use alloc::{boxed::Box, string::ToString, sync::Arc, vec::Vec}; use core::{hash::BuildHasherDefault, sync::atomic::AtomicUsize}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use indexmap::IndexMap; -use spin::{Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use spin::Once; use syscall::{CallFlags, EventFlags, MunmapFlags}; use crate::{ @@ -20,6 +20,7 @@ use crate::{ file::{FileDescription, InternalFlags}, memory::AddrSpaceWrapper, }, + sync::{CleanLockToken, LockToken, RwLock, RwLockReadGuard, RwLockWriteGuard, L0, L1}, syscall::{ error::*, usercopy::{UserSliceRo, UserSliceRw, UserSliceWo}, @@ -361,26 +362,32 @@ impl SchemeList { } /// Schemes list -static SCHEMES: Once> = Once::new(); +static SCHEMES: Once> = Once::new(); /// Initialize schemes, called if needed -fn init_schemes() -> RwLock { +fn init_schemes() -> RwLock { RwLock::new(SchemeList::new()) } /// Get the global schemes list, const -pub fn schemes() -> RwLockReadGuard<'static, SchemeList> { - SCHEMES.call_once(init_schemes).read() +pub fn schemes<'a>(token: LockToken<'a, L0>) -> RwLockReadGuard<'a, L1, SchemeList> { + SCHEMES.call_once(init_schemes).read(token) } /// Get the global schemes list, mutable -pub fn schemes_mut() -> RwLockWriteGuard<'static, SchemeList> { - SCHEMES.call_once(init_schemes).write() +pub fn schemes_mut<'a>(token: LockToken<'a, L0>) -> RwLockWriteGuard<'a, L1, SchemeList> { + SCHEMES.call_once(init_schemes).write(token) } #[allow(unused_variables)] pub trait KernelScheme: Send + Sync + 'static { - fn kopen(&self, path: &str, flags: usize, _ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + flags: usize, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { Err(Error::new(ENOENT)) } @@ -391,6 +398,7 @@ pub trait KernelScheme: Send + Sync + 'static { flags: usize, fcntl_flags: u32, _ctx: CallerCtx, + token: &mut CleanLockToken, ) -> Result { Err(Error::new(EOPNOTSUPP)) } @@ -401,14 +409,28 @@ pub trait KernelScheme: Send + Sync + 'static { addr_space: &Arc, map: &crate::syscall::data::Map, consume: bool, + token: &mut CleanLockToken, ) -> Result { Err(Error::new(EOPNOTSUPP)) } - fn kfunmap(&self, number: usize, offset: usize, size: usize, flags: MunmapFlags) -> Result<()> { + fn kfunmap( + &self, + number: usize, + offset: usize, + size: usize, + flags: MunmapFlags, + token: &mut CleanLockToken, + ) -> Result<()> { Err(Error::new(EOPNOTSUPP)) } - fn kdup(&self, old_id: usize, buf: UserSliceRo, _caller: CallerCtx) -> Result { + fn kdup( + &self, + old_id: usize, + buf: UserSliceRo, + _caller: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { Err(Error::new(EOPNOTSUPP)) } fn kwriteoff( @@ -418,11 +440,12 @@ pub trait KernelScheme: Send + Sync + 'static { offset: u64, flags: u32, stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { if offset != u64::MAX { return Err(Error::new(ESPIPE)); } - self.kwrite(id, buf, flags, stored_flags) + self.kwrite(id, buf, flags, stored_flags, token) } fn kreadoff( &self, @@ -431,28 +454,43 @@ pub trait KernelScheme: Send + Sync + 'static { offset: u64, flags: u32, stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { if offset != u64::MAX { return Err(Error::new(ESPIPE)); } - self.kread(id, buf, flags, stored_flags) + self.kread(id, buf, flags, stored_flags, token) } - fn kwrite(&self, id: usize, buf: UserSliceRo, flags: u32, stored_flags: u32) -> Result { + fn kwrite( + &self, + id: usize, + buf: UserSliceRo, + flags: u32, + stored_flags: u32, + token: &mut CleanLockToken, + ) -> Result { Err(Error::new(EBADF)) } - fn kread(&self, id: usize, buf: UserSliceWo, flags: u32, stored_flags: u32) -> Result { + fn kread( + &self, + id: usize, + buf: UserSliceWo, + flags: u32, + stored_flags: u32, + token: &mut CleanLockToken, + ) -> Result { Err(Error::new(EBADF)) } - fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { + fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { Err(Error::new(EBADF)) } - fn kfutimens(&self, id: usize, buf: UserSliceRo) -> Result { + fn kfutimens(&self, id: usize, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { Err(Error::new(EBADF)) } - fn kfstat(&self, id: usize, buf: UserSliceWo) -> Result<()> { + fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - fn kfstatvfs(&self, id: usize, buf: UserSliceWo) -> Result<()> { + fn kfstatvfs(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } @@ -462,47 +500,83 @@ pub trait KernelScheme: Send + Sync + 'static { buf: UserSliceWo, header_size: u16, opaque_id_first: u64, + token: &mut CleanLockToken, ) -> Result { Err(Error::new(EOPNOTSUPP)) } - fn fsync(&self, id: usize) -> Result<()> { + fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { Ok(()) } - fn ftruncate(&self, id: usize, len: usize) -> Result<()> { + fn ftruncate(&self, id: usize, len: usize, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - fn fsize(&self, id: usize) -> Result { + fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { Err(Error::new(ESPIPE)) } - fn legacy_seek(&self, id: usize, pos: isize, whence: usize) -> Option> { + fn legacy_seek( + &self, + id: usize, + pos: isize, + whence: usize, + token: &mut CleanLockToken, + ) -> Option> { None } - fn fchmod(&self, id: usize, new_mode: u16) -> Result<()> { + fn fchmod(&self, id: usize, new_mode: u16, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(EBADF)) } - fn fchown(&self, id: usize, new_uid: u32, new_gid: u32) -> Result<()> { + fn fchown( + &self, + id: usize, + new_uid: u32, + new_gid: u32, + token: &mut CleanLockToken, + ) -> Result<()> { Err(Error::new(EBADF)) } - fn fevent(&self, id: usize, flags: EventFlags) -> Result { + fn fevent( + &self, + id: usize, + flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { Ok(EventFlags::empty()) } - fn flink(&self, id: usize, new_path: &str, caller_ctx: CallerCtx) -> Result<()> { + fn flink( + &self, + id: usize, + new_path: &str, + caller_ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result<()> { Err(Error::new(EBADF)) } - fn frename(&self, id: usize, new_path: &str, caller_ctx: CallerCtx) -> Result<()> { + fn frename( + &self, + id: usize, + new_path: &str, + caller_ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result<()> { Err(Error::new(EBADF)) } - fn fcntl(&self, id: usize, cmd: usize, arg: usize) -> Result { + fn fcntl( + &self, + id: usize, + cmd: usize, + arg: usize, + token: &mut CleanLockToken, + ) -> Result { Ok(0) } - fn rmdir(&self, path: &str, ctx: CallerCtx) -> Result<()> { + fn rmdir(&self, path: &str, ctx: CallerCtx, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(ENOENT)) } - fn unlink(&self, path: &str, ctx: CallerCtx) -> Result<()> { + fn unlink(&self, path: &str, ctx: CallerCtx, token: &mut CleanLockToken) -> Result<()> { Err(Error::new(ENOENT)) } - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { Ok(()) } fn kcall( @@ -511,16 +585,18 @@ pub trait KernelScheme: Send + Sync + 'static { payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { Err(Error::new(EOPNOTSUPP)) } fn kfdwrite( &self, id: usize, - descs: Vec>>, + descs: Vec>>, flags: CallFlags, args: u64, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { Err(Error::new(EOPNOTSUPP)) } @@ -530,6 +606,7 @@ pub trait KernelScheme: Send + Sync + 'static { payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { Err(Error::new(EOPNOTSUPP)) } @@ -538,7 +615,7 @@ pub trait KernelScheme: Send + Sync + 'static { #[derive(Debug)] pub enum OpenResult { SchemeLocal(usize, InternalFlags), - External(Arc>), + External(Arc>), } pub struct CallerCtx { pub pid: usize, diff --git a/src/scheme/pipe.rs b/src/scheme/pipe.rs index b230a7a87a..ba1cbab3bd 100644 --- a/src/scheme/pipe.rs +++ b/src/scheme/pipe.rs @@ -1,14 +1,12 @@ -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - use alloc::{collections::VecDeque, sync::Arc}; - +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use spin::{Mutex, RwLock}; +use spin::Mutex; use crate::{ context::file::InternalFlags, event, - sync::WaitCondition, + sync::{CleanLockToken, RwLock, WaitCondition, L1}, syscall::{ data::Stat, error::{Error, Result, EAGAIN, EBADF, EINTR, EINVAL, ENOENT, EPIPE}, @@ -24,7 +22,7 @@ use super::{CallerCtx, GlobalSchemes, KernelScheme, OpenResult, StrOrBytes}; static PIPE_NEXT_ID: AtomicUsize = AtomicUsize::new(1); // TODO: SLOB? -static PIPES: RwLock>> = +static PIPES: RwLock>> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); const MAX_QUEUE_SIZE: usize = 65536; @@ -37,10 +35,10 @@ fn from_raw_id(id: usize) -> (bool, usize) { (id & WRITE_NOT_READ_BIT != 0, id & !WRITE_NOT_READ_BIT) } -pub fn pipe() -> Result<(usize, usize)> { +pub fn pipe(token: &mut CleanLockToken) -> Result<(usize, usize)> { let id = PIPE_NEXT_ID.fetch_add(1, Ordering::Relaxed); - PIPES.write().insert( + PIPES.write(token.token()).insert( id, Arc::new(Pipe { queue: Mutex::new(VecDeque::new()), @@ -58,9 +56,19 @@ pub fn pipe() -> Result<(usize, usize)> { pub struct PipeScheme; impl KernelScheme for PipeScheme { - fn fevent(&self, id: usize, flags: EventFlags) -> Result { + fn fevent( + &self, + id: usize, + flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { let (is_writer_not_reader, key) = from_raw_id(id); - let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); + let pipe = Arc::clone( + PIPES + .read(token.token()) + .get(&key) + .ok_or(Error::new(EBADF))?, + ); let mut ready = EventFlags::empty(); @@ -81,36 +89,47 @@ impl KernelScheme for PipeScheme { Ok(ready) } - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let (is_write_not_read, key) = from_raw_id(id); - let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); + let pipe = Arc::clone( + PIPES + .read(token.token()) + .get(&key) + .ok_or(Error::new(EBADF))?, + ); let scheme_id = GlobalSchemes::Pipe.scheme_id(); let can_remove = if is_write_not_read { event::trigger(scheme_id, key, EVENT_READ); - pipe.read_condition.notify(); + pipe.read_condition.notify(token); pipe.writer_is_alive.store(false, Ordering::SeqCst); !pipe.reader_is_alive.load(Ordering::SeqCst) } else { event::trigger(scheme_id, key | WRITE_NOT_READ_BIT, EVENT_WRITE); - pipe.write_condition.notify(); + pipe.write_condition.notify(token); pipe.reader_is_alive.store(false, Ordering::SeqCst); !pipe.writer_is_alive.load(Ordering::SeqCst) }; if can_remove { - let _ = PIPES.write().remove(&key); + let _ = PIPES.write(token.token()).remove(&key); } Ok(()) } - fn kdup(&self, old_id: usize, user_buf: UserSliceRo, _ctx: CallerCtx) -> Result { + fn kdup( + &self, + old_id: usize, + user_buf: UserSliceRo, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let (is_writer_not_reader, key) = from_raw_id(old_id); if is_writer_not_reader { @@ -123,7 +142,12 @@ impl KernelScheme for PipeScheme { return Err(Error::new(EINVAL)); } - let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); + let pipe = Arc::clone( + PIPES + .read(token.token()) + .get(&key) + .ok_or(Error::new(EBADF))?, + ); if pipe.has_run_dup.swap(true, Ordering::SeqCst) { return Err(Error::new(EBADF)); @@ -134,12 +158,18 @@ impl KernelScheme for PipeScheme { InternalFlags::empty(), )) } - fn kopen(&self, path: &str, _flags: usize, _ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { if !path.trim_start_matches('/').is_empty() { return Err(Error::new(ENOENT)); } - let (read_id, _) = pipe()?; + let (read_id, _) = pipe(token)?; Ok(OpenResult::SchemeLocal(read_id, InternalFlags::empty())) } @@ -151,6 +181,7 @@ impl KernelScheme for PipeScheme { _flags: usize, _fcntl_flags: u32, _ctx: CallerCtx, + token: &mut CleanLockToken, ) -> Result { let (_, key) = from_raw_id(id); @@ -159,7 +190,12 @@ impl KernelScheme for PipeScheme { return Err(Error::new(EINVAL)); } - let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); + let pipe = Arc::clone( + PIPES + .read(token.token()) + .get(&key) + .ok_or(Error::new(EBADF))?, + ); if pipe.has_run_dup.swap(true, Ordering::SeqCst) { return Err(Error::new(EBADF)); @@ -177,13 +213,19 @@ impl KernelScheme for PipeScheme { user_buf: UserSliceWo, fcntl_flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let (is_write_not_read, key) = from_raw_id(id); if is_write_not_read { return Err(Error::new(EBADF)); } - let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); + let pipe = Arc::clone( + PIPES + .read(token.token()) + .get(&key) + .ok_or(Error::new(EBADF))?, + ); loop { let mut vec = pipe.queue.lock(); @@ -211,7 +253,7 @@ impl KernelScheme for PipeScheme { key | WRITE_NOT_READ_BIT, EVENT_WRITE, ); - pipe.write_condition.notify(); + pipe.write_condition.notify(token); return Ok(bytes_read); } else if user_buf.is_empty() { @@ -222,7 +264,7 @@ impl KernelScheme for PipeScheme { return Ok(0); } else if fcntl_flags & O_NONBLOCK as u32 != 0 { return Err(Error::new(EAGAIN)); - } else if !pipe.read_condition.wait(vec, "PipeRead::read") { + } else if !pipe.read_condition.wait(vec, "PipeRead::read", token) { return Err(Error::new(EINTR)); } } @@ -233,13 +275,19 @@ impl KernelScheme for PipeScheme { user_buf: UserSliceRo, fcntl_flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let (is_write_not_read, key) = from_raw_id(id); if !is_write_not_read { return Err(Error::new(EBADF)); } - let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); + let pipe = Arc::clone( + PIPES + .read(token.token()) + .get(&key) + .ok_or(Error::new(EBADF))?, + ); loop { let mut vec = pipe.queue.lock(); @@ -272,7 +320,7 @@ impl KernelScheme for PipeScheme { if bytes_written > 0 { event::trigger(GlobalSchemes::Pipe.scheme_id(), key, EVENT_READ); - pipe.read_condition.notify(); + pipe.read_condition.notify(token); return Ok(bytes_written); } else if user_buf.is_empty() { @@ -281,12 +329,12 @@ impl KernelScheme for PipeScheme { if fcntl_flags & O_NONBLOCK as u32 != 0 { return Err(Error::new(EAGAIN)); - } else if !pipe.write_condition.wait(vec, "PipeWrite::write") { + } else if !pipe.write_condition.wait(vec, "PipeWrite::write", token) { return Err(Error::new(EINTR)); } } } - fn kfstat(&self, _id: usize, buf: UserSliceWo) -> Result<()> { + fn kfstat(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<()> { buf.copy_exactly(&Stat { st_mode: MODE_FIFO | 0o666, ..Default::default() diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 2c052ac62d..7a0cc8e97b 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -5,11 +5,12 @@ use crate::{ context::{HardBlockedReason, SignalState}, file::InternalFlags, memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, - Context, Status, + Context, ContextLock, Status, }, memory::{get_page_info, AddRefError, RefKind, PAGE_SIZE}, ptrace, scheme::{self, FileHandle, KernelScheme}, + sync::{CleanLockToken, RwLock, L1}, syscall::{ data::{GrantDesc, Map, SetSighandlerData, Stat}, error::*, @@ -39,8 +40,6 @@ use hashbrown::{ hash_map::{DefaultHashBuilder, Entry}, HashMap, }; -use spin::RwLock; -use spinning_top::RwSpinlock; fn read_from(dst: UserSliceWo, src: &[u8], offset: u64) -> Result { let avail_src = usize::try_from(offset) @@ -51,15 +50,16 @@ fn read_from(dst: UserSliceWo, src: &[u8], offset: u64) -> Result { } fn try_stop_context( - context_ref: Arc>, + context_ref: Arc, + token: &mut CleanLockToken, callback: impl FnOnce(&mut Context) -> Result, ) -> Result { if context::is_current(&context_ref) { - return callback(&mut *context_ref.write()); + return callback(&mut *context_ref.write(token.token())); } // Stop process let (prev_status, mut running) = { - let mut context = context_ref.write(); + let mut context = context_ref.write(token.token()); ( core::mem::replace( @@ -74,12 +74,12 @@ fn try_stop_context( // Wait until stopped while running { - context::switch(); + context::switch(token); - running = context_ref.read().running; + running = context_ref.read(token.token()).running; } - let mut context = context_ref.write(); + let mut context = context_ref.write(token.token()); assert!( !context.running, "process can't have been restarted, we stopped it!" @@ -113,11 +113,11 @@ enum ContextHandle { Sighandler, Start, NewFiletable { - filetable: Arc>, + filetable: Arc>, data: Box<[u8]>, }, Filetable { - filetable: Weak>, + filetable: Weak>, data: Box<[u8]>, }, AddrSpace { @@ -134,7 +134,7 @@ enum ContextHandle { CurrentFiletable, AwaitingFiletableChange { - new_ft: Arc>, + new_ft: Arc>, }, // TODO: Remove this once openat is implemented, or allow openat-via-dup via e.g. the top-level @@ -146,19 +146,19 @@ enum ContextHandle { } #[derive(Clone)] struct Handle { - context: Arc>, + context: Arc, kind: ContextHandle, } pub struct ProcScheme; static NEXT_ID: AtomicUsize = AtomicUsize::new(1); -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); #[cfg(feature = "debugger")] #[allow(dead_code)] -pub fn foreach_addrsp(mut f: impl FnMut(&Arc)) { - for (_, handle) in HANDLES.read().iter() { +pub fn foreach_addrsp(token: &mut CleanLockToken, mut f: impl FnMut(&Arc)) { + for (_, handle) in HANDLES.read(token.token()).iter() { let Handle { kind: ContextHandle::AddrSpace { addrspace, .. }, .. @@ -170,14 +170,17 @@ pub fn foreach_addrsp(mut f: impl FnMut(&Arc)) { } } -fn new_handle((handle, fl): (Handle, InternalFlags)) -> Result<(usize, InternalFlags)> { +fn new_handle( + (handle, fl): (Handle, InternalFlags), + token: &mut CleanLockToken, +) -> Result<(usize, InternalFlags)> { let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - let _ = HANDLES.write().insert(id, handle); + let _ = HANDLES.write(token.token()).insert(id, handle); Ok((id, fl)) } enum OpenTy { - Ctxt(Arc>), + Ctxt(Arc), Auth, } @@ -185,14 +188,15 @@ impl ProcScheme { fn openat_context( &self, path: &str, - context: Arc>, + context: Arc, + token: &mut CleanLockToken, ) -> Result> { Ok(Some(match path { "addrspace" => ( ContextHandle::AddrSpace { addrspace: Arc::clone( context - .read() + .read(token.token()) .addr_space() .map_err(|_| Error::new(ENOENT))?, ), @@ -201,7 +205,7 @@ impl ProcScheme { ), "filetable" => ( ContextHandle::Filetable { - filetable: Arc::downgrade(&context.read().files), + filetable: Arc::downgrade(&context.read(token.token()).files), data: Box::new([]), }, true, @@ -217,7 +221,7 @@ impl ProcScheme { "mmap-min-addr" => ( ContextHandle::MmapMinAddr(Arc::clone( context - .read() + .read(token.token()) .addr_space() .map_err(|_| Error::new(ENOENT))?, )), @@ -239,10 +243,14 @@ impl ProcScheme { _ => return Err(Error::new(ENOENT)), }; - let (hopefully_this_scheme, number) = extract_scheme_number(auth_fd)?; + let (hopefully_this_scheme, number) = extract_scheme_number(auth_fd, token)?; verify_scheme(hopefully_this_scheme)?; if !matches!( - HANDLES.read().get(&number).ok_or(Error::new(ENOENT))?.kind, + HANDLES + .read(token.token()) + .get(&number) + .ok_or(Error::new(ENOENT))? + .kind, ContextHandle::Authority ) { return Err(Error::new(ENOENT)); @@ -258,11 +266,12 @@ impl ProcScheme { ty: OpenTy, operation_str: Option<&str>, _flags: usize, + token: &mut CleanLockToken, ) -> Result<(usize, InternalFlags)> { let operation_name = operation_str.ok_or(Error::new(EINVAL))?; let (mut handle, positioned) = match ty { OpenTy::Ctxt(context) => { - match self.openat_context(operation_name, Arc::clone(&context))? { + match self.openat_context(operation_name, Arc::clone(&context), token)? { Some((kind, positioned)) => (Handle { context, kind }, positioned), _ => { return Err(Error::new(EINVAL)); @@ -275,8 +284,8 @@ impl ProcScheme { "new-context" => { let id = NonZeroUsize::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)) .ok_or(Error::new(EMFILE))?; - let context = context::spawn(true, Some(id), ret)?; - HANDLES.write().insert( + let context = context::spawn(true, Some(id), ret, token)?; + HANDLES.write(token.token()).insert( id.get(), Handle { context, @@ -339,21 +348,30 @@ impl ProcScheme { } }; - let (id, int_fl) = new_handle(( - handle.clone(), - if positioned { - InternalFlags::POSITIONED - } else { - InternalFlags::empty() - }, - ))?; + let (id, int_fl) = new_handle( + ( + handle.clone(), + if positioned { + InternalFlags::POSITIONED + } else { + InternalFlags::empty() + }, + ), + token, + )?; Ok((id, int_fl)) } } impl KernelScheme for ProcScheme { - fn kopen(&self, path: &str, _flags: usize, _ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { if path != "authority" { return Err(Error::new(ENOENT)); } @@ -362,7 +380,7 @@ impl KernelScheme for ProcScheme { return Err(Error::new(EEXIST)); } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert( + HANDLES.write(token.token()).insert( id, Handle { // TODO: placeholder @@ -373,8 +391,13 @@ impl KernelScheme for ProcScheme { Ok(OpenResult::SchemeLocal(id, InternalFlags::empty())) } - fn fevent(&self, id: usize, _flags: EventFlags) -> Result { - let handles = HANDLES.read(); + fn fevent( + &self, + id: usize, + _flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { + let handles = HANDLES.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; match handle { @@ -382,8 +405,11 @@ impl KernelScheme for ProcScheme { } } - fn close(&self, id: usize) -> Result<()> { - let handle = HANDLES.write().remove(&id).ok_or(Error::new(EBADF))?; + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + let handle = HANDLES + .write(token.token()) + .remove(&id) + .ok_or(Error::new(EBADF))?; match handle { Handle { @@ -395,17 +421,17 @@ impl KernelScheme for ProcScheme { new_ip, }, } => { - let _ = try_stop_context(context, |context: &mut Context| { + let _ = try_stop_context(context, token, |context: &mut Context| { let regs = context.regs_mut().ok_or(Error::new(EBADFD))?; regs.set_instr_pointer(new_ip); regs.set_stack_pointer(new_sp); Ok(context.set_addr_space(Some(new))) })?; - let _ = ptrace::send_event(crate::syscall::ptrace_event!( - PTRACE_EVENT_ADDRSPACE_SWITCH, - 0 - )); + let _ = ptrace::send_event( + crate::syscall::ptrace_event!(PTRACE_EVENT_ADDRSPACE_SWITCH, 0), + token, + ); } Handle { kind: ContextHandle::AddrSpace { addrspace } | ContextHandle::MmapMinAddr(addrspace), @@ -416,7 +442,7 @@ impl KernelScheme for ProcScheme { kind: ContextHandle::AwaitingFiletableChange { new_ft }, context, } => { - context.write().files = new_ft; + context.write(token.token()).files = new_ft; } _ => (), } @@ -428,8 +454,13 @@ impl KernelScheme for ProcScheme { dst_addr_space: &Arc, map: &crate::syscall::data::Map, consume: bool, + token: &mut CleanLockToken, ) -> Result { - let handle = HANDLES.read().get(&id).ok_or(Error::new(EBADF))?.clone(); + let handle = HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + .clone(); let Handle { kind, ref context } = handle; match kind { @@ -490,12 +521,12 @@ impl KernelScheme for ProcScheme { )? }; - handle_notify_files(notify_files); + handle_notify_files(notify_files, token); Ok(result_base.start_address().data()) } ContextHandle::Sighandler => { - let context = context.read(); + let context = context.read(token.token()); let sig = context.sig.as_ref().ok_or(Error::new(EBADF))?; let frame = match map.offset { // tctl @@ -542,15 +573,16 @@ impl KernelScheme for ProcScheme { offset: u64, _read_flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { // Don't hold a global lock during the context switch later on let handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); handles.get(&id).ok_or(Error::new(EBADF))?.clone() }; match handle { - Handle { context, kind } => kind.kreadoff(id, context, buf, offset), + Handle { context, kind } => kind.kreadoff(id, context, buf, offset, token), } } fn kcall( @@ -559,10 +591,11 @@ impl KernelScheme for ProcScheme { _payload: UserSliceRw, _flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { // TODO: simplify let handle = { - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -577,7 +610,9 @@ impl KernelScheme for ProcScheme { let verb = ProcSchemeVerb::try_from_raw(verb).ok_or(Error::new(EINVAL))?; match verb { - ProcSchemeVerb::Iopl => context::current().write().set_userspace_io_allowed(true), + ProcSchemeVerb::Iopl => context::current() + .write(token.token()) + .set_userspace_io_allowed(true), } Ok(0) } @@ -588,22 +623,23 @@ impl KernelScheme for ProcScheme { _offset: u64, _fcntl_flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { // TODO: offset // Don't hold a global lock during the context switch later on let handle = { - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.clone() }; match handle { - Handle { context, kind } => kind.kwriteoff(id, context, buf), + Handle { context, kind } => kind.kwriteoff(id, context, buf, token), } } - fn kfstat(&self, id: usize, buffer: UserSliceWo) -> Result<()> { - let handles = HANDLES.read(); + fn kfstat(&self, id: usize, buffer: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let handles = HANDLES.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; buffer.copy_exactly(&Stat { @@ -616,17 +652,23 @@ impl KernelScheme for ProcScheme { Ok(()) } - fn fsize(&self, id: usize) -> Result { - let mut handles = HANDLES.write(); + fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { + let mut handles = HANDLES.write(token.token()); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; handle.fsize() } /// Dup is currently used to implement clone() and execve(). - fn kdup(&self, old_id: usize, raw_buf: UserSliceRo, _: CallerCtx) -> Result { + fn kdup( + &self, + old_id: usize, + raw_buf: UserSliceRo, + _: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let info = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); let handle = handles.get(&old_id).ok_or(Error::new(EBADF))?; handle.clone() @@ -649,121 +691,126 @@ impl KernelScheme for ProcScheme { raw_buf.copy_to_slice(&mut array[..raw_buf.len()])?; let buf = &array[..raw_buf.len()]; - new_handle(match info { - Handle { - kind: ContextHandle::Authority, - .. - } => { - return self - .open_inner( - OpenTy::Auth, - Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) - .filter(|s| !s.is_empty()), - O_RDWR | O_CLOEXEC, - ) - .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) - } - Handle { - kind: ContextHandle::OpenViaDup, - context, - } => { - return self - .open_inner( - OpenTy::Ctxt(context), - Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) - .filter(|s| !s.is_empty()), - O_RDWR | O_CLOEXEC, - ) - .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)); - } - - Handle { - kind: - ContextHandle::Filetable { - ref filetable, - ref data, - }, - context, - } => { - // TODO: Maybe allow userspace to either copy or transfer recently dupped file - // descriptors between file tables. - if buf != b"copy" { - return Err(Error::new(EINVAL)); + new_handle( + match info { + Handle { + kind: ContextHandle::Authority, + .. + } => { + return self + .open_inner( + OpenTy::Auth, + Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) + .filter(|s| !s.is_empty()), + O_RDWR | O_CLOEXEC, + token, + ) + .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) + } + Handle { + kind: ContextHandle::OpenViaDup, + context, + } => { + return self + .open_inner( + OpenTy::Ctxt(context), + Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) + .filter(|s| !s.is_empty()), + O_RDWR | O_CLOEXEC, + token, + ) + .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)); } - let filetable = filetable.upgrade().ok_or(Error::new(EOWNERDEAD))?; - let new_filetable = Arc::try_new(RwLock::new(filetable.read().clone())) - .map_err(|_| Error::new(ENOMEM))?; - - handle( - Handle { - kind: ContextHandle::NewFiletable { - filetable: new_filetable, - data: data.clone(), + Handle { + kind: + ContextHandle::Filetable { + ref filetable, + ref data, }, - context, - }, - true, - ) - } - Handle { - kind: ContextHandle::AddrSpace { ref addrspace }, - context, - } => { - const GRANT_FD_PREFIX: &[u8] = b"grant-fd-"; + context, + } => { + // TODO: Maybe allow userspace to either copy or transfer recently dupped file + // descriptors between file tables. + if buf != b"copy" { + return Err(Error::new(EINVAL)); + } + let filetable = filetable.upgrade().ok_or(Error::new(EOWNERDEAD))?; - let kind = match buf { - // TODO: Better way to obtain new empty address spaces, perhaps using SYS_OPEN. But - // in that case, what scheme? - b"empty" => ContextHandle::AddrSpace { - addrspace: AddrSpaceWrapper::new()?, - }, - b"exclusive" => ContextHandle::AddrSpace { - addrspace: addrspace.try_clone()?, - }, - b"mmap-min-addr" => ContextHandle::MmapMinAddr(Arc::clone(addrspace)), + let new_filetable = Arc::try_new(spin::RwLock::new(filetable.read().clone())) + .map_err(|_| Error::new(ENOMEM))?; - _ if buf.starts_with(GRANT_FD_PREFIX) => { - let string = core::str::from_utf8(&buf[GRANT_FD_PREFIX.len()..]) - .map_err(|_| Error::new(EINVAL))?; - let page_addr = - usize::from_str_radix(string, 16).map_err(|_| Error::new(EINVAL))?; + handle( + Handle { + kind: ContextHandle::NewFiletable { + filetable: new_filetable, + data: data.clone(), + }, + context, + }, + true, + ) + } + Handle { + kind: ContextHandle::AddrSpace { ref addrspace }, + context, + } => { + const GRANT_FD_PREFIX: &[u8] = b"grant-fd-"; - if page_addr % PAGE_SIZE != 0 { - return Err(Error::new(EINVAL)); - } + let kind = match buf { + // TODO: Better way to obtain new empty address spaces, perhaps using SYS_OPEN. But + // in that case, what scheme? + b"empty" => ContextHandle::AddrSpace { + addrspace: AddrSpaceWrapper::new()?, + }, + b"exclusive" => ContextHandle::AddrSpace { + addrspace: addrspace.try_clone()?, + }, + b"mmap-min-addr" => ContextHandle::MmapMinAddr(Arc::clone(addrspace)), - let page = Page::containing_address(VirtualAddress::new(page_addr)); + _ if buf.starts_with(GRANT_FD_PREFIX) => { + let string = core::str::from_utf8(&buf[GRANT_FD_PREFIX.len()..]) + .map_err(|_| Error::new(EINVAL))?; + let page_addr = usize::from_str_radix(string, 16) + .map_err(|_| Error::new(EINVAL))?; - match addrspace - .acquire_read() - .grants - .contains(page) - .ok_or(Error::new(EINVAL))? - { - (_, info) => { - return Ok(OpenResult::External( - info.file_ref() - .map(|r| Arc::clone(&r.description)) - .ok_or(Error::new(EBADF))?, - )) + if page_addr % PAGE_SIZE != 0 { + return Err(Error::new(EINVAL)); + } + + let page = Page::containing_address(VirtualAddress::new(page_addr)); + + match addrspace + .acquire_read() + .grants + .contains(page) + .ok_or(Error::new(EINVAL))? + { + (_, info) => { + return Ok(OpenResult::External( + info.file_ref() + .map(|r| Arc::clone(&r.description)) + .ok_or(Error::new(EBADF))?, + )) + } } } - } - _ => return Err(Error::new(EINVAL)), - }; + _ => return Err(Error::new(EINVAL)), + }; - handle(Handle { context, kind }, true) - } - _ => return Err(Error::new(EINVAL)), - }) + handle(Handle { context, kind }, true) + } + _ => return Err(Error::new(EINVAL)), + }, + token, + ) .map(|(r, fl)| OpenResult::SchemeLocal(r, fl)) } } -fn extract_scheme_number(fd: usize) -> Result<(KernelSchemes, usize)> { +fn extract_scheme_number(fd: usize, token: &mut CleanLockToken) -> Result<(KernelSchemes, usize)> { let (scheme_id, number) = match &*context::current() - .read() + .read(token.token()) .get_file(FileHandle::from(fd)) .ok_or(Error::new(EBADF))? .description @@ -771,7 +818,7 @@ fn extract_scheme_number(fd: usize) -> Result<(KernelSchemes, usize)> { { desc => (desc.scheme, desc.number), }; - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(scheme_id) .ok_or(Error::new(ENODEV))? .clone(); @@ -797,8 +844,9 @@ impl ContextHandle { fn kwriteoff( self, id: usize, - context: Arc>, + context: Arc, buf: UserSliceRo, + token: &mut CleanLockToken, ) -> Result { match self { Self::AddrSpace { addrspace } => { @@ -820,7 +868,7 @@ impl ContextHandle { return Err(Error::new(EOPNOTSUPP)); } - let (scheme, number) = extract_scheme_number(fd)?; + let (scheme, number) = extract_scheme_number(fd, token)?; scheme.kfmap( number, @@ -832,6 +880,7 @@ impl ContextHandle { flags, }, op == ADDRSPACE_OP_TRANSFER, + token, )?; } ADDRSPACE_OP_MUNMAP => { @@ -854,7 +903,7 @@ impl ContextHandle { RegsKind::Float => { let regs = unsafe { buf.read_exact::()? }; - try_stop_context(context, |context| { + try_stop_context(context, token, |context| { // NOTE: The kernel will never touch floats // Ignore the rare case of floating point @@ -867,7 +916,7 @@ impl ContextHandle { RegsKind::Int => { let regs = unsafe { buf.read_exact::()? }; - try_stop_context(context, |context| match context.regs_mut() { + try_stop_context(context, token, |context| match context.regs_mut() { None => { println!( "{}:{}: Couldn't read registers from stopped process", @@ -885,7 +934,7 @@ impl ContextHandle { } RegsKind::Env => { let regs = unsafe { buf.read_exact::()? }; - write_env_regs(context, regs)?; + write_env_regs(context, regs, token)?; Ok(mem::size_of::()) } }, @@ -913,7 +962,7 @@ impl ContextHandle { } }; - let addrsp = Arc::clone(context.read().addr_space()?); + let addrsp = Arc::clone(context.read(token.token()).addr_space()?); Some(SignalState { threadctl_off: validate_off( @@ -929,20 +978,22 @@ impl ContextHandle { excp_handler: NonZeroUsize::new(data.excp_handler), thread_control: addrsp.borrow_frame_enforce_rw_allocated( Page::containing_address(VirtualAddress::new(data.thread_control_addr)), + token, )?, proc_control: addrsp.borrow_frame_enforce_rw_allocated( Page::containing_address(VirtualAddress::new(data.proc_control_addr)), + token, )?, }) } else { None }; - context.write().sig = state; + context.write(token.token()).sig = state; Ok(mem::size_of::()) } - ContextHandle::Start => match context.write().status { + ContextHandle::Start => match context.write(token.token()).status { ref mut status @ Status::HardBlocked { reason: HardBlockedReason::NotYetStarted, } => { @@ -957,10 +1008,10 @@ impl ContextHandle { ContextHandle::CurrentFiletable => { let filetable_fd = buf.read_usize()?; - let (hopefully_this_scheme, number) = extract_scheme_number(filetable_fd)?; + let (hopefully_this_scheme, number) = extract_scheme_number(filetable_fd, token)?; verify_scheme(hopefully_this_scheme)?; - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); let Entry::Occupied(mut entry) = handles.entry(number) else { return Err(Error::new(EBADF)); }; @@ -1004,10 +1055,10 @@ impl ContextHandle { let sp = iter.next().ok_or(Error::new(EINVAL))??; let ip = iter.next().ok_or(Error::new(EINVAL))??; - let (hopefully_this_scheme, number) = extract_scheme_number(addrspace_fd)?; + let (hopefully_this_scheme, number) = extract_scheme_number(addrspace_fd, token)?; verify_scheme(hopefully_this_scheme)?; - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); let &Handle { kind: ContextHandle::AddrSpace { ref addrspace }, .. @@ -1038,7 +1089,10 @@ impl ContextHandle { Self::SchedAffinity => { let mask = unsafe { buf.read_exact::()? }; - context.write().sched_affinity.override_from(&mask); + context + .write(token.token()) + .sched_affinity + .override_from(&mask); Ok(mem::size_of_val(&mask)) } @@ -1056,7 +1110,7 @@ impl ContextHandle { return Err(Error::new(EPERM)) } ContextVerb::Stop => { - let mut guard = context.write(); + let mut guard = context.write(token.token()); match guard.status { Status::Dead { .. } => return Err(Error::new(EOWNERDEAD)), @@ -1072,7 +1126,7 @@ impl ContextHandle { Ok(size_of::()) } ContextVerb::Unstop => { - let mut guard = context.write(); + let mut guard = context.write(token.token()); if let Status::HardBlocked { reason: HardBlockedReason::Stopped, @@ -1083,7 +1137,7 @@ impl ContextHandle { Ok(size_of::()) } ContextVerb::Interrupt => { - let mut guard = context.write(); + let mut guard = context.write(token.token()); guard.unblock(); Ok(size_of::()) } @@ -1098,7 +1152,8 @@ impl ContextHandle { let size = args.next().ok_or(Error::new(EINVAL))??; if size > 0 { - let addrsp = Arc::clone(context.read().addr_space()?); + let addrsp = + Arc::clone(context.read(token.token()).addr_space()?); let res = addrsp.munmap( PageSpan::validate_nonempty( VirtualAddress::new(base), @@ -1108,13 +1163,13 @@ impl ContextHandle { false, )?; for r in res { - let _ = r.unmap(); + let _ = r.unmap(token); } } } - crate::syscall::exit_this_context(None); + crate::syscall::exit_this_context(None, token); } else { - let mut ctxt = context.write(); + let mut ctxt = context.write(token.token()); //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); ctxt.status = context::Status::Runnable; ctxt.being_sigkilled = true; @@ -1125,7 +1180,7 @@ impl ContextHandle { } ContextHandle::Attr => { let info = unsafe { buf.read_exact::()? }; - let mut guard = context.write(); + let mut guard = context.write(token.token()); let len = info .debug_name @@ -1150,9 +1205,10 @@ impl ContextHandle { fn kreadoff( &self, _id: usize, - context: Arc>, + context: Arc, buf: UserSliceWo, offset: u64, + token: &mut CleanLockToken, ) -> Result { match self { ContextHandle::Regs(kind) => { @@ -1164,7 +1220,7 @@ impl ContextHandle { let (output, size) = match kind { RegsKind::Float => { - let context = context.read(); + let context = context.read(token.token()); // NOTE: The kernel will never touch floats ( @@ -1174,25 +1230,27 @@ impl ContextHandle { mem::size_of::(), ) } - RegsKind::Int => try_stop_context(context, |context| match context.regs() { - None => { - assert!(!context.running, "try_stop_context is broken, clearly"); - println!( - "{}:{}: Couldn't read registers from stopped process", - file!(), - line!() - ); - Err(Error::new(ENOTRECOVERABLE)) - } - Some(stack) => { - let mut regs = IntRegisters::default(); - stack.save(&mut regs); - Ok((Output { int: regs }, mem::size_of::())) - } - })?, + RegsKind::Int => { + try_stop_context(context, token, |context| match context.regs() { + None => { + assert!(!context.running, "try_stop_context is broken, clearly"); + println!( + "{}:{}: Couldn't read registers from stopped process", + file!(), + line!() + ); + Err(Error::new(ENOTRECOVERABLE)) + } + Some(stack) => { + let mut regs = IntRegisters::default(); + stack.save(&mut regs); + Ok((Output { int: regs }, mem::size_of::())) + } + })? + } RegsKind::Env => ( Output { - env: read_env_regs(context)?, + env: read_env_regs(context, token)?, }, mem::size_of::(), ), @@ -1246,14 +1304,14 @@ impl ContextHandle { Ok(mem::size_of::()) } ContextHandle::SchedAffinity => { - let mask = context.read().sched_affinity.to_raw(); + let mask = context.read(token.token()).sched_affinity.to_raw(); buf.copy_exactly(crate::cpu_set::mask_as_bytes(&mask))?; Ok(mem::size_of_val(&mask)) } // TODO: Replace write() with SYS_SENDFD? ContextHandle::Status { .. } => { let status = { - let context = context.read(); + let context = context.read(token.token()); match context.status { Status::Runnable | Status::Dead { excp: None } if context.being_sigkilled => @@ -1285,7 +1343,7 @@ impl ContextHandle { } ContextHandle::Attr => { let mut debug_name = [0; 32]; - let (euid, egid, ens, pid, name) = match context.read() { + let (euid, egid, ens, pid, name) = match context.read(token.token()) { ref c => (c.euid, c.egid, c.ens.get() as u32, c.pid as u32, c.name), }; let min = name.len().min(debug_name.len()); @@ -1299,7 +1357,7 @@ impl ContextHandle { }) } ContextHandle::Sighandler => { - let data = match context.read().sig { + let data = match context.read(token.token()).sig { Some(ref sig) => SetSighandlerData { excp_handler: sig.excp_handler.map_or(0, NonZeroUsize::get), user_handler: sig.user_handler.get(), @@ -1319,18 +1377,26 @@ impl ContextHandle { } } -fn write_env_regs(context: Arc>, regs: EnvRegisters) -> Result<()> { +fn write_env_regs( + context: Arc, + regs: EnvRegisters, + token: &mut CleanLockToken, +) -> Result<()> { if context::is_current(&context) { - context::current().write().write_current_env_regs(regs) + context::current() + .write(token.token()) + .write_current_env_regs(regs) } else { - try_stop_context(context, |context| context.write_env_regs(regs)) + try_stop_context(context, token, |context| context.write_env_regs(regs)) } } -fn read_env_regs(context: Arc>) -> Result { +fn read_env_regs(context: Arc, token: &mut CleanLockToken) -> Result { if context::is_current(&context) { - context::current().read().read_current_env_regs() + context::current() + .read(token.token()) + .read_current_env_regs() } else { - try_stop_context(context, |context| context.read_env_regs()) + try_stop_context(context, token, |context| context.read_env_regs()) } } diff --git a/src/scheme/root.rs b/src/scheme/root.rs index e69667edd9..6b93716a06 100644 --- a/src/scheme/root.rs +++ b/src/scheme/root.rs @@ -4,7 +4,6 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; use hashbrown::HashMap; -use spin::RwLock; use syscall::{ dirent::{DirEntry, DirentBuf, DirentKind}, O_EXLOCK, O_FSYNC, @@ -17,6 +16,7 @@ use crate::{ user::{UserInner, UserScheme}, FileDescription, SchemeId, SchemeNamespace, }, + sync::{CleanLockToken, RwLock, L1}, syscall::{ data::Stat, error::*, @@ -38,7 +38,7 @@ pub struct RootScheme { scheme_ns: SchemeNamespace, scheme_id: SchemeId, next_id: AtomicUsize, - handles: RwLock>, + handles: RwLock>, } impl RootScheme { @@ -53,7 +53,13 @@ impl RootScheme { } impl KernelScheme for RootScheme { - fn kopen(&self, path: &str, flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let path = path.trim_start_matches('/'); //TODO: Make this follow standards for flags and errors @@ -72,7 +78,7 @@ impl KernelScheme for RootScheme { let inner = { let path_box = path.to_string().into_boxed_str(); - let mut schemes = scheme::schemes_mut(); + let mut schemes = scheme::schemes_mut(token.token()); let v2 = flags & O_FSYNC == O_FSYNC; let new_close = flags & O_EXLOCK == O_EXLOCK; @@ -110,32 +116,38 @@ impl KernelScheme for RootScheme { inner }; - self.handles.write().insert(id, Handle::Scheme(inner)); + self.handles + .write(token.token()) + .insert(id, Handle::Scheme(inner)); Ok(OpenResult::SchemeLocal(id, InternalFlags::empty())) } else if path.is_empty() { - let ens = context::current().read().ens; + let ens = context::current().read(token.token()).ens; let id = self.next_id.fetch_add(1, Ordering::Relaxed); - self.handles.write().insert(id, Handle::List { ens }); + self.handles + .write(token.token()) + .insert(id, Handle::List { ens }); Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) } else { let inner = Arc::new(path.as_bytes().to_vec().into_boxed_slice()); let id = self.next_id.fetch_add(1, Ordering::Relaxed); - self.handles.write().insert(id, Handle::File(inner)); + self.handles + .write(token.token()) + .insert(id, Handle::File(inner)); Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) } } - fn unlink(&self, path: &str, ctx: CallerCtx) -> Result<()> { + fn unlink(&self, path: &str, ctx: CallerCtx, token: &mut CleanLockToken) -> Result<()> { let path = path.trim_matches('/'); if ctx.uid != 0 { return Err(Error::new(EACCES)); } let inner = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); handles .iter() .find_map(|(_id, handle)| { @@ -152,12 +164,12 @@ impl KernelScheme for RootScheme { .ok_or(Error::new(ENOENT))? }; - inner.unmount() + inner.unmount(token) } - fn fsize(&self, file: usize) -> Result { + fn fsize(&self, file: usize, token: &mut CleanLockToken) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -169,9 +181,14 @@ impl KernelScheme for RootScheme { } } - fn fevent(&self, file: usize, flags: EventFlags) -> Result { + fn fevent( + &self, + file: usize, + flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -183,9 +200,14 @@ impl KernelScheme for RootScheme { } } - fn kfpath(&self, file: usize, mut buf: UserSliceWo) -> Result { + fn kfpath( + &self, + file: usize, + mut buf: UserSliceWo, + token: &mut CleanLockToken, + ) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -206,9 +228,9 @@ impl KernelScheme for RootScheme { Ok(bytes_copied) } - fn fsync(&self, file: usize) -> Result<()> { + fn fsync(&self, file: usize, token: &mut CleanLockToken) -> Result<()> { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -220,15 +242,15 @@ impl KernelScheme for RootScheme { } } - fn close(&self, file: usize) -> Result<()> { + fn close(&self, file: usize, token: &mut CleanLockToken) -> Result<()> { let handle = self .handles - .write() + .write(token.token()) .remove(&file) .ok_or(Error::new(EBADF))?; match handle { Handle::Scheme(inner) => { - scheme::schemes_mut().remove(inner.scheme_id); + scheme::schemes_mut(token.token()).remove(inner.scheme_id); } _ => (), } @@ -241,15 +263,16 @@ impl KernelScheme for RootScheme { _offset: u64, flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; match handle { - Handle::Scheme(inner) => inner.read(buf, flags), + Handle::Scheme(inner) => inner.read(buf, flags, token), Handle::File(_) => Err(Error::new(EBADF)), Handle::List { .. } => Err(Error::new(EISDIR)), } @@ -260,14 +283,20 @@ impl KernelScheme for RootScheme { buf: UserSliceWo, header_size: u16, opaque: u64, + token: &mut CleanLockToken, ) -> Result { - let Handle::List { ens } = *self.handles.read().get(&id).ok_or(Error::new(EBADF))? else { + let Handle::List { ens } = *self + .handles + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + else { return Err(Error::new(ENOTDIR)); }; let mut buf = DirentBuf::new(buf, header_size).ok_or(Error::new(EIO))?; { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); for (i, (name, _)) in schemes .iter_name(ens) .enumerate() @@ -292,23 +321,24 @@ impl KernelScheme for RootScheme { buf: UserSliceRo, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; match handle { - Handle::Scheme(inner) => inner.write(buf), + Handle::Scheme(inner) => inner.write(buf, token), Handle::File(_) => Err(Error::new(EBADF)), Handle::List { .. } => Err(Error::new(EISDIR)), } } - fn kfstat(&self, file: usize, buf: UserSliceWo) -> Result<()> { + fn kfstat(&self, file: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&file).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -334,13 +364,14 @@ impl KernelScheme for RootScheme { fn kfdwrite( &self, id: usize, - descs: Vec>>, + descs: Vec>>, flags: CallFlags, arg: u64, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; handle.clone() }; @@ -358,15 +389,16 @@ impl KernelScheme for RootScheme { payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let handle = { - let handles = self.handles.read(); + let handles = self.handles.read(token.token()); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; handle.clone() }; match handle { - Handle::Scheme(inner) => inner.call_fdread(payload, flags, metadata), + Handle::Scheme(inner) => inner.call_fdread(payload, flags, metadata, token), Handle::File(_) => Err(Error::new(EBADF)), Handle::List { .. } => Err(Error::new(EISDIR)), } diff --git a/src/scheme/serio.rs b/src/scheme/serio.rs index b4afec273d..5c1772b860 100644 --- a/src/scheme/serio.rs +++ b/src/scheme/serio.rs @@ -5,12 +5,10 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; -use spin::RwLock; - use crate::{ event, scheme::*, - sync::WaitQueue, + sync::{CleanLockToken, RwLock, WaitQueue, L1}, syscall::{ flag::{EventFlags, EVENT_READ, O_NONBLOCK}, usercopy::UserSliceWo, @@ -27,17 +25,17 @@ struct Handle { index: usize, } -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); /// Add to the input queue -pub fn serio_input(index: usize, data: u8) { +pub fn serio_input(index: usize, data: u8, token: &mut CleanLockToken) { #[cfg(feature = "profiling")] crate::profiling::serio_command(index, data); - INPUT[index].send(data); + INPUT[index].send(data, token); - for (id, _handle) in HANDLES.read().iter() { + for (id, _handle) in HANDLES.read(token.token()).iter() { event::trigger(GlobalSchemes::Serio.scheme_id(), *id, EVENT_READ); } } @@ -45,7 +43,13 @@ pub fn serio_input(index: usize, data: u8) { pub struct SerioScheme; impl KernelScheme for SerioScheme { - fn kopen(&self, path: &str, _flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { if ctx.uid != 0 { return Err(Error::new(EPERM)); } @@ -56,23 +60,28 @@ impl KernelScheme for SerioScheme { } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, Handle { index }); + HANDLES.write(token.token()).insert(id, Handle { index }); Ok(OpenResult::SchemeLocal(id, InternalFlags::empty())) } - fn fevent(&self, id: usize, _flags: EventFlags) -> Result { + fn fevent( + &self, + id: usize, + _flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { let _handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; Ok(EventFlags::empty()) } - fn fsync(&self, id: usize) -> Result<()> { + fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let _handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; @@ -80,17 +89,24 @@ impl KernelScheme for SerioScheme { } /// Close the file `number` - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let _handle = { - let mut handles = HANDLES.write(); + let mut handles = HANDLES.write(token.token()); handles.remove(&id).ok_or(Error::new(EBADF))? }; Ok(()) } - fn kread(&self, id: usize, buf: UserSliceWo, flags: u32, _stored_flags: u32) -> Result { + fn kread( + &self, + id: usize, + buf: UserSliceWo, + flags: u32, + _stored_flags: u32, + token: &mut CleanLockToken, + ) -> Result { let handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; @@ -98,12 +114,13 @@ impl KernelScheme for SerioScheme { buf, flags & O_NONBLOCK as u32 == 0, "SerioScheme::read", + token, ) } - fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { + fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { let handle = { - let handles = HANDLES.read(); + let handles = HANDLES.read(token.token()); *handles.get(&id).ok_or(Error::new(EBADF))? }; let path = format!("serio:{}", handle.index).into_bytes(); diff --git a/src/scheme/sys/block.rs b/src/scheme/sys/block.rs index 3b64d10d8c..bcdb05bfda 100644 --- a/src/scheme/sys/block.rs +++ b/src/scheme/sys/block.rs @@ -1,17 +1,18 @@ use alloc::{string::String, vec::Vec}; use core::fmt::Write; -use crate::{context, syscall::error::Result}; +use crate::{context, sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { +pub fn resource(token: &mut CleanLockToken) -> Result> { let mut string = String::new(); { let mut rows = Vec::new(); { - let contexts = context::contexts(); + let mut contexts = context::contexts(token.token()); + let (contexts, mut token) = contexts.token_split(); for context_lock in contexts.iter().filter_map(|r| r.upgrade()) { - let context = context_lock.read(); + let context = context_lock.read(token.token()); rows.push((context.pid, context.name.clone(), context.status_reason)); } } diff --git a/src/scheme/sys/context.rs b/src/scheme/sys/context.rs index bd70d19354..cdda2f1129 100644 --- a/src/scheme/sys/context.rs +++ b/src/scheme/sys/context.rs @@ -4,9 +4,9 @@ use alloc::{ }; use core::fmt::Write; -use crate::{context, paging::PAGE_SIZE, syscall::error::Result}; +use crate::{context, paging::PAGE_SIZE, sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { +pub fn resource(token: &mut CleanLockToken) -> Result> { let mut string = format!( "{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<11}{:<12}{:<8}{}\n", "PID", "EUID", "EGID", "ENS", "STAT", "CPU", "AFFINITY", "TIME", "MEM", "NAME" @@ -14,9 +14,10 @@ pub fn resource() -> Result> { let mut rows = Vec::new(); { - let contexts = context::contexts(); + let mut contexts = context::contexts(token.token()); + let (contexts, mut token) = contexts.token_split(); for context_ref in contexts.iter().filter_map(|r| r.upgrade()) { - let context = context_ref.read(); + let context = context_ref.read(token.token()); let mut stat_string = String::new(); // TODO: All user programs must have some grant in order for executable memory to even diff --git a/src/scheme/sys/cpu.rs b/src/scheme/sys/cpu.rs index f9f2c67267..878b0a64a7 100644 --- a/src/scheme/sys/cpu.rs +++ b/src/scheme/sys/cpu.rs @@ -2,10 +2,11 @@ use alloc::vec::Vec; use crate::{ device::cpu::cpu_info, + sync::CleanLockToken, syscall::error::{Error, Result, EIO}, }; -pub fn resource() -> Result> { +pub fn resource(_token: &mut CleanLockToken) -> Result> { let mut string = format!("CPUs: {}\n", crate::cpu_count()); match cpu_info(&mut string) { diff --git a/src/scheme/sys/exe.rs b/src/scheme/sys/exe.rs index 1b5439000f..947879dac0 100644 --- a/src/scheme/sys/exe.rs +++ b/src/scheme/sys/exe.rs @@ -1,7 +1,11 @@ use alloc::vec::Vec; -use crate::{context, syscall::error::Result}; +use crate::{context, sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { - Ok(context::current().read().name.as_bytes().to_vec()) +pub fn resource(token: &mut CleanLockToken) -> Result> { + Ok(context::current() + .read(token.token()) + .name + .as_bytes() + .to_vec()) } diff --git a/src/scheme/sys/fdstat.rs b/src/scheme/sys/fdstat.rs index 5667137f24..8e2686de88 100644 --- a/src/scheme/sys/fdstat.rs +++ b/src/scheme/sys/fdstat.rs @@ -2,6 +2,7 @@ use crate::{ context, context::{file::FileDescription, memory::AddrSpaceWrapper}, scheme, + sync::CleanLockToken, syscall::error::Result, }; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; @@ -9,7 +10,7 @@ use core::{fmt::Write, hash::Hash}; use hashbrown::HashMap; use spin::RwLock; -pub fn resource() -> Result> { +pub fn resource(token: &mut CleanLockToken) -> Result> { #[derive(Debug)] struct Ref(Arc); impl Hash for Ref { @@ -31,7 +32,10 @@ pub fn resource() -> Result> { let mut map = HashMap::>, Descr>::new(); let mut report = String::new(); - 'contexts: for context in context::contexts().iter().filter_map(|c| c.upgrade()) { + 'contexts: for context in context::contexts(token.token()) + .iter() + .filter_map(|c| c.upgrade()) + { let context = context.read(); let files = context.files.read(); writeln!(report, "'{}' {{", context.name).unwrap(); @@ -51,7 +55,7 @@ pub fn resource() -> Result> { let descr = map.entry(fr).or_default(); let scheme_id = file.description.read().scheme; - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .names .iter() .flat_map(|(_, v)| v.iter()) diff --git a/src/scheme/sys/iostat.rs b/src/scheme/sys/iostat.rs index 306b5db8b4..1ce152066b 100644 --- a/src/scheme/sys/iostat.rs +++ b/src/scheme/sys/iostat.rs @@ -5,6 +5,7 @@ use crate::{ }, paging::PAGE_SIZE, scheme, + sync::CleanLockToken, syscall::{ error::Result, flag::MapFlags, @@ -14,16 +15,17 @@ use crate::{ use alloc::{string::String, sync::Arc, vec::Vec}; use core::{fmt::Write, num::NonZeroUsize, str}; -fn inner(fpath_user: UserSliceRw) -> Result> { +fn inner(fpath_user: UserSliceRw, token: &mut CleanLockToken) -> Result> { let mut string = String::new(); let mut fpath_kernel = [0; PAGE_SIZE]; { let mut rows = Vec::new(); { - let contexts = context::contexts(); + let mut contexts = context::contexts(token.token()); + let (contexts, mut token) = contexts.token_split(); for context_ref in contexts.iter().filter_map(|r| r.upgrade()) { - let context = context_ref.read(); + let context = context_ref.read(token.token()); rows.push(( context.pid, context.name.clone(), @@ -54,7 +56,7 @@ fn inner(fpath_user: UserSliceRw) -> Result> { ); let scheme = { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); match schemes.get(description.scheme) { Some(scheme) => scheme.clone(), None => { @@ -64,7 +66,11 @@ fn inner(fpath_user: UserSliceRw) -> Result> { } }; - match scheme.kfpath(description.number, fpath_user.reinterpret_unchecked()) { + match scheme.kfpath( + description.number, + fpath_user.reinterpret_unchecked(), + token, + ) { Ok(path_len) => { fpath_user.copy_to_slice(&mut fpath_kernel)?; let fname = str::from_utf8(&fpath_kernel[..path_len]).unwrap_or("?"); @@ -81,10 +87,10 @@ fn inner(fpath_user: UserSliceRw) -> Result> { Ok(string.into_bytes()) } -pub fn resource() -> Result> { +pub fn resource(token: &mut CleanLockToken) -> Result> { let page_count = NonZeroUsize::new(1).unwrap(); let fpath_page = { - let addr_space = Arc::clone(context::current().read().addr_space()?); + let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?); addr_space.acquire_write().mmap( &addr_space, None, @@ -104,10 +110,11 @@ pub fn resource() -> Result> { )? }; - let res = UserSlice::rw(fpath_page.start_address().data(), PAGE_SIZE).and_then(inner); + let res = UserSlice::rw(fpath_page.start_address().data(), PAGE_SIZE) + .and_then(|fpath_user| inner(fpath_user, token)); { - let addr_space = Arc::clone(context::current().read().addr_space()?); + let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?); addr_space.munmap(PageSpan::new(fpath_page, page_count.get()), false)?; } diff --git a/src/scheme/sys/irq.rs b/src/scheme/sys/irq.rs index 655dc1b4b5..f76de90573 100644 --- a/src/scheme/sys/irq.rs +++ b/src/scheme/sys/irq.rs @@ -1,9 +1,9 @@ use alloc::{string::String, vec::Vec}; use core::fmt::Write; -use crate::syscall::error::Result; +use crate::{sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { +pub fn resource(_token: &mut CleanLockToken) -> Result> { let mut string = String::new(); { diff --git a/src/scheme/sys/log.rs b/src/scheme/sys/log.rs index 709e4a117e..8cdd43dfb0 100644 --- a/src/scheme/sys/log.rs +++ b/src/scheme/sys/log.rs @@ -1,8 +1,8 @@ use alloc::vec::Vec; -use crate::{log::LOG, syscall::error::Result}; +use crate::{log::LOG, sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { +pub fn resource(_token: &mut CleanLockToken) -> Result> { let mut vec = Vec::new(); if let Some(ref log) = *LOG.lock() { diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index 795af9df52..c0451c7daf 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -13,12 +13,12 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use spin::RwLock; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use crate::arch::interrupt; use crate::{ context::file::InternalFlags, + sync::{CleanLockToken, RwLock, L1}, syscall::{ data::Stat, error::{Error, Result, EBADF, ENOENT}, @@ -57,15 +57,15 @@ enum Handle { } enum Kind { - Rd(fn() -> Result>), - Wr(fn(&[u8]) -> Result), + Rd(fn(&mut CleanLockToken) -> Result>), + Wr(fn(&[u8], &mut CleanLockToken) -> Result), } use Kind::*; /// System information scheme pub struct SysScheme; static NEXT_ID: AtomicUsize = AtomicUsize::new(1); -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); const FILES: &[(&'static str, Kind)] = &[ @@ -82,7 +82,7 @@ const FILES: &[(&'static str, Kind)] = &[ ("scheme_num", Rd(scheme_num::resource)), ("syscall", Rd(syscall::resource)), ("uname", Rd(uname::resource)), - ("env", Rd(|| Ok(Vec::from(crate::init_env())))), + ("env", Rd(|_| Ok(Vec::from(crate::init_env())))), #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] ("spurious_irq", Rd(interrupt::irq::spurious_irq_resource)), #[cfg(feature = "sys_stat")] @@ -100,9 +100,9 @@ const FILES: &[(&'static str, Kind)] = &[ ), ( "kstop", - Wr(|arg| unsafe { + Wr(|arg, token| unsafe { match arg.trim_ascii() { - b"shutdown" => crate::stop::kstop(), + b"shutdown" => crate::stop::kstop(token), b"reset" => crate::stop::kreset(), b"emergency_reset" => crate::stop::emergency_reset(), _ => Err(Error::new(EINVAL)), @@ -112,13 +112,19 @@ const FILES: &[(&'static str, Kind)] = &[ ]; impl KernelScheme for SysScheme { - fn kopen(&self, path: &str, _flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let path = path.trim_matches('/'); if path.is_empty() { let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, Handle::TopLevel); + HANDLES.write(token.token()).insert(id, Handle::TopLevel); Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED)) } else { @@ -134,10 +140,10 @@ impl KernelScheme for SysScheme { let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); let data = match entry.1 { - Rd(r) => Some(r()?), + Rd(r) => Some(r(token)?), Wr(_) => None, }; - HANDLES.write().insert( + HANDLES.write(token.token()).insert( id, Handle::Resource { path: entry.0, @@ -148,19 +154,26 @@ impl KernelScheme for SysScheme { } } - fn fsize(&self, id: usize) -> Result { - match HANDLES.read().get(&id).ok_or(Error::new(EBADF))? { + fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result { + match HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + { Handle::TopLevel => Ok(0), Handle::Resource { data, .. } => Ok(data.as_ref().map_or(0, |d| d.len() as u64)), } } - fn close(&self, id: usize) -> Result<()> { - HANDLES.write().remove(&id).ok_or(Error::new(EBADF))?; + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + HANDLES + .write(token.token()) + .remove(&id) + .ok_or(Error::new(EBADF))?; Ok(()) } - fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { - let handles = HANDLES.read(); + fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { + let handles = HANDLES.read(token.token()); let path = match handles.get(&id).ok_or(Error::new(EBADF))? { Handle::TopLevel => "", Handle::Resource { path, .. } => path, @@ -182,12 +195,17 @@ impl KernelScheme for SysScheme { pos: u64, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let Ok(pos) = usize::try_from(pos) else { return Ok(0); }; - match HANDLES.read().get(&id).ok_or(Error::new(EBADF))? { + match HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + { Handle::TopLevel | Handle::Resource { data: None, .. } => { return Err(Error::new(EISDIR)) } @@ -208,8 +226,13 @@ impl KernelScheme for SysScheme { _pos: u64, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { - match HANDLES.read().get(&id).ok_or(Error::new(EBADF))? { + let (handler, intermediate, len) = match HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + { Handle::TopLevel | Handle::Resource { data: Some(_), .. } => { return Err(Error::new(EISDIR)) } @@ -223,9 +246,10 @@ impl KernelScheme for SysScheme { else { return Err(Error::new(EBADFD))?; }; - handler(&intermediate[..len]) + (handler, intermediate, len) } - } + }; + handler(&intermediate[..len], token) } fn getdents( &self, @@ -233,11 +257,16 @@ impl KernelScheme for SysScheme { buf: UserSliceWo, header_size: u16, first_index: u64, + token: &mut CleanLockToken, ) -> Result { let Ok(first_index) = usize::try_from(first_index) else { return Ok(0); }; - match HANDLES.read().get(&id).ok_or(Error::new(EBADF))? { + match HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + { Handle::Resource { .. } => return Err(Error::new(ENOTDIR)), Handle::TopLevel => { let mut buf = DirentBuf::new(buf, header_size).ok_or(Error::new(EIO))?; @@ -254,8 +283,12 @@ impl KernelScheme for SysScheme { } } - fn kfstat(&self, id: usize, buf: UserSliceWo) -> Result<()> { - let stat = match HANDLES.read().get(&id).ok_or(Error::new(EBADF))? { + fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + let stat = match HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))? + { Handle::Resource { data, .. } => Stat { st_mode: 0o666 | MODE_FILE, st_uid: 0, diff --git a/src/scheme/sys/scheme.rs b/src/scheme/sys/scheme.rs index f27f337eb2..8b6c3fe7b9 100644 --- a/src/scheme/sys/scheme.rs +++ b/src/scheme/sys/scheme.rs @@ -1,13 +1,13 @@ use alloc::vec::Vec; -use crate::{context, scheme, syscall::error::Result}; +use crate::{context, scheme, sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { - let scheme_ns = context::current().read().ens; +pub fn resource(token: &mut CleanLockToken) -> Result> { + let scheme_ns = context::current().read(token.token()).ens; let mut data = Vec::new(); - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); for (name, _scheme_id) in schemes.iter_name(scheme_ns) { data.extend_from_slice(name.as_bytes()); data.push(b'\n'); diff --git a/src/scheme/sys/scheme_num.rs b/src/scheme/sys/scheme_num.rs index 1409268a7b..33dbacd6bb 100644 --- a/src/scheme/sys/scheme_num.rs +++ b/src/scheme/sys/scheme_num.rs @@ -1,13 +1,13 @@ use alloc::vec::Vec; -use crate::{context, scheme, syscall::error::Result}; +use crate::{context, scheme, sync::CleanLockToken, syscall::error::Result}; -pub fn resource() -> Result> { - let scheme_ns = context::current().read().ens; +pub fn resource(token: &mut CleanLockToken) -> Result> { + let scheme_ns = context::current().read(token.token()).ens; let mut data = Vec::new(); - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); for (name, &scheme_id) in schemes.iter_name(scheme_ns) { data.extend_from_slice(format!("{:>4}: ", scheme_id.get()).as_bytes()); data.extend_from_slice(name.as_bytes()); diff --git a/src/scheme/sys/stat.rs b/src/scheme/sys/stat.rs index 14afea1dd1..f3b39ae2fa 100644 --- a/src/scheme/sys/stat.rs +++ b/src/scheme/sys/stat.rs @@ -2,16 +2,17 @@ use crate::{ context::{contexts, ContextRef, Status}, cpu_stats::{get_context_switch_count, get_contexts_count, irq_counts}, percpu::get_all_stats, + sync::CleanLockToken, syscall::error::Result, time::START, }; use alloc::{string::String, vec::Vec}; /// Get the sys:stat data as displayed to the user. -pub fn resource() -> Result> { +pub fn resource(token: &mut CleanLockToken) -> Result> { let start_time_sec = *START.lock() / 1_000_000_000; - let (contexts_running, contexts_blocked) = get_contexts_stats(); + let (contexts_running, contexts_blocked) = get_contexts_stats(token); let res = format!( "{}{}\n\ boot_time: {start_time_sec}\n\ @@ -68,11 +69,11 @@ fn get_irq_stats() -> String { } /// Format contexts stats. -fn get_contexts_stats() -> (u64, u64) { +fn get_contexts_stats(token: &mut CleanLockToken) -> (u64, u64) { let mut running = 0; let mut blocked = 0; - let statuses = contexts() + let statuses = contexts(token.token()) .iter() .filter_map(ContextRef::upgrade) .map(|context| context.read_arc().status.clone()) diff --git a/src/scheme/sys/syscall.rs b/src/scheme/sys/syscall.rs index 71b705bbb0..df124d5daa 100644 --- a/src/scheme/sys/syscall.rs +++ b/src/scheme/sys/syscall.rs @@ -1,17 +1,18 @@ use alloc::{string::String, vec::Vec}; use core::fmt::Write; -use crate::{context, syscall, syscall::error::Result}; +use crate::{context, sync::CleanLockToken, syscall, syscall::error::Result}; -pub fn resource() -> Result> { +pub fn resource(token: &mut CleanLockToken) -> Result> { let mut string = String::new(); { let mut rows = Vec::new(); { - let contexts = context::contexts(); + let mut contexts = context::contexts(token.token()); + let (contexts, mut token) = contexts.token_split(); for context_ref in contexts.iter().filter_map(|r| r.upgrade()) { - let context = context_ref.read(); + let context = context_ref.read(token.token()); rows.push((context.pid, context.name.clone(), context.current_syscall())); } } diff --git a/src/scheme/sys/uname.rs b/src/scheme/sys/uname.rs index c4136a0dcb..6583252027 100644 --- a/src/scheme/sys/uname.rs +++ b/src/scheme/sys/uname.rs @@ -1,7 +1,7 @@ -use crate::syscall::error::Result; +use crate::{sync::CleanLockToken, syscall::error::Result}; use alloc::vec::Vec; -pub fn resource() -> Result> { +pub fn resource(_token: &mut CleanLockToken) -> Result> { Ok(format!( "Redox\n\n{}\n\n{}\n", env!("CARGO_PKG_VERSION"), diff --git a/src/scheme/time.rs b/src/scheme/time.rs index 8a57030c18..2ab94d0811 100644 --- a/src/scheme/time.rs +++ b/src/scheme/time.rs @@ -3,10 +3,10 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use spin::RwLock; use crate::{ context::{file::InternalFlags, timeout}, + sync::{CleanLockToken, RwLock, L1}, syscall::{ data::TimeSpec, error::*, @@ -19,13 +19,19 @@ use crate::{ use super::{CallerCtx, GlobalSchemes, KernelScheme, OpenResult}; static NEXT_ID: AtomicUsize = AtomicUsize::new(1); -static HANDLES: RwLock> = +static HANDLES: RwLock> = RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new())); pub struct TimeScheme; impl KernelScheme for TimeScheme { - fn kopen(&self, path: &str, _flags: usize, _ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + _flags: usize, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let clock = path.parse::().map_err(|_| Error::new(ENOENT))?; match clock { @@ -35,37 +41,61 @@ impl KernelScheme for TimeScheme { } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, clock); + HANDLES.write(token.token()).insert(id, clock); Ok(OpenResult::SchemeLocal(id, InternalFlags::empty())) } - fn fcntl(&self, _id: usize, _cmd: usize, _arg: usize) -> Result { + fn fcntl( + &self, + _id: usize, + _cmd: usize, + _arg: usize, + _token: &mut CleanLockToken, + ) -> Result { Ok(0) } - fn fevent(&self, id: usize, _flags: EventFlags) -> Result { + fn fevent( + &self, + id: usize, + _flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { HANDLES - .read() + .read(token.token()) .get(&id) .ok_or(Error::new(EBADF)) .and(Ok(EventFlags::empty())) } - fn fsync(&self, id: usize) -> Result<()> { - HANDLES.read().get(&id).ok_or(Error::new(EBADF))?; + fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { + HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))?; Ok(()) } - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { HANDLES - .write() + .write(token.token()) .remove(&id) .ok_or(Error::new(EBADF)) .and(Ok(())) } - fn kread(&self, id: usize, buf: UserSliceWo, _flags: u32, _stored_flags: u32) -> Result { - let clock = *HANDLES.read().get(&id).ok_or(Error::new(EBADF))?; + fn kread( + &self, + id: usize, + buf: UserSliceWo, + _flags: u32, + _stored_flags: u32, + token: &mut CleanLockToken, + ) -> Result { + let clock = *HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))?; let mut bytes_read = 0; @@ -93,23 +123,30 @@ impl KernelScheme for TimeScheme { buf: UserSliceRo, _flags: u32, _stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { - let clock = *HANDLES.read().get(&id).ok_or(Error::new(EBADF))?; + let clock = *HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))?; let mut bytes_written = 0; for current_chunk in buf.in_exact_chunks(mem::size_of::()) { let time = unsafe { current_chunk.read_exact::()? }; - timeout::register(GlobalSchemes::Time.scheme_id(), id, clock, time); + timeout::register(GlobalSchemes::Time.scheme_id(), id, clock, time, token); bytes_written += mem::size_of::(); } Ok(bytes_written) } - fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { - let clock = *HANDLES.read().get(&id).ok_or(Error::new(EBADF))?; + fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { + let clock = *HANDLES + .read(token.token()) + .get(&id) + .ok_or(Error::new(EBADF))?; let scheme_path = format!("time:{}", clock).into_bytes(); buf.copy_common_bytes_from_slice(&scheme_path) diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 478f0329d7..6028ad0493 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -12,7 +12,6 @@ use core::{ }; use slab::Slab; use spin::{Mutex, RwLock}; -use spinning_top::RwSpinlock; use syscall::{ schemev2::{Cqe, CqeOpcode, Opcode, Sqe, SqeFlags}, CallFlags, FmoveFdFlags, FobtainFdFlags, MunmapFlags, RecvFdFlags, SchemeSocketCall, @@ -29,13 +28,13 @@ use crate::{ AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode, PageSpan, DANGLING, }, - BorrowedHtBuf, Context, Status, + BorrowedHtBuf, ContextLock, Status, }, event, memory::Frame, paging::{Page, VirtualAddress, PAGE_SIZE}, scheme::SchemeId, - sync::WaitQueue, + sync::{CleanLockToken, WaitQueue}, syscall::{ data::{Map, Packet}, error::*, @@ -54,7 +53,7 @@ pub struct UserInner { pub scheme_id: SchemeId, v2: bool, supports_on_close: bool, - context: Weak>, + context: Weak, todo: WaitQueue, // TODO: custom packed radix tree data structure @@ -65,13 +64,13 @@ pub struct UserInner { enum State { Waiting { - context: Weak>, + context: Weak, fds: Option>>>, callee_responsible: PageSpan, canceling: bool, }, Responded(Response), - Fmap(Weak>), + Fmap(Weak), Placeholder, } @@ -119,7 +118,7 @@ enum ParsedCqe { NoOp, // TODO: remove } impl ParsedCqe { - fn parse_packet(packet: &Packet) -> Result { + fn parse_packet(packet: &Packet, token: &mut CleanLockToken) -> Result { Ok(if packet.id == 0 { match packet.a { SYS_FEVENT => Self::TriggerFevent { @@ -130,7 +129,7 @@ impl ParsedCqe { warn!( "Unknown scheme -> kernel message {} from {}", packet.a, - context::current().read().name + context::current().read(token.token()).name ); // Some schemes don't implement cancellation properly yet, so we temporarily @@ -211,7 +210,7 @@ impl UserInner { handle_id: usize, name: Box, _flags: usize, - context: Weak>, + context: Weak, ) -> UserInner { UserInner { root_id, @@ -227,12 +226,12 @@ impl UserInner { } } - pub fn unmount(&self) -> Result<()> { + pub fn unmount(&self, token: &mut CleanLockToken) -> Result<()> { // First, block new requests and prepare to return EOF self.unmounting.store(true, Ordering::SeqCst); // Wake up any blocked scheme handler - unsafe { self.todo.condition.notify_signal() }; + unsafe { self.todo.condition.notify_signal(token) }; // Tell the scheme handler to read event::trigger(self.root_id, self.handle_id, EVENT_READ); @@ -256,9 +255,10 @@ impl UserInner { opcode: Opcode, args: impl Args, caller_responsible: &mut PageSpan, + token: &mut CleanLockToken, ) -> Result { - let ctx = { context::current().read().caller_ctx() }; - match self.call_extended(ctx, None, opcode, args, caller_responsible)? { + let ctx = { context::current().read(token.token()).caller_ctx() }; + match self.call_extended(ctx, None, opcode, args, caller_responsible, token)? { Response::Regular(code, _) => Error::demux(code), Response::Fd(_) => Err(Error::new(EIO)), Response::MultipleFds(_) => Err(Error::new(EIO)), @@ -272,6 +272,7 @@ impl UserInner { opcode: Opcode, args: impl Args, caller_responsible: &mut PageSpan, + token: &mut CleanLockToken, ) -> Result { let next_id = self.next_id()?; self.call_extended_inner( @@ -289,6 +290,7 @@ impl UserInner { }, }, caller_responsible, + token, ) } @@ -297,6 +299,7 @@ impl UserInner { fds: Option>>>, sqe: Sqe, caller_responsible: &mut PageSpan, + token: &mut CleanLockToken, ) -> Result { if self.unmounting.load(Ordering::SeqCst) { return Err(Error::new(ENODEV)); @@ -305,7 +308,9 @@ impl UserInner { { let current_context = context::current(); { - current_context.write().block("UserScheme::call") + current_context + .write(token.token()) + .block("UserScheme::call") }; { let mut states = self.states.lock(); @@ -321,18 +326,18 @@ impl UserInner { }; } } - self.todo.send(sqe); + self.todo.send(sqe, token); event::trigger(self.root_id, self.handle_id, EVENT_READ); loop { - context::switch(); + context::switch(token); { let mut eintr_if_sigkill = |callee_responsible: &mut PageSpan| { // If SIGKILL was found without waiting for scheme, EINTR directly. In that // case, data loss doesn't matter. - if context::current().read().being_sigkilled { + if context::current().read(token.token()).being_sigkilled { // Callee must deallocate memory, rather than the caller. This is less optimal // for TLB, but we don't really have any other choice. The scheme must be able // to access the borrowed memory until it has responded to the request. @@ -368,7 +373,9 @@ impl UserInner { maybe_eintr?; { - context::current().write().block("UserInner::call") + context::current() + .write(token.token()) + .block("UserInner::call") }; } // spurious wakeup @@ -390,15 +397,20 @@ impl UserInner { maybe_eintr?; // TODO: Is this too dangerous when the states lock is held? - self.todo.send(Sqe { - opcode: Opcode::Cancel as u8, - sqe_flags: SqeFlags::ONEWAY, - tag: sqe.tag, - ..Default::default() - }); + self.todo.send( + Sqe { + opcode: Opcode::Cancel as u8, + sqe_flags: SqeFlags::ONEWAY, + tag: sqe.tag, + ..Default::default() + }, + token, + ); event::trigger(self.root_id, self.handle_id, EVENT_READ); { - context::current().write().block("UserInner::call") + context::current() + .write(token.token()) + .block("UserInner::call") }; } @@ -424,21 +436,26 @@ impl UserInner { pub fn capture_user( &self, buf: UserSlice, + token: &mut CleanLockToken, ) -> Result> { - UserInner::capture_inner(&self.context, buf) + UserInner::capture_inner(&self.context, buf, token) } - pub fn copy_and_capture_tail(&self, buf: &[u8]) -> Result> { + pub fn copy_and_capture_tail( + &self, + buf: &[u8], + token: &mut CleanLockToken, + ) -> Result> { let dst_addr_space = { Arc::clone( self.context .upgrade() .ok_or(Error::new(ENODEV))? - .read() + .read(token.token()) .addr_space()?, ) }; - let mut tail = BorrowedHtBuf::tail()?; + let mut tail = BorrowedHtBuf::tail(token)?; let tail_frame = tail.frame(); if buf.len() > tail.buf().len() { return Err(Error::new(EINVAL)); @@ -489,8 +506,9 @@ impl UserInner { // TODO: Hypothetical accept_head_leak, accept_tail_leak options might be useful for // libc-controlled buffer pools. fn capture_inner( - context_weak: &Weak>, + context_weak: &Weak, user_buf: UserSlice, + token: &mut CleanLockToken, ) -> Result> { let mut map_flags = MapFlags::empty(); map_flags.set(MapFlags::PROT_READ, READ); @@ -527,7 +545,7 @@ impl UserInner { context_weak .upgrade() .ok_or(Error::new(ESRCH))? - .read() + .read(token.token()) .addr_space()?, ) }; @@ -567,7 +585,7 @@ impl UserInner { let head = if !head_part_of_buf.is_empty() { // FIXME: Signal context can probably recursively use head/tail. - let mut array = BorrowedHtBuf::head()?; + let mut array = BorrowedHtBuf::head(token)?; let frame = array.frame(); let len = core::cmp::min(PAGE_SIZE - offset, user_buf.len()); @@ -667,7 +685,7 @@ impl UserInner { let tail_dst_page = first_middle_dst_page.next_by(middle_page_count); // FIXME: Signal context can probably recursively use head/tail. - let mut array = BorrowedHtBuf::tail()?; + let mut array = BorrowedHtBuf::tail(token)?; let frame = array.frame(); if READ { @@ -725,7 +743,7 @@ impl UserInner { }) } - pub fn read(&self, buf: UserSliceWo, flags: u32) -> Result { + pub fn read(&self, buf: UserSliceWo, flags: u32, token: &mut CleanLockToken) -> Result { // If O_NONBLOCK is used, do not block let nonblock = flags & O_NONBLOCK as u32 != 0; @@ -735,7 +753,7 @@ impl UserInner { if self.v2 { return match self .todo - .receive_into_user(buf, block, "UserInner::read (v2)") + .receive_into_user(buf, block, "UserInner::read (v2)", token) { // If we received requests, return them to the scheme handler Ok(byte_count) => Ok(byte_count), @@ -751,7 +769,7 @@ impl UserInner { for dst in buf.in_exact_chunks(size_of::()) { match self .todo - .receive(block && bytes_read == 0, "UserInner::read (legacy)") + .receive(block && bytes_read == 0, "UserInner::read (legacy)", token) { Ok(sqe) => { dst.copy_exactly(&self.translate_sqe_to_packet(&sqe)?)?; @@ -876,12 +894,12 @@ impl UserInner { }) } - pub fn write(&self, buf: UserSliceRo) -> Result { + pub fn write(&self, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { let mut bytes_read = 0; if self.v2 { for chunk in buf.in_exact_chunks(size_of::()) { match ParsedCqe::parse_cqe(&unsafe { chunk.read_exact::()? }) - .and_then(|p| self.handle_parsed(&p)) + .and_then(|p| self.handle_parsed(&p, token)) { Ok(()) => bytes_read += size_of::(), Err(_) if bytes_read > 0 => break, @@ -890,8 +908,8 @@ impl UserInner { } } else { for chunk in buf.in_exact_chunks(size_of::()) { - match ParsedCqe::parse_packet(&unsafe { chunk.read_exact::()? }) - .and_then(|p| self.handle_parsed(&p)) + match ParsedCqe::parse_packet(&unsafe { chunk.read_exact::()? }, token) + .and_then(|p| self.handle_parsed(&p, token)) { Ok(()) => bytes_read += size_of::(), Err(_) if bytes_read > 0 => break, @@ -907,6 +925,7 @@ impl UserInner { _offset: u64, required_page_count: usize, flags: MapFlags, + token: &mut CleanLockToken, ) -> Result<()> { info!("REQUEST FMAP"); @@ -926,42 +945,46 @@ impl UserInner { uid: offset as u32, gid: (offset >> 32) as u32, });*/ - self.todo.send(Sqe { - opcode: Opcode::RequestMmap as u8, - sqe_flags: SqeFlags::empty(), - _rsvd: 0, - tag, - args: [ - id as u64, - flags.bits() as u64, - required_page_count as u64, - 0, - 0, - uid_gid_hack_merge(current_uid_gid()), - ], - caller: { context::current().read().pid as u64 }, - }); + self.todo.send( + Sqe { + opcode: Opcode::RequestMmap as u8, + sqe_flags: SqeFlags::empty(), + _rsvd: 0, + tag, + args: [ + id as u64, + flags.bits() as u64, + required_page_count as u64, + 0, + 0, + uid_gid_hack_merge(current_uid_gid(token)), + ], + caller: { context::current().read(token.token()).pid as u64 }, + }, + token, + ); event::trigger(self.root_id, self.handle_id, EVENT_READ); Ok(()) } - fn handle_parsed(&self, cqe: &ParsedCqe) -> Result<()> { + fn handle_parsed(&self, cqe: &ParsedCqe, token: &mut CleanLockToken) -> Result<()> { match *cqe { ParsedCqe::RegularResponse { tag, code, extra0 } => { - self.respond(tag, Response::Regular(code, extra0))? + self.respond(tag, Response::Regular(code, extra0), token)? } ParsedCqe::ResponseWithFd { tag, fd } => self.respond( tag, Response::Fd({ context::current() - .read() + .read(token.token()) .remove_file(FileHandle::from(fd)) .ok_or(Error::new(EINVAL))? .description }), + token, )?, ParsedCqe::ResponseWithMultipleFds { tag, num_fds } => { - self.respond(tag, Response::MultipleFds(None))?; + self.respond(tag, Response::MultipleFds(None), token)?; } ParsedCqe::ObtainFd { tag, @@ -984,7 +1007,7 @@ impl UserInner { // FIXME: Description can leak if there is no additional file table space. if flags.contains(FobtainFdFlags::MANUAL_FD) { - context::current().read().insert_file( + context::current().read(token.token()).insert_file( FileHandle::from(dst_fd_or_ptr), FileDescriptor { description, @@ -993,7 +1016,7 @@ impl UserInner { ); } else { let fd = context::current() - .read() + .read(token.token()) .add_file(FileDescriptor { description, cloexec: true, @@ -1057,7 +1080,7 @@ impl UserInner { .ok_or(Error::new(EFAULT))?; { - let mut context = context.write(); + let mut context = context.write(token.token()); match context.status { Status::HardBlocked { reason: HardBlockedReason::AwaitingMmap { .. }, @@ -1074,7 +1097,7 @@ impl UserInner { } Ok(()) } - fn respond(&self, tag: u32, mut response: Response) -> Result<()> { + fn respond(&self, tag: u32, mut response: Response, token: &mut CleanLockToken) -> Result<()> { let to_close: Vec; { @@ -1128,7 +1151,7 @@ impl UserInner { match context.upgrade() { Some(context) => { { - context.write().unblock() + context.write(token.token()).unblock() }; *o = State::Responded(response); } @@ -1147,7 +1170,7 @@ impl UserInner { } for fd in to_close { - let _ = fd.try_close(); + let _ = fd.try_close(token); } Ok(()) } @@ -1171,6 +1194,7 @@ impl UserInner { dst_addr_space: Arc, file: usize, map: &Map, + token: &mut CleanLockToken, ) -> Result { let unaligned_size = map.size; @@ -1195,7 +1219,7 @@ impl UserInner { self.context .upgrade() .ok_or(Error::new(ENODEV))? - .read() + .read(token.token()) .addr_space()?, ) }; @@ -1205,7 +1229,7 @@ impl UserInner { let (pid, desc) = { let context_lock = context::current(); - let context = context_lock.read(); + let context = context_lock.read(token.token()); let desc = context.files.read().find_by_scheme(self.scheme_id, file)?; (context.pid, desc.description) }; @@ -1239,11 +1263,12 @@ impl UserInner { map.flags.bits() as u64, map.offset as u64, 0, - uid_gid_hack_merge(current_uid_gid()), + uid_gid_hack_merge(current_uid_gid(token)), ], caller: pid as u64, }, &mut PageSpan::empty(), + token, )?; // TODO: I've previously tested that this works, but because the scheme trait all of @@ -1306,13 +1331,14 @@ impl UserInner { &dst_addr_space, mapper, flusher, + token, ) }, )? }; for map in notify_files { - let _ = map.unmap(); + let _ = map.unmap(token); } Ok(dst_base.start_address().data()) @@ -1375,6 +1401,7 @@ impl UserInner { payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { if metadata.is_empty() { return Err(Error::new(EINVAL)); @@ -1401,7 +1428,7 @@ impl UserInner { if flags.contains(CallFlags::FD_EXCLUSIVE) { obtainfd_flags |= FobtainFdFlags::EXCLUSIVE; } - self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags) + self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags, token) } _ => Err(Error::new(EINVAL)), } @@ -1412,6 +1439,7 @@ impl UserInner { payload: UserSliceRw, request_id: usize, flags: FobtainFdFlags, + token: &mut CleanLockToken, ) -> Result { let descriptions = match self .states @@ -1424,9 +1452,9 @@ impl UserInner { }; let num_fds = if flags.contains(FobtainFdFlags::UPPER_TBL) { - Self::bulk_insert_fds(descriptions, payload)? + Self::bulk_insert_fds(descriptions, payload, token)? } else { - Self::bulk_add_fds(descriptions, payload)? + Self::bulk_add_fds(descriptions, payload, token)? }; Ok(num_fds) @@ -1435,6 +1463,7 @@ impl UserInner { fn bulk_add_fds( descriptions: Vec>>, payload: UserSliceRw, + token: &mut CleanLockToken, ) -> Result { let cnt = descriptions.len(); if payload.len() != cnt * size_of::() { @@ -1444,7 +1473,7 @@ impl UserInner { return Ok(0); } let current_lock = context::current(); - let current = current_lock.write(); + let current = current_lock.write(token.token()); let files: Vec = descriptions .into_iter() @@ -1466,6 +1495,7 @@ impl UserInner { fn bulk_insert_fds( descriptions: Vec>>, payload: UserSliceRw, + token: &mut CleanLockToken, ) -> Result { let cnt = descriptions.len(); if payload.len() != cnt * size_of::() { @@ -1485,7 +1515,7 @@ impl UserInner { .read_usize()?; let current_lock = context::current(); - let current = current_lock.write(); + let current = current_lock.write(token.token()); if first_fd == usize::MAX { let files = files_iter.collect::>(); @@ -1600,15 +1630,22 @@ impl UserScheme { } impl KernelScheme for UserScheme { - fn kopen(&self, path: &str, flags: usize, ctx: CallerCtx) -> Result { + fn kopen( + &self, + path: &str, + flags: usize, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.copy_and_capture_tail(path.as_bytes())?; + let mut address = inner.copy_and_capture_tail(path.as_bytes(), token)?; match inner.call_extended( ctx, None, Opcode::Open, [address.base(), address.len(), flags], address.span(), + token, )? { Response::Regular(code, fl) => Ok({ let _ = Error::demux(code)?; @@ -1629,15 +1666,17 @@ impl KernelScheme for UserScheme { flags: usize, fcntl_flags: u32, ctx: CallerCtx, + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.copy_and_capture_tail(path.as_bytes())?; + let mut address = inner.copy_and_capture_tail(path.as_bytes(), token)?; let result = inner.call_extended( ctx, None, Opcode::OpenAt, [file, address.base(), address.len(), flags, fcntl_flags as _], address.span(), + token, ); address.release()?; @@ -1655,51 +1694,54 @@ impl KernelScheme for UserScheme { } } - fn rmdir(&self, path: &str, _ctx: CallerCtx) -> Result<()> { + fn rmdir(&self, path: &str, _ctx: CallerCtx, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.copy_and_capture_tail(path.as_bytes())?; + let mut address = inner.copy_and_capture_tail(path.as_bytes(), token)?; inner.call( Opcode::Rmdir, [address.base(), address.len()], address.span(), + token, )?; Ok(()) } - fn unlink(&self, path: &str, _ctx: CallerCtx) -> Result<()> { + fn unlink(&self, path: &str, _ctx: CallerCtx, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.copy_and_capture_tail(path.as_bytes())?; + let mut address = inner.copy_and_capture_tail(path.as_bytes(), token)?; inner.call( Opcode::Unlink, [address.base(), address.len()], address.span(), + token, )?; Ok(()) } - fn fsize(&self, file: usize) -> Result { + fn fsize(&self, file: usize, token: &mut CleanLockToken) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; if !inner.v2 { return Err(Error::new(ESPIPE)); } inner - .call(Opcode::Fsize, [file], &mut PageSpan::empty()) + .call(Opcode::Fsize, [file], &mut PageSpan::empty(), token) .map(|o| o as u64) } - fn fchmod(&self, file: usize, mode: u16) -> Result<()> { + fn fchmod(&self, file: usize, mode: u16, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; inner.call( Opcode::Fchmod, [file, mode as usize], &mut PageSpan::empty(), + token, )?; Ok(()) } - fn fchown(&self, file: usize, uid: u32, gid: u32) -> Result<()> { + fn fchown(&self, file: usize, uid: u32, gid: u32, token: &mut CleanLockToken) -> Result<()> { { - match context::current().read() { + match context::current().read(token.token()) { ref cx => { if cx.euid != 0 && (uid != cx.euid || gid != cx.egid) { return Err(Error::new(EPERM)); @@ -1713,86 +1755,137 @@ impl KernelScheme for UserScheme { Opcode::Fchown, [file, uid as usize, gid as usize], &mut PageSpan::empty(), + token, )?; Ok(()) } - fn fcntl(&self, file: usize, cmd: usize, arg: usize) -> Result { + fn fcntl( + &self, + file: usize, + cmd: usize, + arg: usize, + token: &mut CleanLockToken, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - inner.call(Opcode::Fcntl, [file, cmd, arg], &mut PageSpan::empty()) + inner.call( + Opcode::Fcntl, + [file, cmd, arg], + &mut PageSpan::empty(), + token, + ) } - fn fevent(&self, file: usize, flags: EventFlags) -> Result { + fn fevent( + &self, + file: usize, + flags: EventFlags, + token: &mut CleanLockToken, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; inner - .call(Opcode::Fevent, [file, flags.bits()], &mut PageSpan::empty()) + .call( + Opcode::Fevent, + [file, flags.bits()], + &mut PageSpan::empty(), + token, + ) .map(EventFlags::from_bits_truncate) } - fn flink(&self, file: usize, path: &str, _ctx: CallerCtx) -> Result<()> { + fn flink( + &self, + file: usize, + path: &str, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.copy_and_capture_tail(path.as_bytes())?; + let mut address = inner.copy_and_capture_tail(path.as_bytes(), token)?; inner.call( Opcode::Flink, [file, address.base(), address.len()], address.span(), + token, )?; Ok(()) } - fn frename(&self, file: usize, path: &str, _ctx: CallerCtx) -> Result<()> { + fn frename( + &self, + file: usize, + path: &str, + _ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.copy_and_capture_tail(path.as_bytes())?; + let mut address = inner.copy_and_capture_tail(path.as_bytes(), token)?; inner.call( Opcode::Frename, [file, address.base(), address.len()], address.span(), + token, )?; Ok(()) } - fn fsync(&self, file: usize) -> Result<()> { + fn fsync(&self, file: usize, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - inner.call(Opcode::Fsync, [file], &mut PageSpan::empty())?; + inner.call(Opcode::Fsync, [file], &mut PageSpan::empty(), token)?; Ok(()) } - fn ftruncate(&self, file: usize, len: usize) -> Result<()> { + fn ftruncate(&self, file: usize, len: usize, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - inner.call(Opcode::Ftruncate, [file, len], &mut PageSpan::empty())?; + inner.call( + Opcode::Ftruncate, + [file, len], + &mut PageSpan::empty(), + token, + )?; Ok(()) } - fn close(&self, id: usize) -> Result<()> { + fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; if !inner.supports_on_close { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - inner.call(Opcode::Close, [id], &mut PageSpan::empty())?; + inner.call(Opcode::Close, [id], &mut PageSpan::empty(), token)?; return Ok(()); } - inner.todo.send(Sqe { - opcode: Opcode::CloseMsg as u8, - sqe_flags: SqeFlags::empty(), - _rsvd: 0, - tag: 0, - args: [id as u64, 0, 0, 0, 0, 0], - caller: 0, // TODO? - }); + inner.todo.send( + Sqe { + opcode: Opcode::CloseMsg as u8, + sqe_flags: SqeFlags::empty(), + _rsvd: 0, + tag: 0, + args: [id as u64, 0, 0, 0, 0, 0], + caller: 0, // TODO? + }, + token, + ); event::trigger(inner.root_id, inner.handle_id, EVENT_READ); Ok(()) } - fn kdup(&self, file: usize, buf: UserSliceRo, ctx: CallerCtx) -> Result { + fn kdup( + &self, + file: usize, + buf: UserSliceRo, + ctx: CallerCtx, + token: &mut CleanLockToken, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(buf)?; + let mut address = inner.capture_user(buf, token)?; let result = inner.call_extended( ctx, None, Opcode::Dup, [file, address.base(), address.len()], address.span(), + token, ); address.release()?; @@ -1809,13 +1902,14 @@ impl KernelScheme for UserScheme { Response::MultipleFds(_) => Err(Error::new(EIO)), } } - fn kfpath(&self, file: usize, buf: UserSliceWo) -> Result { + fn kfpath(&self, file: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(buf)?; + let mut address = inner.capture_user(buf, token)?; let result = inner.call( Opcode::Fpath, [file, address.base(), address.len()], address.span(), + token, ); address.release()?; result @@ -1828,14 +1922,15 @@ impl KernelScheme for UserScheme { offset: u64, call_flags: u32, stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; if call_flags != stored_flags && !inner.v2 { - self.fcntl(file, F_SETFL, call_flags as usize)?; + self.fcntl(file, F_SETFL, call_flags as usize, token)?; } - let mut address = inner.capture_user(buf)?; + let mut address = inner.capture_user(buf, token)?; let result = inner.call( Opcode::Read, [ @@ -1846,11 +1941,12 @@ impl KernelScheme for UserScheme { u64::from(call_flags), ], address.span(), + token, ); address.release()?; if call_flags != stored_flags && !inner.v2 { - self.fcntl(file, F_SETFL, stored_flags as usize)?; + self.fcntl(file, F_SETFL, stored_flags as usize, token)?; } result @@ -1863,13 +1959,14 @@ impl KernelScheme for UserScheme { offset: u64, call_flags: u32, stored_flags: u32, + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; if call_flags != stored_flags && !inner.v2 { - self.fcntl(file, F_SETFL, call_flags as usize)?; + self.fcntl(file, F_SETFL, call_flags as usize, token)?; } - let mut address = inner.capture_user(buf)?; + let mut address = inner.capture_user(buf, token)?; let result = inner.call( Opcode::Write, [ @@ -1880,16 +1977,23 @@ impl KernelScheme for UserScheme { u64::from(call_flags), ], address.span(), + token, ); address.release()?; if call_flags != stored_flags && !inner.v2 { - self.fcntl(file, F_SETFL, stored_flags as usize)?; + self.fcntl(file, F_SETFL, stored_flags as usize, token)?; } result } - fn legacy_seek(&self, id: usize, pos: isize, whence: usize) -> Option> { + fn legacy_seek( + &self, + id: usize, + pos: isize, + whence: usize, + token: &mut CleanLockToken, + ) -> Option> { let inner = self.inner.upgrade()?; if inner.v2 { return None; @@ -1898,15 +2002,22 @@ impl KernelScheme for UserScheme { Opcode::Fsize, [id, pos as usize, whence], &mut PageSpan::empty(), + token, )) } - fn kfutimens(&self, file: usize, buf: UserSliceRo) -> Result { + fn kfutimens( + &self, + file: usize, + buf: UserSliceRo, + token: &mut CleanLockToken, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(buf)?; + let mut address = inner.capture_user(buf, token)?; let result = inner.call( Opcode::Futimens, [file, address.base(), address.len()], address.span(), + token, ); address.release()?; result @@ -1917,9 +2028,10 @@ impl KernelScheme for UserScheme { buf: UserSliceWo, header_size: u16, opaque_id_start: u64, + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(buf)?; + let mut address = inner.capture_user(buf, token)?; // TODO: Support passing the 16-byte record_len of the last dent, to make it possible to // iterate backwards without first interating forward? The last entry will contain the // opaque id to pass to the next getdents. Since this field is small, this would fit in the @@ -1934,28 +2046,31 @@ impl KernelScheme for UserScheme { opaque_id_start as usize, ], address.span(), + token, ); address.release()?; result } - fn kfstat(&self, file: usize, stat: UserSliceWo) -> Result<()> { + fn kfstat(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(stat)?; + let mut address = inner.capture_user(stat, token)?; let result = inner.call( Opcode::Fstat, [file, address.base(), address.len()], address.span(), + token, ); address.release()?; result.map(|_| ()) } - fn kfstatvfs(&self, file: usize, stat: UserSliceWo) -> Result<()> { + fn kfstatvfs(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(stat)?; + let mut address = inner.capture_user(stat, token)?; let result = inner.call( Opcode::Fstatvfs, [file, address.base(), address.len()], address.span(), + token, ); address.release()?; result.map(|_| ()) @@ -1966,21 +2081,30 @@ impl KernelScheme for UserScheme { addr_space: &Arc, map: &Map, _consume: bool, + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - inner.fmap_inner(Arc::clone(addr_space), file, map) + inner.fmap_inner(Arc::clone(addr_space), file, map, token) } - fn kfunmap(&self, number: usize, offset: usize, size: usize, flags: MunmapFlags) -> Result<()> { + fn kfunmap( + &self, + number: usize, + offset: usize, + size: usize, + flags: MunmapFlags, + token: &mut CleanLockToken, + ) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let ctx = { context::current().read().caller_ctx() }; + let ctx = { context::current().read(token.token()).caller_ctx() }; let res = inner.call_extended( ctx, None, Opcode::Munmap, [number, size, flags.bits(), offset], &mut PageSpan::empty(), + token, )?; match res { @@ -1995,11 +2119,12 @@ impl KernelScheme for UserScheme { payload: UserSliceRw, _flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let mut address = inner.capture_user(payload)?; - let ctx = { context::current().read().caller_ctx() }; + let mut address = inner.capture_user(payload, token)?; + let ctx = { context::current().read(token.token()).caller_ctx() }; let mut sqe = Sqe { opcode: Opcode::Call as u8, @@ -2021,7 +2146,7 @@ impl KernelScheme for UserScheme { let len = dst.len().min(metadata.len()); dst[..len].copy_from_slice(&metadata[..len]); } - let res = inner.call_extended_inner(None, sqe, &mut address.span())?; + let res = inner.call_extended_inner(None, sqe, &mut address.span(), token)?; match res { Response::Regular(res, _) => Error::demux(res), @@ -2036,6 +2161,7 @@ impl KernelScheme for UserScheme { flags: CallFlags, arg: u64, _metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; @@ -2044,7 +2170,7 @@ impl KernelScheme for UserScheme { sendfd_flags |= SendFdFlags::EXCLUSIVE; } - let ctx = { context::current().read().caller_ctx() }; + let ctx = { context::current().read(token.token()).caller_ctx() }; let len = descs.len(); let res = inner.call_extended( ctx, @@ -2052,6 +2178,7 @@ impl KernelScheme for UserScheme { Opcode::Sendfd, [number, sendfd_flags.bits(), arg as usize, len], &mut PageSpan::empty(), + token, )?; match res { @@ -2066,6 +2193,7 @@ impl KernelScheme for UserScheme { payload: UserSliceRw, flags: CallFlags, _metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; if payload.len() % mem::size_of::() != 0 { @@ -2077,7 +2205,7 @@ impl KernelScheme for UserScheme { recvfd_flags |= RecvFdFlags::UPPER_TBL; } - let ctx = { context::current().read().caller_ctx() }; + let ctx = { context::current().read(token.token()).caller_ctx() }; let len = payload.len() / mem::size_of::(); let res = inner.call_extended( ctx, @@ -2085,6 +2213,7 @@ impl KernelScheme for UserScheme { Opcode::Recvfd, [id, recvfd_flags.bits(), len], &mut PageSpan::empty(), + token, )?; let descriptions_opt = match res { @@ -2100,9 +2229,9 @@ impl KernelScheme for UserScheme { let num_fds = if let Some(descriptions) = descriptions_opt { if recvfd_flags.contains(RecvFdFlags::UPPER_TBL) { - UserInner::bulk_insert_fds(descriptions, payload)? + UserInner::bulk_insert_fds(descriptions, payload, token)? } else { - UserInner::bulk_add_fds(descriptions, payload)? + UserInner::bulk_add_fds(descriptions, payload, token)? } } else { 0 @@ -2133,8 +2262,8 @@ impl Args for [usize; N] { fn uid_gid_hack_merge([uid, gid]: [u32; 2]) -> u64 { u64::from(uid) | (u64::from(gid) << 32) } -fn current_uid_gid() -> [u32; 2] { - match context::current().read() { +fn current_uid_gid(token: &mut CleanLockToken) -> [u32; 2] { + match context::current().read(token.token()) { ref p => [p.euid, p.egid], } } diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 74a79956e7..6ad2708ba4 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,4 +1,5 @@ -pub use self::{wait_condition::WaitCondition, wait_queue::WaitQueue}; +pub use self::{ordered::*, wait_condition::WaitCondition, wait_queue::WaitQueue}; +pub mod ordered; pub mod wait_condition; pub mod wait_queue; diff --git a/src/sync/ordered.rs b/src/sync/ordered.rs new file mode 100644 index 0000000000..c7ef18a0d3 --- /dev/null +++ b/src/sync/ordered.rs @@ -0,0 +1,440 @@ +// This code was adapted from MIT licensed https://github.com/antialize/ordered-locks +// We cannot use that library directly as it is wrapping std::sync types + +//! This create implement compiletime ordering of locks into levels, [`L1`], [`L2`], [`L3`], [`L4`] and [`L5`]. +//! In order to acquire a lock at level `i` only locks at level `i-1` or below may be held. +//! +//! If locks are alwayes acquired in level order on all threads, then one cannot have a deadlock +//! involving only acquireng locks. +//! +//! In the following example we create two [muteces](Mutex) at level [`L1`] and [`L2`] and lock them +//! in the propper order. +//! ``` +//! use ordered_locks::{L1, L2, Mutex, CleanLockToken}; +//! // Create value at lock level 0, this lock cannot be acquired while a level1 lock is heldt +//! let v1 = Mutex::::new(42); +//! // Create value at lock level 1 +//! let v2 = Mutex::::new(43); +//! // Construct a token indicating that this thread does not hold any locks +//! let mut token = unsafe {CleanLockToken::new()}; +//! +//! { +//! // We can aquire the locks for v1 and v2 at the same time +//! let mut g1 = v1.lock(token.token()); +//! let (g1, token) = g1.token_split(); +//! let mut g2 = v2.lock(token); +//! *g2 = 11; +//! *g1 = 12; +//! } +//! // Once the guards are dropped we can acquire other things +//! *v2.lock(token.token()) = 13; +//! ``` +//! +//! In the following example we create two [muteces](Mutex) at level [`L1`] and [`L2`] and try to lock +//! the mutex at [`L1`] while already holding a [`Mutex`] at [`L2`] which failes to compile. +//! ```compile_fail +//! use ordered_locks::{L1, L2, Mutex, CleanLockToken}; +//! // Create value at lock level 0, this lock cannot be acquired while a level1 lock is heldt +//! let v1 = Mutex::::new(42); +//! // Create value at lock level 1 +//! let v2 = Mutex::::new(43); +//! // Construct a token indicating that this thread does not hold any locks +//! let mut clean_token = unsafe {CleanLockToken::new()}; +//! let token = clean_token.token(); +//! +//! // Try to aquire locks in the wrong order +//! let mut g2 = v2.lock(token); +//! let (g2, token) = g2.token_split(); +//! let mut g1 = v1.lock(token); // shouldn't compile! +//! *g2 = 11; +//! *g1 = 12; +//! ``` +use alloc::sync::Arc; +use core::marker::PhantomData; + +/// Lock level of a mutex +/// +/// While a mutex of L1 is locked on a thread, only mutexes of L2 or higher may be locked. +/// This lock hierarchy prevents deadlocks from occurring. For a dead lock to occour +/// We need some thread TA to hold a resource RA, and request a resource RB, while +/// another thread TB holds RB, and requests RA. This is not possible with a lock +/// hierarchy either RA or RB must be on a level that the other. +/// +/// At some point in time we would want Level to be replaced by usize, however +/// with current cont generics (rust 1.55), we cannot compare const generic arguments +/// so we are left with this mess. +pub trait Level {} + +/// Indicate that the implementor is lower that the level O +pub trait Lower: Level {} + +/// Lowest locking level, no locks can be on this level +#[derive(Debug)] +pub struct L0 {} + +#[derive(Debug)] +pub struct L1 {} + +#[derive(Debug)] +pub struct L2 {} + +#[derive(Debug)] +pub struct L3 {} + +#[derive(Debug)] +pub struct L4 {} + +#[derive(Debug)] +pub struct L5 {} + +impl Level for L0 {} +impl Level for L1 {} +impl Level for L2 {} +impl Level for L3 {} +impl Level for L4 {} +impl Level for L5 {} + +impl Lower for L0 {} +impl Lower for L0 {} +impl Lower for L0 {} +impl Lower for L0 {} +impl Lower for L0 {} + +impl Lower for L1 {} +impl Lower for L1 {} +impl Lower for L1 {} +impl Lower for L1 {} + +impl Lower for L2 {} +impl Lower for L2 {} +impl Lower for L2 {} + +impl Lower for L3 {} +impl Lower for L3 {} + +impl Lower for L4 {} + +/// Indicate that the implementor is higher that the level O +pub trait Higher: Level {} +impl Higher for L1 where L2: Lower {} + +/// While this exists only locks with a level higher than L, may be locked. +/// These tokens are carried around the call stack to indicate tho current locking level. +/// They have no size and should disappear at runtime. +pub struct LockToken<'a, L: Level>(PhantomData<&'a mut L>); + +impl<'a, L: Level> LockToken<'a, L> { + /// Create a borrowed copy of self + pub fn token(&mut self) -> LockToken<'_, L> { + LockToken(Default::default()) + } + + /// Create a borrowed copy of self, on a higher level + pub fn downgrade>(&mut self) -> LockToken<'_, LC> { + LockToken(Default::default()) + } + + pub fn downgraded>(_: LockToken<'a, LP>) -> Self { + LockToken(Default::default()) + } +} + +/// Token indicating that there are no acquired locks while not borrowed. +pub struct CleanLockToken(()); + +impl CleanLockToken { + /// Create a borrowed copy of self + pub fn token(&mut self) -> LockToken<'_, L0> { + LockToken(Default::default()) + } + + /// Create a borrowed copy of self, on a higher level + pub fn downgrade(&mut self) -> LockToken<'_, L> { + LockToken(Default::default()) + } + + /// Create a new instance + /// + /// # Safety + /// + /// This is safe to call as long as there are no currently acquired locks + /// in the thread/task, and as long as there are no other CleanLockToken + /// in the thread/task. + /// + /// A CleanLockToken + pub unsafe fn new() -> Self { + CleanLockToken(()) + } +} + +/// A mutual exclusion primitive useful for protecting shared data +/// +/// This mutex will block threads waiting for the lock to become available. The +/// mutex can also be statically initialized or created via a `new` +/// constructor. Each mutex has a type parameter which represents the data that +/// it is protecting. The data can only be accessed through the RAII guards +/// returned from `lock` and `try_lock`, which guarantees that the data is only +/// ever accessed when the mutex is locked. +#[derive(Debug)] +pub struct Mutex { + inner: spin::Mutex, + _phantom: PhantomData, +} + +impl Default for Mutex { + fn default() -> Self { + Self { + inner: Default::default(), + _phantom: Default::default(), + } + } +} + +impl Mutex { + /// Creates a new mutex in an unlocked state ready for use + pub const fn new(val: T) -> Self { + Self { + inner: spin::Mutex::new(val), + _phantom: PhantomData, + } + } + + /// Acquires a mutex, blocking the current thread until it is able to do so. + /// + /// This function will block the local thread until it is available to acquire the mutex. + /// Upon returning, the thread is the only thread with the mutex held. + /// An RAII guard is returned to allow scoped unlock of the lock. When the guard goes out of scope, the mutex will be unlocked. + pub fn lock<'a, LP: Lower + 'a>( + &'a self, + lock_token: LockToken<'a, LP>, + ) -> MutexGuard<'a, L, T> { + MutexGuard { + inner: self.inner.lock(), + lock_token: LockToken::downgraded(lock_token), + } + } + + /// Attempts to acquire this lock. + /// + /// If the lock could not be acquired at this time, then `None` is returned. + /// Otherwise, an RAII guard is returned. The lock will be unlocked when the + /// guard is dropped. + /// + /// This function does not block. + pub fn try_lock<'a, LP: Lower + 'a>( + &'a self, + lock_token: LockToken<'a, LP>, + ) -> Option> { + match self.inner.try_lock() { + Some(inner) => Some(MutexGuard { + inner, + lock_token: LockToken::downgraded(lock_token), + }), + None => None, + } + } + + /// Consumes this Mutex, returning the underlying data. + pub fn into_inner(self) -> T { + self.inner.into_inner() + } +} + +/// An RAII implementation of a "scoped lock" of a mutex. When this structure is +/// dropped (falls out of scope), the lock will be unlocked. +/// +/// The data protected by the mutex can be accessed through this guard via its +/// `Deref` and `DerefMut` implementations. +pub struct MutexGuard<'a, L: Level, T: ?Sized + 'a> { + inner: spin::MutexGuard<'a, T>, + lock_token: LockToken<'a, L>, +} + +impl<'a, L: Level, T: ?Sized + 'a> MutexGuard<'a, L, T> { + /// Split the guard into two parts, the first a mutable reference to the held content + /// the second a [`LockToken`] that can be used for further locking + pub fn token_split(&mut self) -> (&mut T, LockToken<'_, L>) { + (&mut self.inner, self.lock_token.token()) + } +} + +impl<'a, L: Level, T: ?Sized + 'a> core::ops::Deref for MutexGuard<'a, L, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} +impl<'a, L: Level, T: ?Sized + 'a> core::ops::DerefMut for MutexGuard<'a, L, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +pub struct RwLock { + inner: spin::RwLock, + _phantom: PhantomData, +} + +impl Default for RwLock { + fn default() -> Self { + Self { + inner: Default::default(), + _phantom: Default::default(), + } + } +} + +/// A reader-writer lock +/// +/// This type of lock allows a number of readers or at most one writer at any point in time. +/// The write portion of this lock typically allows modification of the underlying data (exclusive access) +/// and the read portion of this lock typically allows for read-only access (shared access). +/// +/// The type parameter T represents the data that this lock protects. It is required that T satisfies +/// Send to be shared across threads and Sync to allow concurrent access through readers. +/// The RAII guards returned from the locking methods implement Deref (and DerefMut for the write methods) +/// to allow access to the contained of the lock. +impl RwLock { + /// Creates a new instance of an RwLock which is unlocked. + pub const fn new(val: T) -> Self { + Self { + inner: spin::RwLock::new(val), + _phantom: PhantomData, + } + } + + /// Consumes this RwLock, returning the underlying data. + pub fn into_inner(self) -> T { + self.inner.into_inner() + } + + /// Locks this RwLock with exclusive write access, blocking the current thread until it can be acquired. + /// This function will not return while other writers or other readers currently have access to the lock. + /// Returns an RAII guard which will drop the write access of this RwLock when dropped. + pub fn write<'a, LP: Lower + 'a>( + &'a self, + lock_token: LockToken<'a, LP>, + ) -> RwLockWriteGuard<'a, L, T> { + RwLockWriteGuard { + inner: self.inner.write(), + lock_token: LockToken::downgraded(lock_token), + } + } + + /// Locks this RwLock with shared read access, blocking the current thread until it can be acquired. + /// + /// The calling thread will be blocked until there are no more writers which hold the lock. + /// There may be other readers currently inside the lock when this method returns. + /// + /// Note that attempts to recursively acquire a read lock on a RwLock when the current thread + /// already holds one may result in a deadlock. + /// + /// Returns an RAII guard which will release this thread’s shared access once it is dropped. + pub fn read<'a, LP: Lower + 'a>( + &'a self, + lock_token: LockToken<'a, LP>, + ) -> RwLockReadGuard<'a, L, T> { + RwLockReadGuard { + inner: self.inner.read(), + lock_token: LockToken::downgraded(lock_token), + } + } + + // Unsafe due to not using token, currently required by context::switch + pub unsafe fn write_arc(self: &Arc) -> ArcRwLockWriteGuard { + core::mem::forget(self.inner.write()); + ArcRwLockWriteGuard { + rwlock: self.clone(), + } + } +} + +/// RAII structure used to release the exclusive write access of a lock when dropped +pub struct RwLockWriteGuard<'a, L: Level, T> { + inner: spin::RwLockWriteGuard<'a, T>, + lock_token: LockToken<'a, L>, +} + +impl<'a, L: Level, T> RwLockWriteGuard<'a, L, T> { + /// Split the guard into two parts, the first a mutable reference to the held content + /// the second a [`LockToken`] that can be used for further locking + pub fn token_split(&mut self) -> (&mut T, LockToken<'_, L>) { + (&mut self.inner, self.lock_token.token()) + } +} + +impl<'a, L: Level, T> core::ops::Deref for RwLockWriteGuard<'a, L, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, L: Level, T> core::ops::DerefMut for RwLockWriteGuard<'a, L, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +/// RAII structure used to release the shared read access of a lock when dropped. +pub struct RwLockReadGuard<'a, L: Level, T> { + inner: spin::RwLockReadGuard<'a, T>, + lock_token: LockToken<'a, L>, +} + +impl<'a, L: Level, T> RwLockReadGuard<'a, L, T> { + /// Split the guard into two parts, the first a reference to the held content + /// the second a [`LockToken`] that can be used for further locking + pub fn token_split(&mut self) -> (&T, LockToken<'_, L>) { + (&self.inner, self.lock_token.token()) + } +} + +impl<'a, L: Level, T> core::ops::Deref for RwLockReadGuard<'a, L, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +pub struct ArcRwLockWriteGuard { + rwlock: Arc>, +} + +impl ArcRwLockWriteGuard { + pub fn rwlock(s: &Self) -> &Arc> { + &s.rwlock + } +} + +impl core::ops::Deref for ArcRwLockWriteGuard { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + unsafe { &*self.rwlock.inner.as_mut_ptr() } + } +} + +impl core::ops::DerefMut for ArcRwLockWriteGuard { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.rwlock.inner.as_mut_ptr() } + } +} + +impl Drop for ArcRwLockWriteGuard { + #[inline] + fn drop(&mut self) { + unsafe { + self.rwlock.inner.force_write_unlock(); + } + } +} + +/// This function can only be called if no lock is held by the calling thread/task +#[inline] +pub fn check_no_locks(_: LockToken<'_, L0>) {} diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index 57535edd2d..ddbe2e65c9 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -2,14 +2,15 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; -use spin::Mutex; -use spinning_top::RwSpinlock; -use crate::context::{self, Context}; +use crate::{ + context::{self, ContextLock}, + sync::{CleanLockToken, Mutex, L1}, +}; #[derive(Debug)] pub struct WaitCondition { - contexts: Mutex>>>, + contexts: Mutex>>, } impl WaitCondition { @@ -20,35 +21,37 @@ impl WaitCondition { } // Notify all waiters - pub fn notify(&self) -> usize { - let mut contexts = self.contexts.lock(); + pub fn notify(&self, token: &mut CleanLockToken) -> usize { + let mut contexts = self.contexts.lock(token.token()); + let (contexts, mut token) = contexts.token_split(); let len = contexts.len(); while let Some(context_weak) = contexts.pop() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write().unblock(); + context_ref.write(token.token()).unblock(); } } len } // Notify as though a signal woke the waiters - pub unsafe fn notify_signal(&self) -> usize { - let contexts = self.contexts.lock(); + pub unsafe fn notify_signal(&self, token: &mut CleanLockToken) -> usize { + let mut contexts = self.contexts.lock(token.token()); + let (contexts, mut token) = contexts.token_split(); let len = contexts.len(); for context_weak in contexts.iter() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write().unblock(); + context_ref.write(token.token()).unblock(); } } len } // Wait until notified. Unlocks guard when blocking is ready. Returns false if resumed by a signal or the notify_signal function - pub fn wait(&self, guard: T, reason: &'static str) -> bool { + pub fn wait(&self, guard: T, reason: &'static str, token: &mut CleanLockToken) -> bool { let current_context_ref = context::current(); { { - let mut context = current_context_ref.write(); + let mut context = current_context_ref.write(token.token()); if let Some((control, pctl, _)) = context.sigcontrol() && control.currently_pending_unblocked(pctl) != 0 { @@ -58,18 +61,18 @@ impl WaitCondition { } self.contexts - .lock() + .lock(token.token()) .push(Arc::downgrade(¤t_context_ref)); drop(guard); } - context::switch(); + context::switch(token); let mut waited = true; { - let mut contexts = self.contexts.lock(); + let mut contexts = self.contexts.lock(token.token()); // TODO: retain let mut i = 0; @@ -90,6 +93,10 @@ impl WaitCondition { impl Drop for WaitCondition { fn drop(&mut self) { - unsafe { self.notify_signal() }; + //TODO: drop violates lock tokens + unsafe { + let mut token = CleanLockToken::new(); + self.notify_signal(&mut token); + }; } } diff --git a/src/sync/wait_queue.rs b/src/sync/wait_queue.rs index 39a4489c58..408c3714f0 100644 --- a/src/sync/wait_queue.rs +++ b/src/sync/wait_queue.rs @@ -3,7 +3,7 @@ use spin::Mutex; use syscall::{EAGAIN, EINTR}; use crate::{ - sync::WaitCondition, + sync::{CleanLockToken, WaitCondition}, syscall::{ error::{Error, Result, EINVAL}, usercopy::UserSliceWo, @@ -27,7 +27,12 @@ impl WaitQueue { self.inner.lock().is_empty() } - pub fn receive(&self, block: bool, reason: &'static str) -> Result { + pub fn receive( + &self, + block: bool, + reason: &'static str, + token: &mut CleanLockToken, + ) -> Result { loop { let mut inner = self.inner.lock(); @@ -37,7 +42,7 @@ impl WaitQueue { } _ => { if block { - if !self.condition.wait(inner, reason) { + if !self.condition.wait(inner, reason, token) { return Err(Error::new(EINTR)); } continue; @@ -54,13 +59,14 @@ impl WaitQueue { buf: UserSliceWo, block: bool, reason: &'static str, + token: &mut CleanLockToken, ) -> Result { loop { let mut inner = self.inner.lock(); if inner.is_empty() { if block { - if !self.condition.wait(inner, reason) { + if !self.condition.wait(inner, reason, token) { return Err(Error::new(EINTR)); } continue; @@ -100,13 +106,13 @@ impl WaitQueue { } } - pub fn send(&self, value: T) -> usize { + pub fn send(&self, value: T, token: &mut CleanLockToken) -> usize { let len = { let mut inner = self.inner.lock(); inner.push_back(value); inner.len() }; - self.condition.notify(); + self.condition.notify(token); len } } diff --git a/src/syscall/debug.rs b/src/syscall/debug.rs index 5b5538be73..d0acbf707c 100644 --- a/src/syscall/debug.rs +++ b/src/syscall/debug.rs @@ -9,7 +9,7 @@ use super::{ usercopy::UserSlice, }; -use crate::syscall::error::Result; +use crate::{sync::CleanLockToken, syscall::error::Result}; struct ByteStr<'a>(&'a [u8]); @@ -217,12 +217,17 @@ impl SyscallDebugInfo { } #[cfg_attr(feature = "syscall_debug", inline)] -pub fn debug_start([a, b, c, d, e, f]: [usize; 6]) { +pub fn debug_start([a, b, c, d, e, f]: [usize; 6], token: &mut CleanLockToken) { if cfg!(not(feature = "syscall_debug")) { return; } - let do_debug = if false && crate::context::current().read().name.contains("init") { + let do_debug = if false + && crate::context::current() + .read(token.token()) + .name + .contains("init") + { if a == SYS_CLOCK_GETTIME || a == SYS_YIELD || a == SYS_FUTEX { false } else if (a == SYS_WRITE || a == SYS_FSYNC) && (b == 1 || b == 2) { @@ -237,7 +242,7 @@ pub fn debug_start([a, b, c, d, e, f]: [usize; 6]) { let debug_start = if do_debug { let context_lock = crate::context::current(); { - let context = context_lock.read(); + let context = context_lock.read(token.token()); print!("{} (*{}*): ", context.name, context.pid,); } @@ -261,7 +266,11 @@ pub fn debug_start([a, b, c, d, e, f]: [usize; 6]) { } #[cfg_attr(feature = "syscall_debug", inline)] -pub fn debug_end([a, b, c, d, e, f]: [usize; 6], result: Result) { +pub fn debug_end( + [a, b, c, d, e, f]: [usize; 6], + result: Result, + token: &mut CleanLockToken, +) { if cfg!(not(feature = "syscall_debug")) { return; } @@ -278,7 +287,7 @@ pub fn debug_end([a, b, c, d, e, f]: [usize; 6], result: Result) { let context_lock = crate::context::current(); { - let context = context_lock.read(); + let context = context_lock.read(token.token()); print!("{} (*{}*): ", context.name, context.pid,); } diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index 57b6f9b3ce..0842a292c8 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -13,6 +13,7 @@ use crate::{ }, paging::{Page, VirtualAddress, PAGE_SIZE}, scheme::{self, CallerCtx, FileHandle, KernelScheme, OpenResult, StrOrBytes}, + sync::CleanLockToken, syscall::{data::Stat, error::*, flag::*}, }; @@ -20,29 +21,36 @@ use super::usercopy::{UserSlice, UserSliceRo, UserSliceRw, UserSliceWo}; pub fn file_op_generic( fd: FileHandle, - op: impl FnOnce(&dyn KernelScheme, usize) -> Result, + token: &mut CleanLockToken, + op: impl FnOnce(&dyn KernelScheme, usize, &mut CleanLockToken) -> Result, ) -> Result { - file_op_generic_ext(fd, |s, _, desc| op(s, desc.number)) + file_op_generic_ext(fd, token, |s, _, desc, token| op(s, desc.number, token)) } pub fn file_op_generic_ext( fd: FileHandle, - op: impl FnOnce(&dyn KernelScheme, Arc>, FileDescription) -> Result, + token: &mut CleanLockToken, + op: impl FnOnce( + &dyn KernelScheme, + Arc>, + FileDescription, + &mut CleanLockToken, + ) -> Result, ) -> Result { let (file, desc) = { let file = context::current() - .read() + .read(token.token()) .get_file(fd) .ok_or(Error::new(EBADF))?; let desc = *file.description.read(); (file, desc) }; - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(desc.scheme) .ok_or(Error::new(EBADF))? .clone(); - op(&*scheme, file.description, desc) + op(&*scheme, file.description, desc, token) } pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result { let mut path_buf = vec![0_u8; max_len]; @@ -67,8 +75,8 @@ fn is_legacy(path_buf: &String) -> bool { } /// Open syscall -pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { - let (pid, uid, gid, scheme_ns) = match context::current().read() { +pub fn open(raw_path: UserSliceRo, flags: usize, token: &mut CleanLockToken) -> Result { + let (pid, uid, gid, scheme_ns) = match context::current().read(token.token()) { ref cx => (cx.pid.into(), cx.euid, cx.egid, cx.ens), }; @@ -83,7 +91,7 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { // Display a deprecation warning for any usage of the legacy scheme syntax (scheme:/path) // FIXME remove entries from this list as the respective programs get updated if path_buf.contains(':') && !is_legacy(&path_buf) { - let name = context::current().read().name.clone(); + let name = context::current().read(token.token()).name.clone(); if name.contains("cosmic") && (path_buf == "event:" || path_buf.starts_with("time:")) { // FIXME cosmic apps likely need crate updates } else { @@ -95,14 +103,19 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { let description = { let (scheme_id, scheme) = { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); let (scheme_id, scheme) = schemes .get_name(scheme_ns, scheme_name.as_ref()) .ok_or(Error::new(ENODEV))?; (scheme_id, scheme.clone()) }; - match scheme.kopen(reference.as_ref(), flags, CallerCtx { uid, gid, pid })? { + match scheme.kopen( + reference.as_ref(), + flags, + CallerCtx { uid, gid, pid }, + token, + )? { OpenResult::SchemeLocal(number, internal_flags) => { Arc::new(RwLock::new(FileDescription { scheme: scheme_id, @@ -117,7 +130,7 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { }; //drop(path_buf); context::current() - .read() + .read(token.token()) .add_file(FileDescriptor { description, cloexec: flags & O_CLOEXEC == O_CLOEXEC, @@ -130,6 +143,7 @@ pub fn openat( raw_path: UserSliceRo, flags: usize, fcntl_flags: u32, + token: &mut CleanLockToken, ) -> Result { let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?; @@ -139,16 +153,16 @@ pub fn openat( } let pipe = context::current() - .read() + .read(token.token()) .get_file(fh) .ok_or(Error::new(EBADF))?; let description = pipe.description.read(); - let caller_ctx = context::current().read().caller_ctx(); + let caller_ctx = context::current().read(token.token()).caller_ctx(); let new_description = { - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(description.scheme) .ok_or(Error::new(EBADF))? .clone(); @@ -159,6 +173,7 @@ pub fn openat( flags, fcntl_flags, caller_ctx, + token, ); match res? { @@ -176,7 +191,7 @@ pub fn openat( }; context::current() - .read() + .read(token.token()) .add_file(FileDescriptor { description: new_description, cloexec: false, @@ -184,8 +199,8 @@ pub fn openat( .ok_or(Error::new(EMFILE)) } /// rmdir syscall -pub fn rmdir(raw_path: UserSliceRo) -> Result<()> { - let (scheme_ns, caller_ctx) = match context::current().read() { +pub fn rmdir(raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> { + let (scheme_ns, caller_ctx) = match context::current().read(token.token()) { ref cx => (cx.ens, cx.caller_ctx()), }; @@ -198,18 +213,18 @@ pub fn rmdir(raw_path: UserSliceRo) -> Result<()> { let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?; let scheme = { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); let (_scheme_id, scheme) = schemes .get_name(scheme_ns, scheme_name.as_ref()) .ok_or(Error::new(ENODEV))?; scheme.clone() }; - scheme.rmdir(reference.as_ref(), caller_ctx) + scheme.rmdir(reference.as_ref(), caller_ctx, token) } /// Unlink syscall -pub fn unlink(raw_path: UserSliceRo) -> Result<()> { - let (scheme_ns, caller_ctx) = match context::current().read() { +pub fn unlink(raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> { + let (scheme_ns, caller_ctx) = match context::current().read(token.token()) { ref cx => (cx.ens, cx.caller_ctx()), }; /* @@ -221,30 +236,34 @@ pub fn unlink(raw_path: UserSliceRo) -> Result<()> { let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?; let scheme = { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); let (_scheme_id, scheme) = schemes .get_name(scheme_ns, scheme_name.as_ref()) .ok_or(Error::new(ENODEV))?; scheme.clone() }; - scheme.unlink(reference.as_ref(), caller_ctx) + scheme.unlink(reference.as_ref(), caller_ctx, token) } /// Close syscall -pub fn close(fd: FileHandle) -> Result<()> { +pub fn close(fd: FileHandle, token: &mut CleanLockToken) -> Result<()> { let file = { let context_lock = context::current(); - let context = context_lock.read(); + let context = context_lock.read(token.token()); context.remove_file(fd).ok_or(Error::new(EBADF))? }; - file.close() + file.close(token) } -fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result { +fn duplicate_file( + fd: FileHandle, + user_buf: UserSliceRo, + token: &mut CleanLockToken, +) -> Result { let (caller_ctx, file) = { let context_lock = context::current(); - let context = context_lock.read(); + let context = context_lock.read(token.token()); ( context.caller_ctx(), context.get_file(fd).ok_or(Error::new(EBADF))?, @@ -260,12 +279,12 @@ fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result { Arc::new(RwLock::new(FileDescription { offset: 0, @@ -287,25 +306,30 @@ fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result Result { - let new_file = duplicate_file(fd, buf)?; +pub fn dup(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + let new_file = duplicate_file(fd, buf, token)?; context::current() - .read() + .read(token.token()) .add_file(new_file) .ok_or(Error::new(EMFILE)) } /// Duplicate file descriptor, replacing another -pub fn dup2(fd: FileHandle, new_fd: FileHandle, buf: UserSliceRo) -> Result { +pub fn dup2( + fd: FileHandle, + new_fd: FileHandle, + buf: UserSliceRo, + token: &mut CleanLockToken, +) -> Result { if fd == new_fd { Ok(new_fd) } else { - let _ = close(new_fd); - let new_file = duplicate_file(fd, buf)?; + let _ = close(new_fd, token); + let new_file = duplicate_file(fd, buf, token)?; let context_ref = context::current(); - let context = context_ref.read(); + let context = context_ref.read(token.token()); context .insert_file(new_fd, new_file) @@ -317,6 +341,7 @@ pub fn call( payload: UserSliceRw, flags: CallFlags, metadata: UserSliceRo, + token: &mut CleanLockToken, ) -> Result { let mut meta = [0_u64; 3]; @@ -327,12 +352,12 @@ pub fn call( match flags { f if f.contains(CallFlags::WRITE | CallFlags::FD) => { - call_fdwrite(fd, payload, flags, &meta[..copied / 8]) + call_fdwrite(fd, payload, flags, &meta[..copied / 8], token) } f if f.contains(CallFlags::READ | CallFlags::FD) => { - call_fdread(fd, payload, flags, &meta[..copied / 8]) + call_fdread(fd, payload, flags, &meta[..copied / 8], token) } - _ => call_normal(fd, payload, flags, &meta[..copied / 8]), + _ => call_normal(fd, payload, flags, &meta[..copied / 8], token), } } @@ -341,9 +366,10 @@ fn call_normal( payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let file = (match ( - context::current().read(), + context::current().read(token.token()), flags.contains(CallFlags::CONSUME), ) { (ctxt, true) => ctxt.remove_file(fd), @@ -355,12 +381,12 @@ fn call_normal( let desc = file.description.read(); (desc.scheme, desc.number) }; - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(scheme_id) .ok_or(Error::new(EBADFD))? .clone(); - scheme.kcall(number, payload, flags, metadata) + scheme.kcall(number, payload, flags, metadata, token) } fn call_fdwrite( @@ -368,6 +394,7 @@ fn call_fdwrite( payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let payload_chunks = payload.in_exact_chunks(size_of::()); let fds = payload_chunks @@ -379,7 +406,7 @@ fn call_fdwrite( let len = fds.len(); - fdwrite_inner(fd, fds, flags, 0, metadata)?; + fdwrite_inner(fd, fds, flags, 0, metadata, token)?; Ok(len) } @@ -390,25 +417,29 @@ fn fdwrite_inner( flags: CallFlags, arg: u64, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { // TODO: Ensure deadlocks can't happen let (scheme, number, descs_to_send) = { - let current_lock = context::current(); - let current = current_lock.read(); - - let (scheme, number) = match current - .get_file(socket) - .ok_or(Error::new(EBADF))? - .description - .read() - { - ref desc => (desc.scheme, desc.number), + let (scheme, number) = { + let current_lock = context::current(); + let current = current_lock.read(token.token()); + match current + .get_file(socket) + .ok_or(Error::new(EBADF))? + .description + .read() + { + ref desc => (desc.scheme, desc.number), + } }; - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(scheme) .ok_or(Error::new(ENODEV))? .clone(); + let current_lock = context::current(); + let current = current_lock.read(token.token()); ( scheme, number, @@ -438,7 +469,7 @@ fn fdwrite_inner( CallFlags::empty() }; - scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata) + scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata, token) } fn call_fdread( @@ -446,20 +477,22 @@ fn call_fdread( payload: UserSliceRw, flags: CallFlags, metadata: &[u64], + token: &mut CleanLockToken, ) -> Result { let (scheme, number) = { - let current_lock = context::current(); - let current = current_lock.read(); - - let (scheme, number) = match current - .get_file(fd) - .ok_or(Error::new(EBADF))? - .description - .read() - { - ref desc => (desc.scheme, desc.number), + let (scheme, number) = { + let current_lock = context::current(); + let current = current_lock.read(token.token()); + match current + .get_file(fd) + .ok_or(Error::new(EBADF))? + .description + .read() + { + ref desc => (desc.scheme, desc.number), + } }; - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(scheme) .ok_or(Error::new(ENODEV))? .clone(); @@ -467,10 +500,16 @@ fn call_fdread( (scheme, number) }; - scheme.kfdread(number, payload, flags, metadata) + scheme.kfdread(number, payload, flags, metadata, token) } -pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> Result { +pub fn sendfd( + socket: FileHandle, + fd: FileHandle, + flags_raw: usize, + arg: u64, + token: &mut CleanLockToken, +) -> Result { let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?; let mut call_flags = CallFlags::FD | CallFlags::WRITE; if sendfd_flags.contains(SendFdFlags::CLONE) { @@ -479,13 +518,13 @@ pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) { call_flags |= CallFlags::FD_EXCLUSIVE; } - fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[]) + fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[], token) } /// File descriptor controls -pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { +pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) -> Result { let file = context::current() - .read() + .read(token.token()) .get_file(fd) .ok_or(Error::new(EBADF))?; @@ -493,10 +532,10 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { if cmd == F_DUPFD { // Not in match because 'files' cannot be locked - let new_file = duplicate_file(fd, UserSlice::empty())?; + let new_file = duplicate_file(fd, UserSlice::empty(), token)?; let context_lock = context::current(); - let context = context_lock.read(); + let context = context_lock.read(token.token()); return context .add_file_min(new_file, arg) @@ -506,18 +545,18 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { // Communicate fcntl with scheme if cmd != F_GETFD && cmd != F_SETFD { - let scheme = scheme::schemes() + let scheme = scheme::schemes(token.token()) .get(description.scheme) .ok_or(Error::new(EBADF))? .clone(); - scheme.fcntl(description.number, cmd, arg)?; + scheme.fcntl(description.number, cmd, arg, token)?; }; // Perform kernel operation if scheme agrees { let context_lock = context::current(); - let context = context_lock.read(); + let context = context_lock.read(token.token()); let mut files = context.files.write(); match *files.get_mut(fd.get()).ok_or(Error::new(EBADF))? { @@ -548,12 +587,12 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { } } -pub fn flink(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { - let (caller_ctx, scheme_ns) = match context::current().read() { +pub fn flink(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> { + let (caller_ctx, scheme_ns) = match context::current().read(token.token()) { ref cx => (cx.caller_ctx(), cx.ens), }; let file = context::current() - .read() + .read(token.token()) .get_file(fd) .ok_or(Error::new(EBADF))?; @@ -566,7 +605,7 @@ pub fn flink(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?; let (scheme_id, scheme) = { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); let (scheme_id, scheme) = schemes .get_name(scheme_ns, scheme_name.as_ref()) .ok_or(Error::new(ENODEV))?; @@ -579,15 +618,15 @@ pub fn flink(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { return Err(Error::new(EXDEV)); } - scheme.flink(description.number, reference.as_ref(), caller_ctx) + scheme.flink(description.number, reference.as_ref(), caller_ctx, token) } -pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { - let (caller_ctx, scheme_ns) = match context::current().read() { +pub fn frename(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> { + let (caller_ctx, scheme_ns) = match context::current().read(token.token()) { ref cx => (cx.caller_ctx(), cx.ens), }; let file = context::current() - .read() + .read(token.token()) .get_file(fd) .ok_or(Error::new(EBADF))?; @@ -600,7 +639,7 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?; let (scheme_id, scheme) = { - let schemes = scheme::schemes(); + let schemes = scheme::schemes(token.token()); let (scheme_id, scheme) = schemes .get_name(scheme_ns, scheme_name.as_ref()) .ok_or(Error::new(ENODEV))?; @@ -613,13 +652,13 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { return Err(Error::new(EXDEV)); } - scheme.frename(description.number, reference.as_ref(), caller_ctx) + scheme.frename(description.number, reference.as_ref(), caller_ctx, token) } /// File status -pub fn fstat(fd: FileHandle, user_buf: UserSliceWo) -> Result<()> { - file_op_generic_ext(fd, |scheme, _, desc| { - scheme.kfstat(desc.number, user_buf)?; +pub fn fstat(fd: FileHandle, user_buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> { + file_op_generic_ext(fd, token, |scheme, _, desc, token| { + scheme.kfstat(desc.number, user_buf, token)?; // TODO: Ensure only the kernel can access the stat when st_dev is set, or use another API // for retrieving the scheme ID from a file descriptor. @@ -639,7 +678,7 @@ pub fn fstat(fd: FileHandle, user_buf: UserSliceWo) -> Result<()> { }) } -pub fn funmap(virtual_address: usize, length: usize) -> Result { +pub fn funmap(virtual_address: usize, length: usize, token: &mut CleanLockToken) -> Result { // Partial lengths in funmap are allowed according to POSIX, but not particularly meaningful; // since the memory needs to SIGSEGV if later read, the entire page needs to disappear. // @@ -654,14 +693,14 @@ pub fn funmap(virtual_address: usize, length: usize) -> Result { ); } - let addr_space = Arc::clone(context::current().read().addr_space()?); + let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?); let span = PageSpan::validate_nonempty(VirtualAddress::new(virtual_address), length_aligned) .ok_or(Error::new(EINVAL))?; let unpin = false; let notify = addr_space.munmap(span, unpin)?; for map in notify { - let _ = map.unmap(); + let _ = map.unmap(token); } Ok(0) @@ -673,6 +712,7 @@ pub fn mremap( new_address: usize, new_size: usize, flags: usize, + token: &mut CleanLockToken, ) -> Result { if old_address % PAGE_SIZE != 0 || old_size % PAGE_SIZE != 0 @@ -712,7 +752,7 @@ pub fn mremap( return Err(Error::new(EOPNOTSUPP)); } - let raii_frame = addr_space.borrow_frame_enforce_rw_allocated(src_span.base)?; + let raii_frame = addr_space.borrow_frame_enforce_rw_allocated(src_span.base, token)?; let base = addr_space.acquire_write().mmap( &addr_space, @@ -752,17 +792,17 @@ pub fn mremap( } } -pub fn lseek(fd: FileHandle, pos: i64, whence: usize) -> Result { +pub fn lseek(fd: FileHandle, pos: i64, whence: usize, token: &mut CleanLockToken) -> Result { enum Ret { Legacy(usize), Fsize((Option, Arc>)), } - let fsize_or_legacy = file_op_generic_ext(fd, |scheme, desc_arc, desc| { + let fsize_or_legacy = file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| { Ok( - if let Some(new_off) = scheme.legacy_seek(desc.number, pos as isize, whence) { + if let Some(new_off) = scheme.legacy_seek(desc.number, pos as isize, whence, token) { Ret::Legacy(new_off?) } else if whence == SEEK_END { - Ret::Fsize((Some(scheme.fsize(desc.number)?), desc_arc)) + Ret::Fsize((Some(scheme.fsize(desc.number, token)?), desc_arc)) } else { Ret::Fsize((None, desc_arc)) }, @@ -789,19 +829,20 @@ pub fn lseek(fd: FileHandle, pos: i64, whence: usize) -> Result { Ok(guard.offset as usize) } -pub fn sys_read(fd: FileHandle, buf: UserSliceWo) -> Result { - let (bytes_read, desc_arc, desc) = file_op_generic_ext(fd, |scheme, desc_arc, desc| { - let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) { - desc.offset - } else { - u64::MAX - }; - Ok(( - scheme.kreadoff(desc.number, buf, offset, desc.flags, desc.flags)?, - desc_arc, - desc, - )) - })?; +pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) -> Result { + let (bytes_read, desc_arc, desc) = + file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| { + let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) { + desc.offset + } else { + u64::MAX + }; + Ok(( + scheme.kreadoff(desc.number, buf, offset, desc.flags, desc.flags, token)?, + desc_arc, + desc, + )) + })?; if desc.internal_flags.contains(InternalFlags::POSITIONED) { match desc_arc.write().offset { ref mut offset => *offset = offset.saturating_add(bytes_read as u64), @@ -809,19 +850,20 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo) -> Result { } Ok(bytes_read) } -pub fn sys_write(fd: FileHandle, buf: UserSliceRo) -> Result { - let (bytes_written, desc_arc, desc) = file_op_generic_ext(fd, |scheme, desc_arc, desc| { - let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) { - desc.offset - } else { - u64::MAX - }; - Ok(( - scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags)?, - desc_arc, - desc, - )) - })?; +pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + let (bytes_written, desc_arc, desc) = + file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| { + let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) { + desc.offset + } else { + u64::MAX + }; + Ok(( + scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags, token)?, + desc_arc, + desc, + )) + })?; if desc.internal_flags.contains(InternalFlags::POSITIONED) { match desc_arc.write().offset { ref mut offset => *offset = offset.saturating_add(bytes_written as u64), diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index 2d053d4628..860e2bcc2f 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -8,18 +8,17 @@ use alloc::{ }; use core::sync::atomic::{AtomicU32, Ordering}; use rmm::Arch; -use spin::RwLock; -use spinning_top::RwSpinlock; use syscall::EINTR; use crate::{ context::{ self, memory::{AddrSpace, AddrSpaceWrapper}, - Context, + ContextLock, }, memory::PhysicalAddress, paging::{Page, VirtualAddress}, + sync::{CleanLockToken, Mutex, L1}, time, }; @@ -42,7 +41,7 @@ pub struct FutexEntry { // TODO: FUTEX_REQUEUE target_virtaddr: VirtualAddress, // Context to wake up, and compare address spaces. - context_lock: Arc>, + context_lock: Arc, // address space to check against if virt matches but not phys addr_space: Weak, } @@ -53,7 +52,7 @@ pub struct FutexEntry { // lwp_park/lwp_unpark from NetBSD) could be a simpler replacement. // // TODO: Use an actual hash table. -static FUTEXES: RwLock = RwLock::new(FutexList::new()); +static FUTEXES: Mutex = Mutex::new(FutexList::new()); fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option { // TODO: Move this elsewhere! @@ -69,7 +68,14 @@ fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Optio Some(frame.add(off)) } -pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> Result { +pub fn futex( + addr: usize, + op: usize, + val: usize, + val2: usize, + _addr2: usize, + token: &mut CleanLockToken, +) -> Result { let current_addrsp = AddrSpace::current()?; // Keep the address space locked so we can safely read from the physical address. Unlock it @@ -89,7 +95,8 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> .transpose()?; { - let mut futexes = FUTEXES.write(); + let mut futexes = FUTEXES.lock(token.token()); + let (futexes, mut token) = futexes.token_split(); let context_lock = context::current(); @@ -137,7 +144,7 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> } { - let mut context = context_lock.write(); + let mut context = context_lock.write(token.token()); context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| { tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128 @@ -161,10 +168,10 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> drop(addr_space_guard); - context::switch(); + context::switch(token); if timeout_opt.is_some() { - context::current().write().wake = None; + context::current().write(token.token()).wake = None; Err(Error::new(ETIMEDOUT)) } else { Ok(0) @@ -174,7 +181,8 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> let mut woken = 0; { - let mut futexes = FUTEXES.write(); + let mut futexes = FUTEXES.lock(token.token()); + let (futexes, mut token) = futexes.token_split(); let mut i = 0; @@ -187,7 +195,7 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> i += 1; continue; } - futexes[i].context_lock.write().unblock(); + futexes[i].context_lock.write(token.token()).unblock(); futexes.swap_remove_back(i); woken += 1; } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index ba86bfde93..49c47912f7 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -23,11 +23,11 @@ use self::{ usercopy::UserSlice, }; -use crate::percpu::PercpuBlock; - use crate::{ context::memory::AddrSpace, + percpu::PercpuBlock, scheme::{memory::MemoryScheme, FileHandle}, + sync::CleanLockToken, }; /// Debug @@ -54,13 +54,29 @@ pub mod usercopy; /// This function is the syscall handler of the kernel, it is composed of an inner function that returns a `Result`. After the inner function runs, the syscall /// function calls [`Error::mux`] on it. #[must_use] -pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> usize { +pub fn syscall( + a: usize, + b: usize, + c: usize, + d: usize, + e: usize, + f: usize, + token: &mut CleanLockToken, +) -> usize { #[inline(always)] - fn inner(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> Result { + fn inner( + a: usize, + b: usize, + c: usize, + d: usize, + e: usize, + f: usize, + token: &mut CleanLockToken, + ) -> Result { let fd = FileHandle::from(b); //SYS_* is declared in kernel/syscall/src/number.rs match a { - SYS_WRITE2 => file_op_generic_ext(fd, |scheme, _, desc| { + SYS_WRITE2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| { let flags = if f == usize::MAX { None } else { @@ -77,17 +93,18 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us e as u64, flags.map_or(desc.flags, |f| desc.rw_flags(f)), desc.flags, + token, ) }), - SYS_WRITE => sys_write(fd, UserSlice::ro(c, d)?), + SYS_WRITE => sys_write(fd, UserSlice::ro(c, d)?, token), SYS_FMAP => { let addrspace = AddrSpace::current()?; let map = unsafe { UserSlice::ro(c, d)?.read_exact::()? }; if b == !0 { - MemoryScheme::fmap_anonymous(&addrspace, &map, false) + MemoryScheme::fmap_anonymous(&addrspace, &map, false, token) } else { - file_op_generic(fd, |scheme, number| { - scheme.kfmap(number, &addrspace, &map, false) + file_op_generic(fd, token, |scheme, number, token| { + scheme.kfmap(number, &addrspace, &map, false, token) }) } } @@ -99,15 +116,15 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us return Err(Error::new(EINVAL)); } - file_op_generic(fd, |scheme, number| { - scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64) + file_op_generic(fd, token, |scheme, number, token| { + scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64, token) }) } - SYS_FUTIMENS => file_op_generic(fd, |scheme, number| { - scheme.kfutimens(number, UserSlice::ro(c, d)?) + SYS_FUTIMENS => file_op_generic(fd, token, |scheme, number, token| { + scheme.kfutimens(number, UserSlice::ro(c, d)?, token) }), - SYS_READ2 => file_op_generic_ext(fd, |scheme, _, desc| { + SYS_READ2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| { let flags = if f == usize::MAX { None } else { @@ -124,79 +141,97 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us e as u64, flags.map_or(desc.flags, |f| desc.rw_flags(f)), desc.flags, + token, ) }), - SYS_READ => sys_read(fd, UserSlice::wo(c, d)?), - SYS_FPATH => file_op_generic(fd, |scheme, number| { - scheme.kfpath(number, UserSlice::wo(c, d)?) + SYS_READ => sys_read(fd, UserSlice::wo(c, d)?, token), + SYS_FPATH => file_op_generic(fd, token, |scheme, number, token| { + scheme.kfpath(number, UserSlice::wo(c, d)?, token) }), - SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?).map(|()| 0), - SYS_FSTATVFS => file_op_generic(fd, |scheme, number| { - scheme.kfstatvfs(number, UserSlice::wo(c, d)?).map(|()| 0) + SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?, token).map(|()| 0), + SYS_FSTATVFS => file_op_generic(fd, token, |scheme, number, token| { + scheme + .kfstatvfs(number, UserSlice::wo(c, d)?, token) + .map(|()| 0) }), - SYS_DUP => dup(fd, UserSlice::ro(c, d)?).map(FileHandle::into), - SYS_DUP2 => dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?).map(FileHandle::into), - - #[cfg(target_pointer_width = "32")] - SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64 | ((f as u64) << 32)), - - #[cfg(target_pointer_width = "64")] - SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64), - - SYS_LSEEK => lseek(fd, c as i64, d), - SYS_FCHMOD => file_op_generic(fd, |scheme, number| { - scheme.fchmod(number, c as u16).map(|()| 0) - }), - SYS_FCHOWN => file_op_generic(fd, |scheme, number| { - scheme.fchown(number, c as u32, d as u32).map(|()| 0) - }), - SYS_FCNTL => fcntl(fd, c, d), - SYS_FEVENT => file_op_generic(fd, |scheme, number| { - Ok(scheme - .fevent(number, EventFlags::from_bits_truncate(c))? - .bits()) - }), - SYS_FLINK => flink(fd, UserSlice::ro(c, d)?).map(|()| 0), - SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?).map(|()| 0), - SYS_FUNMAP => funmap(b, c), - - SYS_FSYNC => file_op_generic(fd, |scheme, number| scheme.fsync(number).map(|()| 0)), - // TODO: 64-bit lengths on 32-bit platforms - SYS_FTRUNCATE => { - file_op_generic(fd, |scheme, number| scheme.ftruncate(number, c).map(|()| 0)) + SYS_DUP => dup(fd, UserSlice::ro(c, d)?, token).map(FileHandle::into), + SYS_DUP2 => { + dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?, token).map(FileHandle::into) } - SYS_CLOSE => close(fd).map(|()| 0), + #[cfg(target_pointer_width = "32")] + SYS_SENDFD => sendfd( + fd, + FileHandle::from(c), + d, + e as u64 | ((f as u64) << 32), + token, + ), + + #[cfg(target_pointer_width = "64")] + SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64, token), + + SYS_LSEEK => lseek(fd, c as i64, d, token), + SYS_FCHMOD => file_op_generic(fd, token, |scheme, number, token| { + scheme.fchmod(number, c as u16, token).map(|()| 0) + }), + SYS_FCHOWN => file_op_generic(fd, token, |scheme, number, token| { + scheme.fchown(number, c as u32, d as u32, token).map(|()| 0) + }), + SYS_FCNTL => fcntl(fd, c, d, token), + SYS_FEVENT => file_op_generic(fd, token, |scheme, number, token| { + Ok(scheme + .fevent(number, EventFlags::from_bits_truncate(c), token)? + .bits()) + }), + SYS_FLINK => flink(fd, UserSlice::ro(c, d)?, token).map(|()| 0), + SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?, token).map(|()| 0), + SYS_FUNMAP => funmap(b, c, token), + + SYS_FSYNC => file_op_generic(fd, token, |scheme, number, token| { + scheme.fsync(number, token).map(|()| 0) + }), + // TODO: 64-bit lengths on 32-bit platforms + SYS_FTRUNCATE => file_op_generic(fd, token, |scheme, number, token| { + scheme.ftruncate(number, c, token).map(|()| 0) + }), + + SYS_CLOSE => close(fd, token).map(|()| 0), SYS_CALL => call( fd, UserSlice::rw(c, d)?, CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?, UserSlice::ro(f, (e & 0xff) * 8)?, + token, ), - SYS_OPEN => open(UserSlice::ro(b, c)?, d).map(FileHandle::into), - SYS_OPENAT => openat(fd, UserSlice::ro(c, d)?, e, f as _).map(FileHandle::into), - SYS_RMDIR => rmdir(UserSlice::ro(b, c)?).map(|()| 0), - SYS_UNLINK => unlink(UserSlice::ro(b, c)?).map(|()| 0), - SYS_YIELD => sched_yield().map(|()| 0), + SYS_OPEN => open(UserSlice::ro(b, c)?, d, token).map(FileHandle::into), + SYS_OPENAT => openat(fd, UserSlice::ro(c, d)?, e, f as _, token).map(FileHandle::into), + SYS_RMDIR => rmdir(UserSlice::ro(b, c)?, token).map(|()| 0), + SYS_UNLINK => unlink(UserSlice::ro(b, c)?, token).map(|()| 0), + SYS_YIELD => sched_yield(token).map(|()| 0), SYS_NANOSLEEP => nanosleep( UserSlice::ro(b, core::mem::size_of::())?, UserSlice::wo(c, core::mem::size_of::())?.none_if_null(), + token, ) .map(|()| 0), SYS_CLOCK_GETTIME => { clock_gettime(b, UserSlice::wo(c, core::mem::size_of::())?).map(|()| 0) } - SYS_FUTEX => futex(b, c, d, e, f), + SYS_FUTEX => futex(b, c, d, e, f, token), SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)).map(|()| 0), - SYS_MKNS => mkns(UserSlice::ro( - b, - c.checked_mul(core::mem::size_of::<[usize; 2]>()) - .ok_or(Error::new(EOVERFLOW))?, - )?), - SYS_MREMAP => mremap(b, c, d, e, f), + SYS_MKNS => mkns( + UserSlice::ro( + b, + c.checked_mul(core::mem::size_of::<[usize; 2]>()) + .ok_or(Error::new(EOVERFLOW))?, + )?, + token, + ), + SYS_MREMAP => mremap(b, c, d, e, f, token), _ => return Err(Error::new(ENOSYS)), } @@ -204,17 +239,17 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us PercpuBlock::current().inside_syscall.set(true); - debug_start([a, b, c, d, e, f]); + debug_start([a, b, c, d, e, f], token); - let result = inner(a, b, c, d, e, f); + let result = inner(a, b, c, d, e, f, token); - debug_end([a, b, c, d, e, f], result); + debug_end([a, b, c, d, e, f], result, token); let percpu = PercpuBlock::current(); percpu.inside_syscall.set(false); if percpu.switch_internals.being_sigkilled.get() { - exit_this_context(None); + exit_this_context(None, token); } // errormux turns Result into -errno diff --git a/src/syscall/privilege.rs b/src/syscall/privilege.rs index 7b8fed4bf0..20bb340dcd 100644 --- a/src/syscall/privilege.rs +++ b/src/syscall/privilege.rs @@ -1,14 +1,14 @@ use alloc::vec::Vec; -use crate::{context, scheme, syscall::error::*}; +use crate::{context, scheme, sync::CleanLockToken, syscall::error::*}; use super::{ copy_path_to_buf, usercopy::{UserSlice, UserSliceRo}, }; -pub fn mkns(mut user_buf: UserSliceRo) -> Result { - let (uid, from) = match context::current().read() { +pub fn mkns(mut user_buf: UserSliceRo, token: &mut CleanLockToken) -> Result { + let (uid, from) = match context::current().read(token.token()) { ref cx => (cx.euid, cx.ens), }; @@ -36,6 +36,6 @@ pub fn mkns(mut user_buf: UserSliceRo) -> Result { user_buf = next_part; } - let to = scheme::schemes_mut().make_ns(from, names)?; + let to = scheme::schemes_mut(token.token()).make_ns(from, names)?; Ok(to.into()) } diff --git a/src/syscall/process.rs b/src/syscall/process.rs index ca4f3ea54f..bbcec2e33e 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -11,6 +11,7 @@ use crate::{ }, event, scheme::GlobalSchemes, + sync::CleanLockToken, syscall::EventFlags, }; @@ -24,13 +25,13 @@ use crate::{ use super::usercopy::UserSliceWo; -pub fn exit_this_context(excp: Option) -> ! { +pub fn exit_this_context(excp: Option, token: &mut CleanLockToken) -> ! { let mut close_files; let addrspace_opt; let context_lock = context::current(); { - let mut context = context_lock.write(); + let mut context = context_lock.write(token.token()); close_files = Arc::try_unwrap(mem::take(&mut context.files)) .map_or_else(|_| FdTbl::new(), RwLock::into_inner); addrspace_opt = context @@ -41,11 +42,11 @@ pub fn exit_this_context(excp: Option) -> ! { } // Files must be closed while context is valid so that messages can be passed - close_files.force_close_all(); + close_files.force_close_all(token); drop(addrspace_opt); // TODO: Should status == Status::HardBlocked be handled differently? let owner = { - let mut guard = context_lock.write(); + let mut guard = context_lock.write(token.token()); guard.status = context::Status::Dead { excp }; guard.owner_proc_id }; @@ -57,9 +58,9 @@ pub fn exit_this_context(excp: Option) -> ! { ); } { - let _ = context::contexts_mut().remove(&ContextRef(context_lock)); + let _ = context::contexts_mut(token.token()).remove(&ContextRef(context_lock)); } - context::switch(); + context::switch(token); unreachable!(); } @@ -72,13 +73,13 @@ pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result<()> { AddrSpace::current()?.mprotect(span, flags) } -pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) { +pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockToken) { assert_ne!(bootstrap.page_count, 0); { let addr_space = Arc::clone( context::current() - .read() + .read(token.token()) .addr_space() .expect("expected bootstrap context to have an address space"), ); @@ -128,7 +129,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) { // Start in a minimal environment without any stack. match context::current() - .write() + .write(token.token()) .regs_mut() .expect("bootstrap needs registers to be available") { diff --git a/src/syscall/time.rs b/src/syscall/time.rs index ddf6b2385a..27e0e20993 100644 --- a/src/syscall/time.rs +++ b/src/syscall/time.rs @@ -1,5 +1,6 @@ use crate::{ context, + sync::CleanLockToken, syscall::{ data::TimeSpec, error::*, @@ -24,7 +25,11 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo) -> Result<()> { } /// Nanosleep will sleep by switching the current context -pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option) -> Result<()> { +pub fn nanosleep( + req_buf: UserSliceRo, + rem_buf_opt: Option, + token: &mut CleanLockToken, +) -> Result<()> { let req = unsafe { req_buf.read_exact::()? }; let start = time::monotonic(); @@ -32,7 +37,7 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option) -> Resu let current_context = context::current(); { - let mut context = current_context.write(); + let mut context = current_context.write(token.token()); if let Some((tctl, pctl, _)) = context.sigcontrol() { if tctl.currently_pending_unblocked(pctl) != 0 { @@ -46,9 +51,9 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option) -> Resu // TODO: The previous wakeup reason was most likely signals, but is there any other possible // reason? - context::switch(); + context::switch(token); - let was_interrupted = current_context.write().wake.take().is_some(); + let was_interrupted = current_context.write(token.token()).wake.take().is_some(); if let Some(rem_buf) = rem_buf_opt { let current = time::monotonic(); @@ -74,9 +79,9 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option) -> Resu } } -pub fn sched_yield() -> Result<()> { - context::switch(); +pub fn sched_yield(token: &mut CleanLockToken) -> Result<()> { + context::switch(token); // TODO: Do this check in userspace - context::signal::signal_handler(); + context::signal::signal_handler(token); Ok(()) } diff --git a/src/time.rs b/src/time.rs index 2de43b4183..c041dfa149 100644 --- a/src/time.rs +++ b/src/time.rs @@ -1,6 +1,9 @@ use spin::Mutex; -use crate::syscall::error::{Error, Result, EINVAL}; +use crate::{ + sync::CleanLockToken, + syscall::error::{Error, Result, EINVAL}, +}; pub const NANOS_PER_SEC: u128 = 1_000_000_000; @@ -18,7 +21,7 @@ pub fn realtime() -> u128 { *START.lock() + monotonic() } -pub fn sys_update_time_offset(buf: &[u8]) -> Result { +pub fn sys_update_time_offset(buf: &[u8], _token: &mut CleanLockToken) -> Result { let start = <[u8; 16]>::try_from(buf).map_err(|_| Error::new(EINVAL))?; *START.lock() = u128::from_ne_bytes(start); Ok(16)