diff --git a/.gitignore b/.gitignore index 7774614018..c72df02094 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ target +/build /config.toml +/sysroot .gitlab-ci-local/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d82652f70..a8d38d1381 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,3 @@ -image: "redoxos/redoxer:latest" - -variables: - GIT_SUBMODULE_STRATEGY: recursive - workflow: rules: - if: '$CI_PROJECT_NAMESPACE == "redox-os"' @@ -12,14 +7,12 @@ stages: - build - cross-build - test - - other-features # TODO: benchmarks and profiling (maybe manually enabled for relevant MRs)? x86_64: stage: build script: - - mkdir -p target/${ARCH} - - redoxer env make BUILD=target/${ARCH} + - redoxer env make variables: ARCH: "x86_64" @@ -27,64 +20,50 @@ aarch64: stage: cross-build image: "redoxos/redoxer:aarch64" script: - - mkdir -p target/${ARCH} - - redoxer env make BUILD=target/${ARCH} + - redoxer env make variables: ARCH: "aarch64" i586: stage: cross-build script: - - mkdir -p target/${ARCH} - - TARGET=${ARCH}-unknown-redox redoxer env make BUILD=target/${ARCH} + - TARGET=${ARCH}-unknown-redox redoxer env make variables: ARCH: "i586" riscv64gc: stage: cross-build script: - - mkdir -p target/${ARCH} - - TARGET=${ARCH}-unknown-redox redoxer env make BUILD=target/${ARCH} + - TARGET=${ARCH}-unknown-redox redoxer env make variables: ARCH: "riscv64gc" fmt: stage: build script: - - rustup component add rustfmt - - rustfmt --check + - redoxer env cargo fmt --check x86_64:boot: stage: test needs: [x86_64] script: - - mkdir -p target/${ARCH} - export COOKBOOK_SOURCE_IDENT=$CI_COMMIT_SHA - - redoxer env make BUILD=target/${ARCH} - - timeout -s KILL 9m redoxer exec --folder target/${ARCH}/:/usr/lib/boot uname -a - variables: - ARCH: "x86_64" + - redoxer env make test x86_64:relibc: stage: test needs: [x86_64] + # TODO: remove + allow_failure: true script: - - redoxer pkg relibc-tests-bins - export COOKBOOK_SOURCE_IDENT=$CI_COMMIT_SHA - - mkdir -p target/${TARGET}/sysroot/{usr/lib/boot,root} target/${TARGET}/root - - redoxer env make BUILD=target/${TARGET}/sysroot/usr/lib/boot - - (cd target/${TARGET}/sysroot && mv home/user/relibc-tests/* root/) - - timeout -s KILL 9m redoxer exec --folder target/${TARGET}/sysroot/:/ make run - # It is fine if failing sometimes - allow_failure: true - variables: - TARGET: "x86_64-unknown-redox" + - timeout -s KILL 9m redoxer env make test-relibc -profiling-compile: - stage: other-features - allow_failure: true +x86_64:profiling: + stage: build + needs: [x86_64] script: - make check + - make check variables: ARCH: "x86_64" KERNEL_CHECK_FEATURES: profiling diff --git a/Makefile b/Makefile index 733415908f..feeb66a9c7 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ .PHONY: all check SOURCE:=$(dir $(realpath $(lastword $(MAKEFILE_LIST)))) -BUILD?=$(CURDIR) export RUST_TARGET_PATH=$(SOURCE)/targets ifeq ($(TARGET),) @@ -11,6 +10,9 @@ else ARCH?=$(shell echo "$(TARGET)" | cut -d - -f1) endif +BUILD?=$(CURDIR)/build/$(ARCH) +DESTDIR?=./sysroot + ifeq ($(ARCH),riscv64gc) override ARCH:=riscv64 GNU_TARGET=riscv64-unknown-redox @@ -21,6 +23,7 @@ else GNU_TARGET=$(ARCH)-unknown-redox endif +OBJCOPY?=$(GNU_TARGET)-objcopy all: $(BUILD)/kernel $(BUILD)/kernel.sym @@ -31,7 +34,10 @@ TARGET_SPEC=$(RUST_TARGET_PATH)/$(ARCH)-unknown-kernel.json KERNEL_CARGO_FEATURES?= -$(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell find $(SOURCE) -name "*.rs" -type f) +$(BUILD): + mkdir -p "$@" + +$(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell find $(SOURCE) -name "*.rs" -type f) | $(BUILD) cargo rustc \ --bin kernel \ --manifest-path "$(MANIFEST)" \ @@ -45,13 +51,13 @@ $(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell --emit link="$(BUILD)/kernel.all" $(BUILD)/kernel.sym: $(BUILD)/kernel.all - $(GNU_TARGET)-objcopy \ + $(OBJCOPY) \ --only-keep-debug \ "$(BUILD)/kernel.all" \ "$(BUILD)/kernel.sym" $(BUILD)/kernel: $(BUILD)/kernel.all - $(GNU_TARGET)-objcopy \ + $(OBJCOPY) \ --strip-debug \ "$(BUILD)/kernel.all" \ "$(BUILD)/kernel" @@ -65,3 +71,29 @@ check: --target "$(TARGET_SPEC)" \ -Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \ --features=$(KERNEL_CHECK_FEATURES) + +clean: + rm -rf build sysroot target config.toml + +install: all + @mkdir -pv "$(DESTDIR)/usr/lib/boot/" + cp -v $(BUILD)/kernel.all "$(DESTDIR)/usr/lib/boot/" + cp -v $(BUILD)/kernel.sym "$(DESTDIR)/usr/lib/boot/" + cp -v $(BUILD)/kernel "$(DESTDIR)/usr/lib/boot/" + +# test if booting +# to ensure it's using this kernel, set COOKBOOK_SOURCE_IDENT env before build +test: all + $(MAKE) install + REDOXER_SYSROOT=$(DESTDIR) redoxer exec uname -a + +# test with interactive gui +test-gui: all + $(MAKE) install + REDOXER_SYSROOT=$(DESTDIR) redoxer exec --gui ion + +# test with relibc tests +test-relibc: all + $(MAKE) install + REDOXER_SYSROOT=$(DESTDIR) redoxer pkg relibc-tests-bins + REDOXER_SYSROOT=$(DESTDIR) redoxer exec relibc-tests-runner diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 83900eb226..301edef047 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -6,6 +6,7 @@ use core::ptr::NonNull; use alloc::{boxed::Box, string::String, vec::Vec}; use hashbrown::HashMap; +use rmm::{BumpAllocator, FrameAllocator, PageMapper}; use spin::{Once, RwLock}; use crate::{ @@ -33,7 +34,11 @@ mod spcr; pub mod srat; mod xsdt; -unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) { +unsafe fn map_linearly( + addr: PhysicalAddress, + len: usize, + mapper: &mut PageMapper, +) { unsafe { let base = PhysicalAddress::new(crate::memory::round_down_pages(addr.data())); let aligned_len = crate::memory::round_up_pages(len + (addr.data() - base.data())); @@ -50,7 +55,10 @@ unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::me } } -pub fn get_sdt(sdt_address: PhysicalAddress, mapper: &mut KernelMapper) -> &'static Sdt { +pub fn get_sdt( + sdt_address: PhysicalAddress, + mapper: &mut PageMapper, +) -> &'static Sdt { let sdt; unsafe { @@ -93,16 +101,20 @@ impl Rxsdt for RxsdtEnum { pub static RXSDT_ENUM: Once = Once::new(); -/// Initialses the global `RXSDT_ENUM` if RSDT or XSDT was found and maps the SDT pages. -/// It does not perform any allocations -pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { +/// Initialises the global `RXSDT_ENUM` if RSDT or XSDT was found and maps the SDT pages. +/// +/// It does not use `TheFrameAllocator` nor does it heap-allocate. +pub unsafe fn init_before_mem( + already_supplied_rsdp: Option>, + mapper: &mut PageMapper>, +) { unsafe { // Search for RSDP let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp); if let Some(rsdp) = rsdp_opt { debug!("SDT address: {:#x}", rsdp.sdt_address().data()); - let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw()); + let rxsdt = get_sdt(rsdp.sdt_address(), mapper); let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) { let mut initialized = false; @@ -139,7 +151,7 @@ pub unsafe fn init_before_mem(already_supplied_rsdp: Option>) { // TODO: Don't touch ACPI tables in kernel? for sdt in rxsdt.iter() { - get_sdt(sdt, &mut KernelMapper::lock_rw()); + get_sdt(sdt, mapper); } } else { error!("NO RSDP FOUND"); diff --git a/src/arch/aarch64/device/irqchip/gic.rs b/src/arch/aarch64/device/irqchip/gic.rs index d5aef88c95..d79ed945a2 100644 --- a/src/arch/aarch64/device/irqchip/gic.rs +++ b/src/arch/aarch64/device/irqchip/gic.rs @@ -1,8 +1,8 @@ use super::InterruptController; use crate::{ dtb::{ - get_mmio_address, irqchip::{InterruptHandler, IrqCell, IrqDesc}, + translate_mmio_address, }, sync::CleanLockToken, }; @@ -60,7 +60,7 @@ impl GenericInterruptController { if chunk.size.is_none() { break; } - let addr = get_mmio_address(fdt, node, &chunk).unwrap(); + let addr = translate_mmio_address(fdt, node, &chunk).unwrap(); match idx { 0 => (regs.0, regs.1) = (addr, chunk.size.unwrap()), 2 => (regs.2, regs.3) = (addr, chunk.size.unwrap()), diff --git a/src/arch/aarch64/device/irqchip/gicv3.rs b/src/arch/aarch64/device/irqchip/gicv3.rs index 9d8a0d5211..77d42a35df 100644 --- a/src/arch/aarch64/device/irqchip/gicv3.rs +++ b/src/arch/aarch64/device/irqchip/gicv3.rs @@ -5,8 +5,8 @@ use fdt::{node::NodeProperty, Fdt}; use super::{gic::GicDistIf, InterruptController}; use crate::{ dtb::{ - get_mmio_address, irqchip::{InterruptHandler, IrqCell, IrqDesc}, + translate_mmio_address, }, sync::CleanLockToken, }; @@ -53,7 +53,7 @@ impl GicV3 { // Read registers let mut chunks = node.reg().unwrap(); if let Some(gicd) = chunks.next() - && let Some(addr) = get_mmio_address(fdt, &node, &gicd) + && let Some(addr) = translate_mmio_address(fdt, &node, &gicd) { unsafe { self.gic_dist_if.init(crate::PHYS_OFFSET + addr); @@ -62,7 +62,7 @@ impl GicV3 { for _ in 0..gicrs { if let Some(gicr) = chunks.next() { self.gicrs.push(( - get_mmio_address(fdt, &node, &gicr).unwrap(), + translate_mmio_address(fdt, &node, &gicr).unwrap(), gicr.size.unwrap(), )); } diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index 65e3fe339b..0b9c5d43ca 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -100,6 +100,15 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { // Initialize paging paging::init(); + let mut mapper = rmm::PageMapper::current(rmm::TableKind::Kernel, bump_allocator); + #[cfg(feature = "acpi")] + { + use crate::acpi; + acpi::init_before_mem(args.acpi_rsdp(), &mut mapper); + } + + crate::memory::init_mm(mapper.allocator_mut()); + crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0)); // Setup kernel heap diff --git a/src/arch/riscv64/start.rs b/src/arch/riscv64/start.rs index a825536aa9..ce14de38e9 100644 --- a/src/arch/riscv64/start.rs +++ b/src/arch/riscv64/start.rs @@ -105,7 +105,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { } // Initialize RMM - crate::startup::memory::init(&args, None, None); + let mut bump_allocator = crate::startup::memory::init(&args, None, None); let boot_hart_id = get_boot_hart_id(args.env()).expect("Didn't get boot HART id from bootloader"); @@ -114,6 +114,8 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! { paging::init(); + crate::memory::init_mm(&mut bump_allocator); + crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0)); // Setup kernel heap diff --git a/src/arch/x86_shared/start.rs b/src/arch/x86_shared/start.rs index 133ef4c845..912f8a01a2 100644 --- a/src/arch/x86_shared/start.rs +++ b/src/arch/x86_shared/start.rs @@ -4,6 +4,8 @@ //! defined in other files inside of the `arch` module use core::{arch::naked_asm, cell::SyncUnsafeCell, mem::offset_of}; +use rmm::PageMapper; + use crate::{ allocator, arch::{device, gdt, idt, interrupt, paging}, @@ -112,13 +114,14 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! { // Initialize paging paging::init(); + let mut mapper = PageMapper::current(rmm::TableKind::Kernel, bump_allocator); if cfg!(feature = "acpi") { - crate::acpi::init_before_mem(args.acpi_rsdp()); + crate::acpi::init_before_mem(args.acpi_rsdp(), &mut mapper); } - numa::init(&mut bump_allocator); + numa::init(mapper.allocator_mut()); - crate::memory::init_mm(bump_allocator); + crate::memory::init_mm(mapper.allocator_mut()); #[cfg(target_arch = "x86_64")] crate::arch::alternative::early_init(true); diff --git a/src/context/context.rs b/src/context/context.rs index 12862aec9d..7ff6f14d68 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -333,7 +333,6 @@ impl Context { pub fn unblock_no_ipi(&mut self) -> bool { if self.status.is_soft_blocked() { self.status = Status::Runnable; - self.wake = None; self.status_reason = ""; true diff --git a/src/context/mod.rs b/src/context/mod.rs index 1518dc3367..6999bc2802 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -9,13 +9,17 @@ use alloc::{ use core::{cmp::Reverse, num::NonZeroUsize, ops::Deref}; use crate::{ - context::memory::AddrSpaceWrapper, + context::{ + memory::AddrSpaceWrapper, + switch::{BASE_SLICE_TICKS, NANOS_PER_TICK, SCALE, SCHED_PRIO_TO_WEIGHT, TICK_INTERVAL}, + }, cpu_set::LogicalCpuSet, + ipi::{ipi, IpiKind, IpiTarget}, memory::{RmmA, RmmArch, TableKind}, percpu::PercpuBlock, sync::{ ArcRwLockWriteGuard, CleanLockToken, LockToken, Mutex, MutexGuard, RwLock, RwLockReadGuard, - RwLockWriteGuard, L0, L1, L2, L4, + RwLockWriteGuard, L0, L1, L2, L3, L4, }, syscall::error::Result, }; @@ -83,7 +87,7 @@ static IDLE_CONTEXTS: Mutex> = Mutex::new(VecDeque: pub struct RunContextData { // queue: VecDeque, queue: BTreeMap<(u64, Reverse, u32), (u64, u64, WeakContextRef)>, // ((vd, rem_slice, ctxt_id), (vtime, weight, context)) - timers: BTreeSet<(u128, WeakContextRef)>, // (wake, context) + timers: BTreeSet<(u128, WeakContextRef)>, // (wake, context) count: usize, v: u64, total_weight: u64, @@ -135,6 +139,36 @@ pub fn run_contexts_try(token: LockToken<'_, L0>) -> Option, token: &mut LockToken<'_, L3>) -> bool { + let cpu_id = { + let mut guard = context_lock.write(token.token()); + if !guard.unblock_no_ipi() { + if guard.status.is_runnable() { + // already set to runnable externally + wakeup_context(context_lock); + } + return false; + } + guard.cpu_id + }; + + wakeup_context(context_lock); + + if let Some(cpu_id) = cpu_id + && cpu_id != crate::cpu_id() + { + ipi(IpiKind::Wakeup, IpiTarget::Other); + } + + true +} + +pub fn wakeup_context(context_lock: &Arc) { + let percpu: &PercpuBlock = PercpuBlock::current(); + let weak = WeakContextRef(Arc::downgrade(context_lock)); + percpu.switch_internals.wakeup_list.borrow_mut().push(weak); +} + pub fn init(token: &mut CleanLockToken) { let owner = None; // kmain not owned by any fd let mut context = Context::new(owner).expect("failed to create kmain context"); @@ -261,7 +295,6 @@ pub fn spawn( let context_lock = Arc::new(ContextLock::new(context)); let context_ref = ContextRef(Arc::clone(&context_lock)); let run_ref = WeakContextRef(Arc::downgrade(&context_ref.0)); - idle_contexts(token.downgrade()).push_back(run_ref); contexts_mut(token.downgrade()).insert(context_ref); Ok(context_lock) diff --git a/src/context/switch.rs b/src/context/switch.rs index b1722eaf61..7b6b375b2c 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -13,7 +13,7 @@ use crate::{ percpu::PercpuBlock, sync::{ArcRwLockWriteGuard, CleanLockToken, L4}, }; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use core::{ cell::{Cell, RefCell}, cmp::Reverse, @@ -32,16 +32,16 @@ enum UpdateResult { } // A simple geometric series where value[i] ~= value[i + 1] * 1.25 -const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [ +pub const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [ 88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904, 3906, 3121, 2501, 1991, 1586, 1277, 1024, 820, 655, 526, 423, 335, 272, 215, 172, 137, 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, ]; -const SCALE: u128 = 1 << 40; -const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms -const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms -const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms +pub const SCALE: u128 = 1 << 40; +pub const TICK_INTERVAL: u64 = 3; // Approx 6.75 ms +pub const BASE_SLICE_TICKS: u64 = TICK_INTERVAL * 3; // Approx 20.25 ms +pub const NANOS_PER_TICK: u128 = 2_250_000; // 2.25 ms /// Determines if a given context is eligible to be scheduled on a given CPU (in /// principle, the current CPU). @@ -190,6 +190,8 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { let mut wakeups = wakeup_contexts(token); let mut push_idle: SmallVec<[WeakContextRef; 16]> = SmallVec::new(); + // These timers coukd have expired + let mut timers: SmallVec<[(u128, WeakContextRef); 16]> = SmallVec::new(); if let Some(mut run_contexts) = run_contexts_try(token.token()) { // Pop Timers while let Some((wake, _)) = run_contexts.timers.first() { @@ -197,12 +199,31 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { break; } - if let Some((_, context_ref)) = run_contexts.timers.pop_first() { - wakeups.push(context_ref); + if let Some(entry) = run_contexts.timers.pop_first() { + timers.push(entry); } } } + for (wake, context_ref) in timers { + let Some(context_lock) = context_ref.upgrade() else { + continue; + }; + + let guard = context_lock.read(token.token()); + if guard.status.is_soft_blocked() && guard.wake == Some(wake) { + wakeups.push(context_ref); + } + } + + // Drain from percpu + { + let mut percpu_wake = percpu.switch_internals.wakeup_list.borrow_mut(); + for context_ref in percpu_wake.drain(..) { + wakeups.push(context_ref.clone()); + } + } + if wakeups.len() > 0 { let mut run_contexts = run_contexts(token.token()); for context_ref in wakeups { @@ -388,9 +409,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult { } } -fn wakeup_contexts( - token: &mut CleanLockToken, -) -> SmallVec<[WeakContextRef; 16]> { +fn wakeup_contexts(token: &mut CleanLockToken) -> SmallVec<[WeakContextRef; 16]> { // TODO: Optimise this somehow let mut wakeups = SmallVec::new(); let current_context = context::current(); @@ -687,6 +706,9 @@ pub struct ContextSwitchPercpu { /// The idle process. idle_ctxt: RefCell>>, pub(crate) being_sigkilled: Cell, + + // wakeups + pub(crate) wakeup_list: RefCell>, } impl ContextSwitchPercpu { @@ -698,6 +720,7 @@ impl ContextSwitchPercpu { current_ctxt: RefCell::new(None), idle_ctxt: RefCell::new(None), being_sigkilled: Cell::new(false), + wakeup_list: RefCell::new(Vec::new()), #[cfg(feature = "profiling")] current_dbg_id: core::sync::atomic::AtomicU32::new(!0), diff --git a/src/dtb/mod.rs b/src/dtb/mod.rs index b9b8b23094..26a3b292da 100644 --- a/src/dtb/mod.rs +++ b/src/dtb/mod.rs @@ -102,53 +102,145 @@ pub fn register_dev_memory_ranges(dt: &Fdt) { } } - let Some(soc_node) = dt.find_node("/soc") else { - warn!("failed to find /soc in devicetree"); - return; - }; - let Some(reg) = soc_node.ranges() else { - warn!("devicetree /soc has no ranges"); - return; - }; - for chunk in reg { - debug!( - "dev mem 0x{:08x} 0x{:08x} 0x{:08x} 0x{:08x}", - chunk.child_bus_address_hi, - chunk.child_bus_address, - chunk.parent_bus_address, - chunk.size - ); + if let Some(soc_node) = dt.find_node("/soc") { + if let Some(reg) = soc_node.ranges() { + for chunk in reg { + debug!( + "dev mem 0x{:08x} 0x{:08x} 0x{:08x} 0x{:08x}", + chunk.child_bus_address_hi, + chunk.child_bus_address, + chunk.parent_bus_address, + chunk.size + ); - /*TODO: soc memory may contain all free memory! - register_memory_region( - chunk.parent_bus_address, - chunk.size, - BootloaderMemoryKind::Device, - );*/ - } + /*TODO: soc memory may contain all free memory! + register_memory_region( + chunk.parent_bus_address, + chunk.size, + BootloaderMemoryKind::Device, + );*/ + } + } else { + warn!("devicetree /soc has no ranges"); + } - // also add all soc-internal devices because they might not be shown in ranges - // (identity-mapped soc bus may have empty ranges) - for device in soc_node.children() { - if let Some(reg) = device.reg() { - for entry in reg { - if let Some(size) = entry.size { - let addr = entry.starting_address as usize; - if let Some(mapped_addr) = get_mmio_address(dt, &device, &entry) { - debug!( - "soc device {} 0x{:08x} -> 0x{:08x} size 0x{:08x}", - device.name, addr, mapped_addr, size - ); - register_memory_region(mapped_addr, size, BootloaderMemoryKind::Device); + // also add direct /soc children because they might not be shown in + // ranges (an identity-mapped bus may have empty ranges) + for device in soc_node.children() { + if let Some(reg) = device.reg() { + for entry in reg { + if let Some(size) = entry.size { + let addr = entry.starting_address as usize; + if let Some(mapped_addr) = get_mmio_address(dt, &device, &entry) { + debug!( + "soc device {} 0x{:08x} -> 0x{:08x} size 0x{:08x}", + device.name, addr, mapped_addr, size + ); + register_memory_region(mapped_addr, size, BootloaderMemoryKind::Device); + } } } } } + } else { + warn!("failed to find /soc in devicetree"); + } + + // The selected console may be below a nested bus whose own `reg` range + // does not cover all children. Register the exact translated range so it + // remains mapped after the boot-time identity map is replaced. + if let Some((address, size, _, _, _)) = diag_uart_range(dt) { + debug!( + "diagnostic UART 0x{:08x} size 0x{:08x}", + address.data(), + size + ); + register_memory_region(address.data(), size, BootloaderMemoryKind::Device); + } + + // Interrupt controllers are not required to live below /soc. Some + // devicetrees place the primary GIC directly below the root node, so its + // register ranges would otherwise be absent from the kernel physmap. + if let Some(root) = dt.find_node("/") { + for controller in root + .children() + .filter(|node| node.property("interrupt-controller").is_some()) + { + let Some(regions) = controller.reg() else { + continue; + }; + for region in regions { + let Some(size) = region.size else { + continue; + }; + let Some(address) = translate_mmio_address(dt, &controller, ®ion) else { + continue; + }; + debug!( + "root interrupt controller {} 0x{:08x} size 0x{:08x}", + controller.name, address, size + ); + register_memory_region(address, size, BootloaderMemoryKind::Device); + } + } } } -// FIXME return PhysicalAddress -pub fn get_mmio_address(fdt: &Fdt, _device: &FdtNode, region: &MemoryRegion) -> Option { +fn same_node(left: FdtNode<'_, '_>, right: FdtNode<'_, '_>) -> bool { + if left.name != right.name { + return false; + } + // FdtNode is reconstructed while walking the tree and has no identity + // operation. A property's value is a slice into the original DTB, so equal + // `reg` slice pointers identify the same node occurrence without relying + // on names that may repeat below different buses. + match (left.property("reg"), right.property("reg")) { + (Some(left_reg), Some(right_reg)) => core::ptr::eq(left_reg.value, right_reg.value), + _ => false, + } +} + +fn translate_bus_address(bus: FdtNode<'_, '_>, address: usize, size: usize) -> Option { + let ranges_property = bus.property("ranges")?; + if ranges_property.value.is_empty() { + return Some(address); + } + + let last_offset = size.saturating_sub(1); + let ranges = bus.ranges()?; + for range in ranges { + let Some(offset) = address.checked_sub(range.child_bus_address) else { + continue; + }; + if offset < range.size && last_offset < range.size - offset { + return range.parent_bus_address.checked_add(offset); + } + } + None +} + +fn translate_from_subtree( + bus_or_device: FdtNode<'_, '_>, + target: FdtNode<'_, '_>, + address: usize, + size: usize, +) -> Option { + if same_node(bus_or_device, target) { + return Some(address); + } + + for child in bus_or_device.children() { + if let Some(child_address) = translate_from_subtree(child, target, address, size) { + // `bus_or_device` is the bus parent of the subtree that matched. + return translate_bus_address(bus_or_device, child_address, size); + } + } + None +} + +/// Translate a device's bus-relative `reg` address through every ancestor's +/// `ranges` property until it reaches the CPU physical address space. +pub fn translate_mmio_address(fdt: &Fdt, device: &FdtNode, region: &MemoryRegion) -> Option { /* DT spec 2.3.8 "ranges": * The ranges property provides a means of defining a mapping or translation between * the address space of the bus (the child address space) and the address space of the bus @@ -159,15 +251,32 @@ pub fn get_mmio_address(fdt: &Fdt, _device: &FdtNode, region: &MemoryRegion) -> * children of the node and the parent address space. */ - // FIXME assumes all the devices are connected to CPUs via the /soc bus + let address = region.starting_address as usize; + let size = region.size.unwrap_or(1); + let root = fdt.find_node("/")?; + + // The root is already the CPU address space, so only its children perform + // translations. This also supports devices that are not under `/soc`. + for child in root.children() { + if let Some(translated) = translate_from_subtree(child, *device, address, size) { + translated.checked_add(size.saturating_sub(1))?; + return Some(translated); + } + } + None +} + +// FIXME return PhysicalAddress +pub fn get_mmio_address(fdt: &Fdt, _device: &FdtNode, region: &MemoryRegion) -> Option { let mut mapped_addr = region.starting_address as usize; let size = region.size.unwrap_or(0).saturating_sub(1); let last_address = mapped_addr.saturating_add(size); if let Some(parent) = fdt.find_node("/soc") { - let mut ranges = parent.ranges().map(|f| f.peekable())?; + let mut ranges = parent.ranges().map(|ranges| ranges.peekable())?; if ranges.peek().is_some() { - let parent_range = ranges.find(|x| { - x.child_bus_address <= mapped_addr && last_address - x.child_bus_address <= x.size + let parent_range = ranges.find(|range| { + range.child_bus_address <= mapped_addr + && last_address - range.child_bus_address <= range.size })?; mapped_addr = parent_range .parent_bus_address @@ -219,7 +328,7 @@ pub fn diag_uart_range<'a>(dtb: &'a Fdt) -> Option<(PhysicalAddress, usize, bool let mut reg = uart_node.reg()?; let memory = reg.next()?; - let address = get_mmio_address(dtb, &uart_node, &memory)?; + let address = translate_mmio_address(dtb, &uart_node, &memory)?; Some(( PhysicalAddress::new(address), @@ -244,3 +353,38 @@ pub fn fill_env_data(dt: &Fdt, env_base: usize) -> usize { 0 } } + +#[cfg(test)] +mod tests { + use super::translate_mmio_address; + use fdt::Fdt; + + static MMIO_DTB: &[u8] = include_bytes!("testdata/mmio.dtb"); + + fn translated(path: &str) -> Option { + let fdt = Fdt::new(MMIO_DTB).unwrap(); + let node = fdt.find_node(path).unwrap(); + let region = node.reg().unwrap().next().unwrap(); + translate_mmio_address(&fdt, &node, ®ion) + } + + #[test] + fn translates_uart_through_nested_and_empty_ranges() { + assert_eq!(translated("/soc/bus@1000/serial@200"), Some(0x1200)); + } + + #[test] + fn accepts_region_ending_exactly_at_range_boundary() { + assert_eq!(translated("/soc/bus@1000/device@ff0"), Some(0x1ff0)); + } + + #[test] + fn rejects_region_crossing_range_boundary() { + assert_eq!(translated("/soc/bus@1000/device@ff1"), None); + } + + #[test] + fn empty_ranges_is_identity_mapping() { + assert_eq!(translated("/identity-bus/device@3000"), Some(0x3000)); + } +} diff --git a/src/dtb/testdata/mmio.dtb b/src/dtb/testdata/mmio.dtb new file mode 100644 index 0000000000..c919ca99a1 Binary files /dev/null and b/src/dtb/testdata/mmio.dtb differ diff --git a/src/dtb/testdata/mmio.dts b/src/dtb/testdata/mmio.dts new file mode 100644 index 0000000000..0ff7049c62 --- /dev/null +++ b/src/dtb/testdata/mmio.dts @@ -0,0 +1,48 @@ +/dts-v1/; + +/ { + #address-cells = <2>; + #size-cells = <2>; + + chosen { + stdout-path = "/soc/bus@1000/serial@200:115200n8"; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bus@1000 { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x0 0x1000 0x0 0x1000>; + ranges = <0x0 0x0 0x1000 0x1000>; + + serial@200 { + reg = <0x200 0x18>; + }; + + device@ff0 { + reg = <0xff0 0x10>; + }; + + device@ff1 { + reg = <0xff1 0x10>; + }; + }; + }; + + identity-bus { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + device@3000 { + reg = <0x0 0x3000 0x0 0x20>; + }; + }; +}; diff --git a/src/event.rs b/src/event.rs index 9457837773..91c229d82f 100644 --- a/src/event.rs +++ b/src/event.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use core::sync::atomic::{AtomicUsize, Ordering}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use smallvec::SmallVec; -use syscall::data::GlobalSchemes; +use syscall::{data::GlobalSchemes, EAGAIN, EINTR}; use crate::{ context, @@ -29,6 +29,9 @@ pub struct EventQueue { /// handle to the SAME queue; the queue must survive until the last fd /// closes. Starts at 1 for the fd created by `open`. refs: AtomicUsize, + /// Pending EVENT_TIMEOUT_ID timeout recorded by `write()` and consumed + /// by `read_with_timeout()` (upstream event-timeout support). + timeout_opt: Mutex>, } const EVENTFD_COUNTER_MAX: u64 = u64::MAX - 1; @@ -48,6 +51,7 @@ impl EventQueue { id, queue: WaitQueue::new(), refs: AtomicUsize::new(1), + timeout_opt: Mutex::new(None), } } @@ -71,8 +75,61 @@ impl EventQueue { .receive_into_user(buf, block, "EventQueue::read", token) } + pub fn read_with_timeout( + &self, + buf: UserSliceWo, + timeout: usize, + token: &mut CleanLockToken, + ) -> Result { + // if zero, instant timeout + let block = timeout > 0; + if block { + let mut time = crate::time::monotonic(token); + time += timeout as u128 * 1_000_000; + context::current().write(token.token()).wake = Some(time); + } + let r = self + .queue + .receive_into_user(buf, block, "EventQueue::read_with_timeout", token); + match (r, block) { + (Ok(r), _) => return Ok(r), + (err @ Err(Error { errno: EINTR }), true) => { + let old_wake = context::current().write(token.token()).wake.take(); + // The scheduler clears `wake` on timeout + if !old_wake.is_none() { + return err; + } + // proceed writing timeout + } + (Err(Error { errno: EAGAIN }), false) => { + // proceed writing timeout + } + (err, _) => return err, + }; + + // TODO: let the scheduler write this for us? + let event = Event { + id: syscall::EVENT_TIMEOUT_ID, + // TODO: it's undefined when flags is written to EVENT_TIMEOUT_ID + flags: EventFlags::EVENT_READ, + data: timeout, + }; + + let bytes_copied = buf.copy_common_bytes_from_slice(&event)?; + return Ok(bytes_copied); + } + pub fn write(&self, events: &[Event], token: &mut CleanLockToken) -> Result { for event in events { + if event.id == syscall::EVENT_TIMEOUT_ID { + if event.flags.is_empty() { + self.timeout_opt.lock(token.token()).take(); + } else { + *self.timeout_opt.lock(token.token()) = Some(event.data); + } + + continue; + } let file = { let context_ref = context::current(); let mut context = context_ref.read(token.token()); diff --git a/src/memory/mod.rs b/src/memory/mod.rs index c6f4e99216..39891bc9fb 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -74,30 +74,36 @@ pub fn total_frames() -> usize { /// Allocate a range of frames pub fn allocate_p2frame(order: u32) -> Option { - let initial_index = get_round_robin_index(); - let mut index = initial_index; + static RR_INDEX: Mutex = Mutex::new(0); + let len = FREE_LISTS.get().unwrap().len(); - loop { - if let Some(frame) = allocate_p2frame_complex(order, (), None, order, index).map(|(f, _)| f) - { - return Some(frame); - } - index = get_round_robin_index(); - if index == initial_index { - return None; + if len == 1 { + return allocate_p2frame_complex(order, (), None, order, 0).map(|e| e.0); + } + + let index = { + let mut lock = RR_INDEX.lock(); + let index = + usize::try_from(*lock).expect("Maximum number of memory regions supported is 128"); + *lock = u8::try_from((index + 1) % len).expect("Maximum number of memory regions is 128"); + index + }; + for i in index..len { + if let Some(frame) = allocate_p2frame_complex(order, (), None, order, i) { + return Some(frame.0); } } + for i in 0..index { + if let Some(frame) = allocate_p2frame_complex(order, (), None, order, i) { + return Some(frame.0); + } + } + None } pub fn allocate_frame() -> Option { allocate_p2frame(0) } -fn get_round_robin_index() -> usize { - static CURRENT_INDEX: AtomicUsize = AtomicUsize::new(0); - let len = FREE_LISTS.get().unwrap().len(); - CURRENT_INDEX.fetch_add(1, Ordering::Relaxed) % len -} - // TODO: Flags, strategy pub fn allocate_p2frame_complex( _req_order: u32, @@ -543,7 +549,7 @@ const _: () = { }; #[cold] -fn init_sections(mut allocator: BumpAllocator) { +fn init_sections(mut allocator: &mut BumpAllocator) { let number_of_memory_regions = numa::number_of_memory_regions(); let (free_areas, offset_into_first_free_area) = allocator.free_areas(); @@ -952,7 +958,7 @@ fn init_sections(mut allocator: BumpAllocator) { } #[cold] -pub fn init_mm(allocator: BumpAllocator) { +pub fn init_mm(allocator: &mut BumpAllocator) { init_sections(allocator); unsafe { diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 9989e17662..70a22550bf 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -3,8 +3,8 @@ use crate::{ self, context::{HardBlockedReason, LockedFdTbl, SignalState}, file::InternalFlags, - memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, - Context, ContextLock, Status, + memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec}, + unblock_context, wakeup_context, Context, ContextLock, Status, }, memory::{Page, VirtualAddress, PAGE_SIZE}, ptrace, @@ -1580,6 +1580,7 @@ impl ContextHandle { reason: HardBlockedReason::NotYetStarted, } => { *status = Status::Runnable; + wakeup_context(&context); Ok(buf.len()) } _ => Err(Error::new(EINVAL)), @@ -1747,12 +1748,12 @@ impl ContextHandle { } = guard.status { guard.status = Status::Runnable; + wakeup_context(&context); } Ok(size_of::()) } ContextVerb::Interrupt => { - let mut guard = context.write(token.token()); - guard.unblock(); + unblock_context(&context, &mut token.token().downgrade()); Ok(size_of::()) } ContextVerb::ForceKill => { @@ -1783,13 +1784,16 @@ impl ContextHandle { } crate::syscall::exit_this_context(None, token); } else { - let mut ctxt = context.write(token.token()); - //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); - if let context::Status::Dead { .. } = ctxt.status { - return Ok(size_of::()); + { + let mut ctxt = context.write(token.token()); + //trace!("FORCEKILL NONSELF={} {}, SELF={}", ctxt.debug_id, ctxt.pid, context::current().read().debug_id); + if ctxt.status.is_dead() { + return Ok(size_of::()); + } + ctxt.status = context::Status::Runnable; + ctxt.being_sigkilled = true; } - ctxt.status = context::Status::Runnable; - ctxt.being_sigkilled = true; + wakeup_context(&context); Ok(size_of::()) } } diff --git a/src/scheme/user.rs b/src/scheme/user.rs index f09b21665a..c3edb4c9b4 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -22,10 +22,12 @@ use crate::{ AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode, PageSpan, DANGLING, }, - BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status, + unblock_context, wakeup_context, BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, + Status, WeakContextRef, }, event, memory::{Frame, Page, VirtualAddress, PAGE_SIZE}, + percpu::PercpuBlock, scheme::SchemeId, sync::{CleanLockToken, LockToken, Mutex, RwLock, WaitQueue, L1}, syscall::{ @@ -894,7 +896,7 @@ impl UserInner { } }; - let context = context.upgrade().ok_or(Error::new(ESRCH))?; + let context_lock = context.upgrade().ok_or(Error::new(ESRCH))?; let mut lock_token = token.token(); let (frame, _) = AddrSpace::current()? @@ -905,12 +907,13 @@ impl UserInner { .ok_or(Error::new(EFAULT))?; { - let mut context = context.write(token.token()); + let mut context = context_lock.write(token.token()); if let Status::HardBlocked { reason: HardBlockedReason::AwaitingMmap { .. }, } = context.status { - context.status = Status::Runnable + context.status = Status::Runnable; + wakeup_context(&context_lock); } context.fmap_ret = Some(Frame::containing(frame)); } @@ -982,7 +985,7 @@ impl UserInner { match context.upgrade() { Some(context) => { *o = State::Responded(response); - context.write(lock_token.token()).unblock(); + unblock_context(&context, &mut lock_token.token().downgrade()); } _ => { states.remove(tag as usize); diff --git a/src/startup/mod.rs b/src/startup/mod.rs index 42cd38760e..3bf05e3753 100644 --- a/src/startup/mod.rs +++ b/src/startup/mod.rs @@ -7,7 +7,7 @@ use core::{ use crate::{ arch::interrupt, - context::{self, switch::SwitchResult}, + context::{self, switch::SwitchResult, wakeup_context, Status}, memory::{PhysicalAddress, RmmA, RmmArch}, profiling, scheme, sync::CleanLockToken, @@ -175,13 +175,15 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! { match context::spawn(true, owner, userspace_init, &mut token) { Ok(context_lock) => { let mut context = context_lock.write(token.token()); - context.status = context::Status::Runnable; + context.status = Status::Runnable; context.name.clear(); context.name.push_str("[bootstrap]"); // TODO: Remove these from kernel context.euid = 0; context.egid = 0; + + wakeup_context(&context_lock); } Err(err) => { panic!("failed to spawn userspace_init: {:?}", err); diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index c508eb64fe..55325eaf1e 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -6,7 +6,7 @@ use alloc::{ }; use crate::{ - context::{self, ContextLock, PreemptGuardL2}, + context::{self, unblock_context, ContextLock, PreemptGuardL2}, sync::{CleanLockToken, LockToken, Mutex, L1, L2, L3}, }; @@ -33,7 +33,7 @@ impl WaitCondition { let len = contexts.len(); while let Some(context_weak) = contexts.pop() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write(token.token()).unblock(); + unblock_context(&context_ref, &mut token.token()); } } len @@ -46,7 +46,7 @@ impl WaitCondition { let len = contexts.len(); for context_weak in contexts.iter() { if let Some(context_ref) = context_weak.upgrade() { - context_ref.write(token.token()).unblock(); + unblock_context(&context_ref, &mut token.token()); } } len diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index 98b152eaa6..f81b55ed89 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -311,43 +311,36 @@ pub fn dup2( buf: UserSliceRo, token: &mut CleanLockToken, ) -> Result { - if fd == new_fd { - // Self-dup: a no-op if no buffer is supplied. But a self-dup2 WITH a - // buffer is a meaningful in-place operation — e.g. relibc issues - // dup2(ft, ft, "refresh") to re-materialize the inherited filetable, so - // it must NOT short-circuit. Duplicate first (fd is still valid), then - // replace the descriptor in place. Only this self-dup path is new; the - // normal path below is unchanged so ordinary dup2 (process-spawn stdio) - // keeps its exact original behavior. - if buf.is_empty() { - return Ok(new_fd); - } - let new_file = duplicate_file(fd, buf, false, token)?; - let old_file = { - let current_lock = context::current(); - let mut current = current_lock.read(token.token()); - let (context, mut token) = current.token_split(); - let old_file = context.remove_file(new_fd, &mut token); - context - .insert_file(new_fd, new_file, &mut token) - .ok_or(Error::new(EMFILE))?; - old_file - }; - if let Some(old) = old_file { - let _ = old.close(token); - } - Ok(new_fd) - } else { - let _ = close(new_fd, token); - let new_file = duplicate_file(fd, buf, false, token)?; + if fd == new_fd && buf.is_empty() { + return Ok(new_fd); + } + let new_file = duplicate_file(fd, buf, false, token)?; + + let old_file = { let current_lock = context::current(); let mut current = current_lock.read(token.token()); let (context, mut token) = current.token_split(); - context + + let old_file = context.remove_file(new_fd, &mut token); + + if context .insert_file(new_fd, new_file, &mut token) - .ok_or(Error::new(EMFILE)) + .is_none() + { + if let Some(old) = old_file { + context.insert_file(new_fd, old, &mut token); + } + return Err(Error::new(EMFILE)); + } + old_file + }; + + if let Some(old) = old_file { + let _ = old.close(token); } + + Ok(new_fd) } pub fn call( fd: FileHandle, diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index 65216532e6..ab4bf2259b 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -15,11 +15,10 @@ use crate::{ context::{ self, memory::{AddrSpace, AddrSpaceWrapper}, - ContextLock, + unblock_context, ContextLock, }, memory::{Page, PhysicalAddress, VirtualAddress}, sync::{CleanLockToken, Mutex, L1}, - time, }; use crate::syscall::{ @@ -39,9 +38,11 @@ pub struct FutexEntry { // CoW. target_virtaddr: VirtualAddress, // Context to wake up, and compare address spaces. - context_lock: Arc, + context_lock: Weak, // address space to check against if virt matches but not phys addr_space: Weak, + // FUTEX_WAIT_MULTIPLE: which array index this waiter corresponds to, + // reported back through Context::futex_wake_index on wake. waitv_index: Option, } @@ -52,6 +53,17 @@ pub struct FutexEntry { static FUTEXES: Mutex = Mutex::new(FutexList::with_hasher(DefaultHashBuilder::new())); +pub fn get_futex_stat(token: &mut CleanLockToken) -> (usize, usize) { + let mut regc = 0; + let mut regl = 0; + let registry = FUTEXES.lock(token.token()); + for (_, v) in registry.iter() { + regl += v.len(); + regc += 1; + } + (regc, regl) +} + fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option { // TODO: Move this elsewhere! if addr.data().saturating_add(size_of::()) >= crate::USER_END_OFFSET { @@ -142,9 +154,7 @@ pub fn futex( { let mut context = context_lock.write(token.token()); - context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| { - tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128 - }); + context.wake = timeout_opt.map(|time| time.to_nanos()); if let Some((tctl, pctl, _)) = context.sigcontrol() && tctl.currently_pending_unblocked(pctl) != 0 { @@ -159,7 +169,7 @@ pub fn futex( .or_insert_with(Vec::new) .push(FutexEntry { target_virtaddr, - context_lock: context_lock.clone(), + context_lock: Arc::downgrade(&context_lock), addr_space: Arc::downgrade(¤t_addrsp), waitv_index: None, }); @@ -198,25 +208,28 @@ pub fn futex( if futex.target_virtaddr != target_virtaddr || !current_addrsp_weak.ptr_eq(&futex.addr_space) { - i += 1; + if futex.addr_space.strong_count() == 0 { + futexes.swap_remove(i); + } else { + i += 1; + } continue; } - let should_wake = { - let mut context = futex.context_lock.write(token.token()); + if let Some(ctx) = futex.context_lock.upgrade() { + // FUTEX_WAIT_MULTIPLE wake-index bookkeeping: + // record which array index woke the waiter (the + // first wake wins) before unblocking. if let Some(index) = futex.waitv_index { + let mut context = ctx.write(token.token()); if context.futex_wake_index.is_none() { context.futex_wake_index = Some(index); } + drop(context); } - context.unblock(); - true - }; - if should_wake { - futexes.swap_remove(i); - woken += 1; - } else { - i += 1; + unblock_context(&ctx, &mut token.token().downgrade()); } + futexes.swap_remove(i); + woken += 1; } futexes.is_empty() @@ -260,7 +273,9 @@ pub fn futex( continue; } if woken < val { - futex.context_lock.write(token.token()).unblock(); + if let Some(ctx) = futex.context_lock.upgrade() { + unblock_context(&ctx, &mut token.token().downgrade()); + } futexes.remove(i); woken += 1; } else if requeued < val2 { @@ -288,9 +303,10 @@ pub fn futex( Ok(woken + requeued) } FUTEX_WAIT_MULTIPLE => { - // Wait on multiple futexes atomically; return the index of the first - // one that is woken. Semantics: val = number of FutexWaitv entries, - // addr2 = pointer to the array, val2 = optional TimeSpec timeout. + // Wait on multiple futexes atomically; return the index of the + // first one that is woken. Semantics: val = number of FutexWaitv + // entries, addr2 = pointer to the array, val2 = optional TimeSpec + // timeout. Linux 7.1 kernel/futex/waitv.c futex_waitv(). let count = val; if count == 0 || count > 128 { return Err(Error::new(EINVAL)); @@ -344,9 +360,7 @@ pub fn futex( { let mut context = context_lock.write(token.token()); - context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| { - tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128 - }); + context.wake = timeout_opt.map(|ts| ts.to_nanos()); context.futex_wake_index = None; if let Some((tctl, pctl, _)) = context.sigcontrol() && tctl.currently_pending_unblocked(pctl) != 0 @@ -362,7 +376,7 @@ pub fn futex( .or_insert_with(Vec::new) .push(FutexEntry { target_virtaddr: virt, - context_lock: context_lock.clone(), + context_lock: Arc::downgrade(&context_lock), addr_space: Arc::downgrade(¤t_addrsp), waitv_index: Some(i), });