vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rb: reapply Red Bear patches on upstream 00cfa2d2

Browse Source
This commit is contained in:
vasilito
2026-07-12 16:55:28 +03:00
parent 00cfa2d225
commit 2086faecb0
67 changed files with 4760 additions and 1321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
+5
View File
@@ -0,0 +1,5 @@
[build]
rustflags = [
# Kernel should preserve floating-point registers
"-Ctarget-feature=-sse,-sse2",
]
.verify-fork-functions.exclude
+32
View File
@@ -0,0 +1,32 @@
# Kernel functions intentionally removed/replaced by Red Bear commits.
# These exist in upstream but were replaced with RB-specific implementations
# or removed as part of RB refactoring. Documented per Phase 17 verification.
#
# Format: <file>:<fn_name>
# profiling was removed/replaced by RB diagnostics refactoring
src/profiling.rs:pub lookup_dbg_id
src/profiling.rs:unsafe walk_kstack
src/profiling.rs:unsafe walk_ustack
# context functions replaced by RB's _upper variants or refactored
src/context/context.rs:bulk_insert_files
src/context/context.rs:pub bulk_insert_files
src/context/context.rs:pub resize
src/context/memory.rs:insert_hole
src/context/memory.rs:remove_hole
src/context/memory.rs:resize_hole
# statistics functions removed during RB diagnostics cleanup
src/context/timeout.rs:pub get_timeout_stat
src/event.rs:pub get_event_stat
src/scheme/pipe.rs:pub get_pipe_stat
src/syscall/futex.rs:pub get_futex_stat
# event.rs — trait impls replaced by RB's event refactoring
src/event.rs:eq
src/event.rs:hash
src/event.rs:pub read_with_timeout
# syscall instruction end marker — RB refactored syscall dispatch
src/arch/x86_64/interrupt/syscall.rs:pub __syscall_instruction_end
Cargo.lock
Generated
+29 -27
View File
@@ -16,9 +16,9 @@ dependencies = [
[[package]]
name = "arrayvec"
version = "0.7.6"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "bit_field"
@@ -40,15 +40,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.12.1"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
[[package]]
name = "cc"
version = "1.2.63"
version = "1.2.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
dependencies = [
"find-msvc-tools",
"shlex",
@@ -104,11 +104,11 @@ dependencies = [
[[package]]
name = "kernel"
version = "0.5.12"
version = "0.5.12+rb0.3.1"
dependencies = [
"arrayvec",
"bitfield",
"bitflags 2.12.1",
"bitflags 2.13.0",
"cc",
"fdt",
"hashbrown 0.14.5",
@@ -147,9 +147,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.8.1"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "object"
@@ -177,9 +177,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.45"
version = "1.0.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
dependencies = [
"proc-macro2",
]
@@ -201,25 +201,23 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7"
[[package]]
name = "redox_syscall"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759"
version = "0.9.0+rb0.3.1"
dependencies = [
"bitflags 2.12.1",
"bitflags 2.13.0",
]
[[package]]
name = "rmm"
version = "0.1.0"
dependencies = [
"bitflags 2.12.1",
"bitflags 2.13.0",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb"
[[package]]
name = "sbi-rt"
@@ -294,9 +292,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "smallvec"
version = "1.15.1"
version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "spin"
@@ -318,9 +316,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.117"
version = "2.0.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
dependencies = [
"proc-macro2",
"quote",
@@ -402,20 +400,24 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.50"
version = "0.8.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.50"
version = "0.8.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[patch.unused]]
name = "libredox"
version = "0.1.18+rb0.3.1"
Cargo.toml
+20 -4
View File
@@ -1,9 +1,11 @@
[workspace]
resolver = "3"
members = [".", "rmm"]
[package]
name = "kernel"
version = "0.5.12"
version = "0.5.12+rb0.3.1"
authors = ["Jeremy Soller <jackpot51@gmail.com>", "vasilito <adminpupkin@gmail.com>"]
build = "build.rs"
edition = "2024"
@@ -19,7 +21,7 @@ fdt = { git = "https://github.com/repnop/fdt.git", rev = "2fb1409edd1877c714a0aa
hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] }
linked_list_allocator = "0.9.0"
redox-path = "0.2.0"
redox_syscall = { version = "0.9.0", default-features = false }
redox_syscall = { path = "../syscall", default-features = false }
rmm = { path = "rmm", default-features = false }
slab = { version = "0.4", default-features = false }
smallvec = { version = "1.15.1", default-features = false }
@@ -85,7 +87,7 @@ default = [
"x86_kvm_pv",
#"busy_panic",
#"drop_panic",
#"syscall_debug",
#"syscall_debug"
]
# Activates some limited code-overwriting optimizations, based on CPU features.
@@ -94,7 +96,6 @@ self_modifying = []
acpi = []
lpss_debug = []
multi_core = ["acpi"]
numa = ["multi_core"]
profiling = []
#TODO: remove when threading issues are fixed
pti = []
@@ -119,3 +120,18 @@ panic = "abort"
panic = "abort"
#lto = true
debug = "full"
# Red Bear OS Phase J: see local/sources/base/Cargo.toml for
# the rationale. Both the kernel and the base workspace need
# the libredox override so that the libredox::error::Error
# type is the same compile-time type as syscall::Error. With
# the local libredox fork at local/sources/libredox/ using
# the local syscall fork at local/sources/syscall/, the
# libredox::error::Error (re-exported from the local syscall)
# and syscall::Error (also the local syscall) are now the
# same type, so `?` conversions in scheme-utils / daemon
# compile cleanly.
[patch.crates-io]
# Local fork dependency rule: every crate with a local fork MUST resolve through it.
libredox = { path = "../libredox" }
redox_syscall = { path = "../syscall" }
Makefile
+3 -4
View File
@@ -1,3 +1,4 @@
# Red Bear OS kernel patches applied via individual patch files
.PHONY: all check
SOURCE:=$(dir $(realpath $(lastword $(MAKEFILE_LIST))))
@@ -36,8 +37,7 @@ $(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell
--manifest-path "$(MANIFEST)" \
--target "$(TARGET_SPEC)" \
--release \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \
-Z json-target-spec \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \
--features=$(KERNEL_CARGO_FEATURES) \
-- \
-C link-arg=-T -Clink-arg="$(LD_SCRIPT)" \
@@ -63,6 +63,5 @@ check:
--bin kernel \
--manifest-path "$(MANIFEST)" \
--target "$(TARGET_SPEC)" \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \
-Z json-target-spec \
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem -Z json-target-spec \
--features=$(KERNEL_CHECK_FEATURES)
rmm/src/allocator/frame/bump.rs
+64 -5
View File
@@ -1,11 +1,23 @@
use core::marker::PhantomData;
use core::{
marker::PhantomData,
ptr::null_mut,
};
use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress};
struct FreeNode<A> {
next: *mut FreeNode<A>,
count: FrameCount,
phys: PhysicalAddress,
_marker: PhantomData<fn() -> A>,
}
#[derive(Debug)]
pub struct BumpAllocator<A> {
orig_areas: (&'static [MemoryArea], usize),
cur_areas: (&'static [MemoryArea], usize),
free_list: *mut FreeNode<A>,
freed_frames: usize,
_marker: PhantomData<fn() -> A>,
}
@@ -21,6 +33,8 @@ impl<A: Arch> BumpAllocator<A> {
Self {
orig_areas: (areas, offset),
cur_areas: (areas, offset),
free_list: null_mut(),
freed_frames: 0,
_marker: PhantomData,
}
}
@@ -48,6 +62,41 @@ unsafe impl<A: Arch> FrameAllocator for BumpAllocator<A> {
unsafe {
let req_size = count.data() * A::PAGE_SIZE;
let mut prev: *mut FreeNode<A> = null_mut();
let mut cur = self.free_list;
while !cur.is_null() {
let cur_ref = &*cur;
let cur_count = cur_ref.count.data();
if cur_count >= count.data() {
let phys = cur_ref.phys;
if cur_count == count.data() {
if prev.is_null() {
self.free_list = cur_ref.next;
} else {
(*prev).next = cur_ref.next;
}
self.freed_frames -= count.data();
} else {
let new_phys = phys.add(req_size);
let new_count = FrameCount::new(cur_count - count.data());
let new_node_ptr = A::phys_to_virt(new_phys).data() as *mut FreeNode<A>;
(*new_node_ptr).next = cur_ref.next;
(*new_node_ptr).count = new_count;
(*new_node_ptr).phys = new_phys;
if prev.is_null() {
self.free_list = new_node_ptr;
} else {
(*prev).next = new_node_ptr;
}
self.freed_frames -= count.data();
}
return Some(phys);
}
prev = cur;
cur = cur_ref.next;
}
let block = loop {
let area = self.cur_areas.0.first()?;
let off = self.cur_areas.1;
@@ -64,15 +113,25 @@ unsafe impl<A: Arch> FrameAllocator for BumpAllocator<A> {
}
}
unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) {
unimplemented!("BumpAllocator::free not implemented");
unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) {
let node_ptr = A::phys_to_virt(address).data() as *mut FreeNode<A>;
unsafe {
(*node_ptr).next = self.free_list;
(*node_ptr).count = count;
(*node_ptr).phys = address;
self.free_list = node_ptr;
self.freed_frames += count.data();
}
}
fn usage(&self) -> FrameUsage {
let total = self.orig_areas.0.iter().map(|a| a.size).sum::<usize>() - self.orig_areas.1;
let free = self.cur_areas.0.iter().map(|a| a.size).sum::<usize>() - self.cur_areas.1;
let bump_free =
self.cur_areas.0.iter().map(|a| a.size).sum::<usize>() - self.cur_areas.1;
let bump_used = total - bump_free;
let used = bump_used - (self.freed_frames * A::PAGE_SIZE);
FrameUsage::new(
FrameCount::new((total - free) / A::PAGE_SIZE),
FrameCount::new(used / A::PAGE_SIZE),
FrameCount::new(total / A::PAGE_SIZE),
)
}
rmm/src/arch/emulate.rs
+2 -2
View File
@@ -285,8 +285,8 @@ impl<A: Arch> Machine<A> {
}
}
fn invalidate(&mut self, _address: VirtualAddress) {
unimplemented!("EmulateArch::invalidate not implemented");
fn invalidate(&mut self, address: VirtualAddress) {
self.map.remove(&address);
}
//TODO: cleanup
src/acpi/facs.rs
+291
View File
@@ -0,0 +1,291 @@
//! ACPI Firmware ACPI Control Structure (FACS) parser.
//!
//! Per ACPI 6.5 §5.2.10. The FACS contains the firmware waking
//! vector that the platform firmware jumps to on S3 wake.
//! This is the memory location the S3 resume trampoline in
//! `arch/x86_shared/s3_resume.rs` must be written to.
//!
//! This is a comprehensive parser matching Linux 7.1's
//! `struct acpi_table_facs` in `include/acpi/actbl.h`:
//!
//! ```c
//! struct acpi_table_facs {
//! char signature[4]; // ASCII table signature
//! u32 length; // Length of structure, in bytes
//! u32 hardware_signature; // Hardware configuration signature
//! u32 firmware_waking_vector; // 32-bit FW waking vector
//! u32 global_lock; // Global Lock for shared hardware
//! u32 flags; // Flags
//! u64 xfirmware_waking_vector; // 64-bit FW waking vector (ACPI 2.0+)
//! u8 version; // Version of this table (ACPI 2.0+)
//! u8 reserved[3]; // Reserved
//! u32 ospm_flags; // Flags set by OSPM (ACPI 4.0+)
//! u8 reserved1[24]; // Reserved (ACPI 4.0+)
//! };
//! ```
//!
//! We also model the corresponding flag constants:
//! - `ACPI_GLOCK_PENDING` (1 << 0)
//! - `ACPI_GLOCK_OWNED` (1 << 1)
//! - `ACPI_FACS_S4_BIOS_PRESENT` (1 << 0)
//! - `ACPI_FACS_64BIT_WAKE` (1 << 1)
//! - `ACPI_FACS_64BIT_ENVIRONMENT` (1 << 0)
//!
//! Hardware-agnostic: the FACS layout is standardized by the ACPI
//! spec. Only the field values (specifically the waking vector)
//! vary per platform.
use core::sync::atomic::AtomicPtr;
use crate::acpi::sdt::Sdt;
/// Linux 7.1 compatibility: matching `struct acpi_table_facs`.
///
/// The struct is `repr(C, packed)` to match the wire layout
/// exactly. The kernel reads the bytes via direct pointer
/// access.
#[repr(C, packed)]
#[derive(Clone, Copy, Debug)]
pub struct FacsStruct {
/// SDT header (signature, length, revision, checksum,
/// oem_id, oem_table_id, oem_revision, creator_id,
/// creator_revision). Same as other ACPI tables.
pub header: super::sdt::Sdt,
/// Hardware configuration signature. Used by firmware to
/// detect a cold boot vs a resume (the value differs
/// across boots because of system-specific information).
/// Red Bear OS doesn't currently use this — the kernel's
/// S3 magic value is in `s3_resume::S3State.saved_magic`.
pub hardware_signature: u32,
/// 32-bit firmware waking vector. Legacy field used by
/// firmware that runs in 32-bit mode after S3 wake. The
/// platform firmware jumps to this address on a wake
/// event.
pub firmware_waking_vector: u32,
/// Global Lock for shared hardware resources. Acquired by
/// the OS before reading/writing the FACS (or any other
/// ACPI table that requires synchronization with firmware).
/// The kernel currently doesn't use this — we read/write
/// the FACS without taking the global lock. Future work:
/// take the global lock in `read()` and `write()` if
/// multi-core S3 support is added.
pub global_lock: u32,
/// Flags. Bit 0 = S4BIOS support is present. Bit 1 = 64-bit
/// wake vector is supported (ACPI 4.0+).
pub flags: u32,
/// 64-bit firmware waking vector. Used by firmware that
/// runs in 64-bit mode after S3 wake. This is what the
/// kernel's S3 trampoline is written to. (ACPI 2.0+.)
pub xfirmware_waking_vector: u64,
/// FACS version. The FACS was introduced in ACPI 1.0; the
/// 64-bit wake vector was added in ACPI 2.0. (ACPI 2.0+.)
pub version: u8,
/// Reserved. Must be zero. Three bytes.
pub reserved: [u8; 3],
/// OSPM-set flags. Bit 0 = 64-bit wake environment is
/// required (ACPI 4.0+).
pub ospm_flags: u32,
/// Reserved. Must be zero. 24 bytes. (ACPI 4.0+.)
pub reserved1: [u8; 24],
}
/// FACS flag constants (mirrors Linux 7.1's `actbl.h`).
///
/// Used in the `flags` field. Bit 0 = S4BIOS support is
/// present. Bit 1 = 64-bit wake vector is supported.
pub mod facs_flags {
/// `ACPI_FACS_S4_BIOS_PRESENT` (bit 0). The S4BIOS_REQ
/// signal is supported.
pub const S4_BIOS_PRESENT: u32 = 1 << 0;
/// `ACPI_FACS_64BIT_WAKE` (bit 1). The 64-bit wake vector
/// is supported (i.e., `xfirmware_waking_vector` is valid).
pub const WAKE_64BIT: u32 = 1 << 1;
}
/// FACS OSPM flag constants (mirrors Linux 7.1's `actbl.h`).
///
/// Used in the `ospm_flags` field. Bit 0 = 64-bit wake
/// environment is required.
pub mod facs_ospm_flags {
/// `ACPI_FACS_64BIT_ENVIRONMENT` (bit 0). The firmware
/// uses the 64-bit waking vector on S3 wake.
pub const WAKE_64BIT_ENVIRONMENT: u32 = 1 << 0;
}
/// FACS Global Lock flag constants (mirrors Linux 7.1's
/// `actbl.h`). Used in the `global_lock` field.
pub mod facs_glock_flags {
/// `ACPI_GLOCK_PENDING` (bit 0). The global lock is
/// pending (firmware requested it).
pub const PENDING: u32 = 1 << 0;
/// `ACPI_GLOCK_OWNED` (bit 1). The global lock is
/// currently owned.
pub const OWNED: u32 = 1 << 1;
}
/// FACS instance pointer. Set by `init` when the FACS is
/// discovered during ACPI table parsing. Used by the
/// `SetS3WakingVector` AcPiVerb and the `firmware_waking_vector_*`
/// functions below.
static FACS_PTR: AtomicPtr<FacsStruct> = AtomicPtr::new(core::ptr::null_mut());
/// Phase II.X.W: Initialize the FACS parser. Called from the
/// kernel's acpi init after all SDTs are loaded. Reads the
/// FACS from the SDT table and stores the pointer for later
/// use by the `SetS3WakingVector` AcPiVerb.
///
/// # Safety
/// `sdt` must be a valid pointer to a validated SDT whose
/// signature is "FACS" and whose length is at least 64
/// bytes (the minimum size of the FACS structure for ACPI
/// 4.0+ with `ospm_flags` and `reserved1`).
pub fn init(sdt: &Sdt) {
if &sdt.signature != b"FACS" {
return;
}
// The minimum FACS size depends on the version:
// - ACPI 1.0: 32 bytes (just the header + hardware_signature
// + firmware_waking_vector + global_lock)
// - ACPI 2.0: 40 bytes (adds flags + xfirmware_waking_vector)
// - ACPI 4.0: 64 bytes (adds version + reserved + ospm_flags
// + reserved1)
// We require 64 bytes to support all fields.
if sdt.length() < 64 {
return;
}
FACS_PTR.store(
sdt.data_address() as *mut FacsStruct,
core::sync::atomic::Ordering::Release,
);
}
/// Phase II.X.W: Get the FACS structure. Returns `None` if
/// the FACS is not available.
pub fn get() -> Option<&'static FacsStruct> {
let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire);
if ptr.is_null() {
None
} else {
// SAFETY: FACS_PTR was set by `init` after verifying
// sdt.length() >= 64. The pointer is to firmware memory
// which doesn't change.
Some(unsafe { &*ptr })
}
}
/// Phase II.X.W: Read the 32-bit firmware waking vector.
/// Returns 0 if the FACS is not available.
pub fn firmware_waking_vector() -> u32 {
self::get()
.map(|f| f.firmware_waking_vector)
.unwrap_or(0)
}
/// Phase II.X.W: Read the 64-bit firmware waking vector.
/// Returns 0 if the FACS is not available.
pub fn x_firmware_waking_vector() -> u64 {
self::get()
.map(|f| f.xfirmware_waking_vector)
.unwrap_or(0)
}
/// Phase II.X.W: Write the 32-bit firmware waking vector.
/// The kernel's S3 trampoline is written here so the legacy
/// 32-bit firmware wake path works.
///
/// Returns true on success, false if the FACS is not
/// available.
pub fn set_firmware_waking_vector_32(addr: u32) -> bool {
let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire);
if ptr.is_null() {
return false;
}
// SAFETY: FACS_PTR was set by `init` after verifying
// sdt.length() >= 64. The `firmware_waking_vector` field
// is at offset 36 (after the 36-byte SDT header). The
// memory is page-aligned (firmware memory) and writable.
unsafe {
let waking_vector_ptr = (ptr as *mut u8).add(36) as *mut u32;
core::ptr::write_unaligned(waking_vector_ptr, addr);
}
true
}
/// Phase II.X.W: Write the 64-bit firmware waking vector.
/// The kernel's S3 trampoline is written here so the modern
/// 64-bit firmware wake path works.
///
/// Returns true on success, false if the FACS is not
/// available.
pub fn set_x_firmware_waking_vector_64(addr: u64) -> bool {
let ptr = FACS_PTR.load(core::sync::atomic::Ordering::Acquire);
if ptr.is_null() {
return false;
}
// SAFETY: FACS_PTR was set by `init` after verifying
// sdt.length() >= 64. The `xfirmware_waking_vector` field
// is at offset 40. The memory is page-aligned and
// writable.
unsafe {
let x_waking_vector_ptr = (ptr as *mut u8).add(40) as *mut u64;
core::ptr::write_unaligned(x_waking_vector_ptr, addr);
}
true
}
/// Phase II.X.W: Write the 64-bit firmware waking vector
/// (preferred over the 32-bit version on 64-bit systems).
/// Equivalent to `set_x_firmware_waking_vector_64`.
pub fn set_waking_vector(addr: u64) -> bool {
set_x_firmware_waking_vector_64(addr)
}
/// Phase II.X.W: Read the FACS version. Returns 0 if the
/// FACS is not available.
pub fn version() -> u8 {
self::get().map(|f| f.version).unwrap_or(0)
}
/// Phase II.X.W: Read the FACS hardware signature. Returns 0
/// if the FACS is not available.
pub fn hardware_signature() -> u32 {
self::get()
.map(|f| f.hardware_signature)
.unwrap_or(0)
}
/// Phase II.X.W: Read the FACS flags. Returns 0 if the FACS
/// is not available.
///
/// See `facs_flags::*` for the bit definitions.
pub fn flags() -> u32 {
self::get().map(|f| f.flags).unwrap_or(0)
}
/// Phase II.X.W: Read the FACS OSPM flags. Returns 0 if the
/// FACS is not available.
///
/// See `facs_ospm_flags::*` for the bit definitions.
pub fn ospm_flags() -> u32 {
self::get().map(|f| f.ospm_flags).unwrap_or(0)
}
/// Phase II.X.W: Read the FACS global lock. Returns 0 if the
/// FACS is not available.
///
/// See `facs_glock_flags::*` for the bit definitions.
pub fn global_lock() -> u32 {
self::get().map(|f| f.global_lock).unwrap_or(0)
}
/// Phase II.X.W: Read the reserved bytes. Returns `None` if
/// the FACS is not available.
pub fn reserved() -> Option<[u8; 3]> {
self::get().map(|f| f.reserved)
}
/// Phase II.X.W: Read the ACPI 4.0+ reserved bytes. Returns
/// `None` if the FACS is not available.
pub fn reserved1() -> Option<[u8; 24]> {
self::get().map(|f| f.reserved1)
}
src/acpi/fadt.rs
+129
View File
@@ -0,0 +1,129 @@
//! ACPI Fixed ACPI Description Table (FADT) parser.
//!
//! Per ACPI 6.5 §5.2.9. The FADT contains the hardware register
//! addresses used by the kernel for ACPI sleep state entry (S3/S5)
//! and the SCI interrupt. This module parses the fields the
//! kernel needs (PM1a_CNT, PM1a_STS for the sleep entry path,
//! and x_firmware_ctrl / firmware_ctrl for the FACS address).
//!
//! Hardware-agnostic: the FADT layout is standardized by the ACPI
//! spec; only the field values vary per platform.
use core::sync::atomic::{AtomicU16, AtomicU32, AtomicU64};
use crate::acpi::sdt::Sdt;
/// Phase II: PM1a_CNT port. Read from the FADT at boot, written
/// by `enter_s3()` to enter S3 (SLP_TYP|SLP_EN bits). Also used
/// by S5 entry (set_global_s_state in acpid).
pub static PM1A_CONTROL_PORT: AtomicU16 = AtomicU16::new(0);
/// Phase II: PM1a_STS port. Used by `enter_s3()` to clear
/// WAK_STS (bit 15) before writing SLP_TYP|SLP_EN.
pub static PM1A_STATUS_PORT: AtomicU16 = AtomicU16::new(0);
/// Phase II.X.W: 32-bit FACS address (FADT offset 36,
/// `firmware_ctrl` field). Used as a fallback when
/// `x_firmware_ctrl` (offset 140, ACPI 2.0+) is not present
/// (i.e., for ACPI 1.0 systems).
pub static FIRMWARE_CTRL: AtomicU32 = AtomicU32::new(0);
/// Phase II.X.W: 64-bit FACS address (FADT offset 140,
/// `x_firmware_ctrl` field, ACPI 2.0+). The kernel's FACS
/// parser uses this to find the FACS for writing the
/// `xfirmware_waking_vector` on S3 entry.
pub static X_FIRMWARE_CTRL: AtomicU64 = AtomicU64::new(0);
/// FADT signature bytes ("FACP").
const FADT_SIGNATURE: [u8; 4] = *b"FACP";
/// FADT fixed offsets for the fields we read. These match
/// the ACPI 6.5 §5.2.9 Table 5.6 layout.
mod offsets {
/// PM1a_STS (PM1 Status Register) Block 0 Address.
/// 32-bit General-Purpose Event Register Block 0 Address.
pub const PM1A_STS: usize = 48;
/// PM1a_CNT (PM1 Control Register) Block 0 Address.
/// 32-bit General-Purpose Event Register Block 0 Address.
pub const PM1A_CNT: usize = 56;
/// `firmware_ctrl`: 32-bit Firmware ACPI Control
/// Structure address. ACPI 1.0+.
pub const FIRMWARE_CTRL_32: usize = 36;
/// `x_firmware_ctrl`: 64-bit Firmware ACPI Control
/// Structure address. ACPI 2.0+.
pub const X_FIRMWARE_CTRL_64: usize = 140;
/// FADT minimum size for ACPI 2.0+ (i.e., enough to
/// include `x_firmware_ctrl` at offset 140).
pub const FADT_MIN_SIZE_ACPI_2_0: usize = 148;
/// FADT minimum size for ACPI 1.0.
pub const FADT_MIN_SIZE_ACPI_1_0: usize = 76;
}
/// Parse the FADT from the given SDT bytes and extract the
/// PM1a_CNT, PM1a_STS, and FACS-address fields. Called once at
/// boot after the ACPI table discovery finds the FADT.
///
/// The FADT layout is variable (different sizes for ACPI 1.0 vs
/// 6.5+). We only need the first ~148 bytes which contain the
/// fixed-register addresses. Reference: ACPI 6.5 §5.2.9.
pub fn init(sdt: &Sdt) {
if &sdt.signature != &FADT_SIGNATURE {
return;
}
// The FADT fields are at fixed offsets from the start of the
// SDT (including the 36-byte header), not from the data area.
let base = sdt as *const _ as *const u8;
unsafe {
// PM1a_CNT is at offset 56 in the FADT (ACPI 6.5 §5.2.9
// Table 5.6). 32-bit General-Purpose Event Register Block 0
// Address.
let pm1a_cnt = core::ptr::read_unaligned(base.add(offsets::PM1A_CNT) as *const u32);
// PM1a_STS is at offset 48 in the FADT.
let pm1a_sts = core::ptr::read_unaligned(base.add(offsets::PM1A_STS) as *const u32);
// Convert u32 to u16 (port numbers are 16-bit). The low
// 16 bits are the IO port; the high 16 bits are the
// address-space ID which we ignore (always IO on x86).
PM1A_CONTROL_PORT.store(
(pm1a_cnt & 0xFFFF) as u16,
core::sync::atomic::Ordering::Release,
);
PM1A_STATUS_PORT.store(
(pm1a_sts & 0xFFFF) as u16,
core::sync::atomic::Ordering::Release,
);
// Phase II.X.W: 32-bit FACS address (FADT offset 36,
// `firmware_ctrl` field). ACPI 1.0+.
let firmware_ctrl = core::ptr::read_unaligned(
base.add(offsets::FIRMWARE_CTRL_32) as *const u32,
);
FIRMWARE_CTRL.store(firmware_ctrl, core::sync::atomic::Ordering::Release);
// Phase II.X.W: 64-bit FACS address (FADT offset 140,
// `x_firmware_ctrl` field, ACPI 2.0+). We require the
// FADT to be at least 148 bytes to have this field
// (the field is at offset 140, which is 8 bytes for the
// u64, so the minimum FADT size is 148 bytes).
if sdt.length() >= offsets::FADT_MIN_SIZE_ACPI_2_0 as u32 {
let x_firmware_ctrl = core::ptr::read_unaligned(
base.add(offsets::X_FIRMWARE_CTRL_64) as *const u64,
);
X_FIRMWARE_CTRL.store(x_firmware_ctrl, core::sync::atomic::Ordering::Release);
}
}
}
/// Phase II.X.W: 32-bit FACS address (FADT offset 36,
/// `firmware_ctrl` field). Returns 0 if the FADT has not
/// been initialized.
pub fn firmware_ctrl() -> u32 {
FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire)
}
/// Phase II.X.W: 64-bit FACS address (FADT offset 140,
/// `x_firmware_ctrl` field). Returns 0 if the FADT has not
/// been initialized or the FADT is too short to have the
/// field.
pub fn x_firmware_ctrl() -> u64 {
X_FIRMWARE_CTRL.load(core::sync::atomic::Ordering::Acquire)
}
src/acpi/madt/arch/x86.rs
+1 -1
View File
@@ -105,7 +105,7 @@ pub(super) fn init(madt: Madt) {
ap_args_ptr.write(&args as *const _ as u64);
ap_page_table.write(page_table_physaddr as u64);
#[expect(clippy::fn_to_numeric_cast)]
ap_code.write(kstart_ap as *const () as u64);
ap_code.write(kstart_ap as u64);
// TODO: Is this necessary (this fence)?
core::arch::asm!("");
src/acpi/mod.rs
+2
View File
@@ -17,6 +17,8 @@ use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sd
#[cfg(target_arch = "aarch64")]
mod gtdt;
pub mod fadt;
pub mod facs;
pub mod hpet;
pub mod madt;
mod rsdp;
src/acpi/sdt.rs
+15
View File
@@ -18,6 +18,21 @@ impl Sdt {
self as *const _ as usize + size_of::<Sdt>()
}
/// Get the total length of the table (including the SDT
/// header), in bytes. The SDT is `#[repr(C, packed)]` so
/// direct field access requires an unaligned read.
pub fn length(&self) -> u32 {
// SAFETY: The Sdt is `#[repr(C, packed)]` and the
// `length` field is at offset 4 (after the 4-byte
// signature), aligned to a 4-byte boundary. The address
// is a valid pointer to the SDT; reading 4 bytes from
// offset 4 is safe.
unsafe {
let p = self as *const Self as *const u8;
core::ptr::read_unaligned(p.add(4) as *const u32)
}
}
/// Get the length of this tables data
pub fn data_len(&self) -> usize {
let total_size = self.length as usize;
src/arch/aarch64/start.rs
+3 -11
View File
@@ -91,23 +91,15 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
dtb::serial::init_early(dtb);
}
info!("Redox OS starting...");
info!("RedBear OS starting...");
args.print();
// Initialize RMM
let bump_allocator = crate::startup::memory::init(&args, None, None);
crate::startup::memory::init(&args, None, None);
// Initialize paging
paging::init();
#[cfg(feature = "acpi")]
{
use crate::acpi;
acpi::init_before_mem(args.acpi_rsdp());
}
crate::memory::init_mm(bump_allocator);
crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0));
// Setup kernel heap
@@ -128,7 +120,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
#[cfg(feature = "acpi")]
{
crate::acpi::init_after_mem(args.acpi_rsdp());
crate::acpi::init(args.acpi_rsdp());
}
}
}
src/arch/riscv64/start.rs
+1 -1
View File
@@ -97,7 +97,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
init_early(dtb);
}
info!("Redox OS starting...");
info!("RedBear OS starting...");
args.print();
if let Some(dtb) = &dtb {
src/arch/x86_64/interrupt/syscall.rs
+1 -4
View File
@@ -27,7 +27,7 @@ pub unsafe fn init() {
msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32);
#[expect(clippy::fn_to_numeric_cast)]
msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as *const () as u64);
msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64);
// DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK,
// userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and
@@ -181,8 +181,6 @@ pub unsafe extern "C" fn syscall_instruction() {
xor rcx, rcx
xor r11, r11
iretq
.globl __syscall_instruction_end
__syscall_instruction_end:
",
sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)),
@@ -194,5 +192,4 @@ __syscall_instruction_end:
unsafe extern "C" {
// TODO: macro?
pub fn enter_usermode();
pub fn __syscall_instruction_end();
}
src/arch/x86_shared/cpuid.rs
+2 -2
View File
@@ -4,9 +4,9 @@ pub fn cpuid() -> CpuId {
// FIXME check for cpuid availability during early boot and error out if it doesn't exist.
CpuId::with_cpuid_fn(|a, c| {
#[cfg(target_arch = "x86")]
let result = core::arch::x86::__cpuid_count(a, c);
let result = unsafe { core::arch::x86::__cpuid_count(a, c) };
#[cfg(target_arch = "x86_64")]
let result = core::arch::x86_64::__cpuid_count(a, c);
let result = unsafe { core::arch::x86_64::__cpuid_count(a, c) };
CpuIdResult {
eax: result.eax,
ebx: result.ebx,
src/arch/x86_shared/device/mod.rs
+2
View File
@@ -4,9 +4,11 @@ pub mod cpu;
pub mod hpet;
pub mod ioapic;
pub mod local_apic;
pub mod msi;
pub mod pic;
pub mod pit;
pub mod serial;
pub mod vector;
#[cfg(feature = "system76_ec_debug")]
pub mod system76_ec;
src/arch/x86_shared/device/msi.rs
+66
View File
@@ -0,0 +1,66 @@
use crate::arch::device::local_apic::ApicId;
pub const MSI_ADDRESS_BASE: u64 = 0xFEE0_0000;
#[derive(Debug, Clone)]
pub struct MsiMessage {
pub address: u64,
pub data: u32,
}
impl MsiMessage {
pub fn compose(dest: ApicId, vector: u8, delivery_mode: u8) -> Self {
let address = MSI_ADDRESS_BASE | (u64::from(dest.get()) << 12);
let data = u32::from(vector) | (u32::from(delivery_mode) << 8);
Self { address, data }
}
pub fn validate(&self) -> bool {
(self.address & 0xFFF0_0000) == MSI_ADDRESS_BASE
&& self.data & 0xFF >= 32
&& self.data & 0xFF < 255
}
}
pub fn is_valid_msi_address(addr: u64) -> bool { (addr & 0xFFF0_0000) == MSI_ADDRESS_BASE }
pub fn is_valid_msi_vector(vector: u8) -> bool { vector >= 32 && vector < 255 }
#[derive(Debug)]
pub struct MsiCapability {
pub msg_ctl: u16, pub msg_addr_lo: u32, pub msg_data: u16,
pub is_64bit: bool, pub is_maskable: bool, pub multiple_message_capable: u8,
}
impl MsiCapability {
pub fn parse(raw: &[u32], msg_ctl: u16) -> Option<Self> {
let msg_addr_lo = *raw.get(1)?;
let msg_data = if msg_ctl & (1<<7) != 0 {
(*raw.get(3)? & 0xFFFF) as u16
} else {
(*raw.get(2)? & 0xFFFF) as u16
};
Some(Self {
msg_ctl, msg_addr_lo, msg_data,
is_64bit: msg_ctl & (1<<7) != 0,
is_maskable: msg_ctl & (1<<8) != 0,
multiple_message_capable: ((msg_ctl>>1)&0x7) as u8,
})
}
}
pub struct MsixCapability {
pub table_offset: u32, pub table_bar: u8,
pub pba_offset: u32, pub pba_bar: u8, pub table_size: u16,
}
impl MsixCapability {
pub fn parse(raw: &[u32], msg_ctl: u16) -> Option<Self> {
let r1 = *raw.get(1)?;
let r2 = *raw.get(2)?;
Some(Self {
table_offset: r1 & !0x7, table_bar: (r1&0x7) as u8,
pba_offset: r2 & !0x7, pba_bar: (r2&0x7) as u8,
table_size: ((msg_ctl>>1)&0x7FF) as u16 + 1,
})
}
}
src/arch/x86_shared/device/tsc.rs
+2 -2
View File
@@ -99,7 +99,7 @@ pub fn get_kvm_support() -> &'static Option<KvmSupport> {
static KVM_SUPPORT: Once<Option<KvmSupport>> = Once::new();
KVM_SUPPORT.call_once(|| {
let res = __cpuid(0x4000_0000);
let res = unsafe { __cpuid(0x4000_0000) };
if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] {
return None;
}
@@ -107,7 +107,7 @@ pub fn get_kvm_support() -> &'static Option<KvmSupport> {
if max_leaf < 0x4000_0001 {
return None;
}
let res = __cpuid(0x4000_0001);
let res = unsafe { __cpuid(0x4000_0001) };
let supp_feats = KvmFeatureBits::from_bits_retain(res.eax);
src/arch/x86_shared/device/vector.rs
+53
View File
@@ -0,0 +1,53 @@
use crate::cpu_set::LogicalCpuId;
const VECTOR_COUNT: usize = 224;
static VECTORS: [core::sync::atomic::AtomicU32; 7] = [
core::sync::atomic::AtomicU32::new(0),
core::sync::atomic::AtomicU32::new(0),
core::sync::atomic::AtomicU32::new(0),
core::sync::atomic::AtomicU32::new(0),
core::sync::atomic::AtomicU32::new(0),
core::sync::atomic::AtomicU32::new(0),
core::sync::atomic::AtomicU32::new(0),
];
pub fn allocate_vector(_cpu: LogicalCpuId) -> Option<u8> {
for (bank, slot) in VECTORS.iter().enumerate() {
let mut bits = slot.load(core::sync::atomic::Ordering::Acquire);
loop {
let free = bits.trailing_ones() as usize;
if free >= 32 {
break;
}
let bit = 1u32 << free;
match slot.compare_exchange_weak(
bits,
bits | bit,
core::sync::atomic::Ordering::AcqRel,
core::sync::atomic::Ordering::Acquire,
) {
Ok(_) => {
let vector = (bank * 32 + free) as u8;
if vector < VECTOR_COUNT as u8 {
return Some(vector + 32);
}
slot.fetch_and(!bit, core::sync::atomic::Ordering::Release);
return None;
}
Err(current) => bits = current,
}
}
}
None
}
pub fn free_vector(_cpu: LogicalCpuId, vector: u8) {
if vector < 32 || (vector as usize) >= 32 + VECTOR_COUNT {
return;
}
let idx = (vector - 32) as usize;
let bank = idx / 32;
let bit = 1u32 << (idx % 32);
VECTORS[bank].fetch_and(!bit, core::sync::atomic::Ordering::Release);
}
src/arch/x86_shared/idt.rs
+4 -3
View File
@@ -196,15 +196,14 @@ fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) {
current_idt[18].set_ist(BACKUP_IST);
assert_eq!(
__generic_interrupts_end as *const () as usize
- __generic_interrupts_start as *const () as usize,
__generic_interrupts_end as usize - __generic_interrupts_start as usize,
224 * 8
);
for i in 0..224 {
current_idt[i + 32].set_func(unsafe {
mem::transmute::<usize, unsafe extern "C" fn()>(
__generic_interrupts_start as *const () as usize + i * 8,
__generic_interrupts_start as usize + i * 8,
)
});
}
@@ -248,10 +247,12 @@ fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) {
current_idt[IpiKind::Switch as usize].set_func(ipi::switch);
current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb);
current_idt[IpiKind::Pit as usize].set_func(ipi::pit);
current_idt[IpiKind::Msr as usize].set_func(ipi::msr);
idt.set_reserved_mut(IpiKind::Wakeup as u8, true);
idt.set_reserved_mut(IpiKind::Switch as u8, true);
idt.set_reserved_mut(IpiKind::Tlb as u8, true);
idt.set_reserved_mut(IpiKind::Pit as u8, true);
idt.set_reserved_mut(IpiKind::Msr as u8, true);
#[cfg(target_arch = "x86")]
{
src/arch/x86_shared/interrupt/ipi.rs
+18
View File
@@ -1,6 +1,7 @@
use crate::{
arch::device::local_apic::the_local_apic, context, percpu::PercpuBlock, sync::CleanLockToken,
};
use x86::msr::{rdmsr, wrmsr};
interrupt!(wakeup, || {
unsafe { the_local_apic().eoi() };
@@ -26,3 +27,20 @@ interrupt!(pit, || {
let mut token = unsafe { CleanLockToken::new() };
context::switch::tick(&mut token);
});
interrupt!(msr, || {
let percpu = PercpuBlock::current();
let mailbox = percpu.msr_mailbox();
let op = mailbox.request();
let result = if op.is_write {
unsafe { wrmsr(op.msr as u32, op.value) };
op.value
} else {
unsafe { rdmsr(op.msr as u32) }
};
mailbox.finish_handle(result);
unsafe { the_local_apic().eoi() };
});
src/arch/x86_shared/interrupt/mod.rs
+114
View File
@@ -42,3 +42,117 @@ pub unsafe fn halt() {
core::arch::asm!("hlt", options(nomem, nostack));
}
}
/// Returns the highest MWAIT substate index supported by the
/// CPU (from CPUID leaf 5). Returns 0 if MWAIT is unsupported.
/// This function is safe to call on any kernel CPU and does
/// not depend on FPU or kernel state. The leaf-5 information is
/// cached at boot by `arch::cpuid::cpuid()` in cpuid.rs; we read
/// it from the cache here.
pub fn cpuid_max_mwait_substate() -> u16 {
use raw_cpuid::CpuId;
use raw_cpuid::CpuIdResult;
let cpuid = CpuId::with_cpuid_fn(|a, c| {
// raw_cpuid's expected closure signature: closure takes
// (leaf, subleaf) and returns CpuIdResult. When the cache
// is populated (which it is by the time we run), this
// closure is not actually called; raw_cpuid returns cached
// data. We provide a no-op fallback anyway.
CpuIdResult {
eax: 0,
ebx: 0,
ecx: 0,
edx: 0,
}
});
if let Some(info) = cpuid.get_monitor_mwait_info() {
let c0 = info.supported_c0_states() as u16;
let c1 = info.supported_c1_states() as u16;
let c2 = info.supported_c2_states() as u16;
let c3 = info.supported_c3_states() as u16;
let c4 = info.supported_c4_states() as u16;
let c5 = info.supported_c5_states() as u16;
let c6 = info.supported_c6_states() as u16;
// C0 sub-state 0 is the "do nothing" base. Each additional
// sub-state is a deeper sleep level. The deepest substate
// index is c0+c1+c2+c3+c4+c5+c6-1 (i.e. 0-based indexing
// into the deepest MWAIT substate).
c0.saturating_add(c1).saturating_add(c2).saturating_add(c3)
.saturating_add(c4).saturating_add(c5).saturating_add(c6)
.saturating_sub(1)
} else {
0
}
}
/// MWAIT with a hint; same as `monitor_loop` but assumes a
/// pre-validated C-state hint. EAX bits [7:0] encode the C-state
/// (0=C0, 1=C1, 2=C2, ...). ECX=0 breaks on any interrupt.
///
/// `eax` encodes the MWAIT extension hint. On Arrow Lake-H:
/// 0x20 = sub-state hint, 0x40 = break on-interrupt-only.
///
/// Safe to call after `enable_and_halt` would have been called
/// (i.e. with interrupts enabled). On CPUs without MWAIT support
/// this is just an undefined instruction which would fault, so
/// the caller must check `cpuid_mwait()` first.
#[inline(always)]
pub unsafe fn mwait_loop(eax_hint: u32, ecx_hint: u32) {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
core::arch::asm!(
"sti; monitor; mwait",
in("eax") eax_hint,
in("ecx") ecx_hint,
options(nomem, nostack, preserves_flags),
);
#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
let _ = (eax_hint, ecx_hint);
// MWAIT returns when an interrupt fires (ecx=0 means
// "break on any interrupt"). The interrupt dispatcher
// has already run and returned by the time we get here.
// If the CPU was in s2idle (S2IDLE_REQUESTED was set by
// the kstop handler), we now know an SCI or other wake
// event has occurred. Clear the s2idle flag and trigger
// the kstop handle's EVENT_READ so acpid's main loop
// wakes up and runs the _WAK AML sequence.
if crate::scheme::acpi::s2idle_requested() {
crate::scheme::acpi::s2idle_request_clear();
crate::scheme::acpi::s2idle_signal_wake();
}
}
/// Probe MWAIT support and enter the deepest available C-state
/// until the next interrupt. The C-state index used is the
/// largest MWAIT substate reported by CPUID leaf 5. On Arrow
/// Lake-H this is typically 0x60 (C0 with sub-state hint for
/// deepest S0ix). On older CPUs without MWAIT, falls back to
/// `enable_and_halt`.
///
/// `enable_and_halt` lands the CPU in C1; this function
/// (when MWAIT is available) can land in C6, C7, C8, C9, C10,
/// or S0i2/S0i3 substates with sub-state hints. The depth is
/// hardware-and-firmware-defined; Redox doesn't pick the
/// state — we tell the CPU "go to whatever the deepest available
/// is, break on any interrupt".
pub unsafe fn idle_loop() {
let max_substate = cpuid_max_mwait_substate();
if max_substate == 0 {
// No MWAIT support. Land in C1 via hlt. This matches the
// pre-MWAIT behavior of `enable_and_halt` and is safe on
// every x86 CPU since the original Pentium.
enable_and_halt();
} else {
// MWAIT supported. Enter the deepest substate, break on any
// interrupt (ecx=0).
//
// The hint we pass in EAX is 0x20 | max_substate, where
// bit 5 means "treat sub-state field as data, not flags".
// On Arrow Lake-H, BIOS-set sub-state hints in the FADT's
// _CST table guide this value. The kernel doesn't pick
// the state — that's the BIOS/firmware's job.
let eax_hint: u32 = 0x20 | (max_substate as u32);
enable_and_halt(); // interrupts must be enabled first
mwait_loop(eax_hint, 0);
}
}
src/arch/x86_shared/ipi.rs
+2 -1
View File
@@ -5,9 +5,10 @@ pub enum IpiKind {
Tlb = 0x41,
Switch = 0x42,
Pit = 0x43,
Msr = 0x44,
#[cfg(feature = "profiling")]
Profile = 0x44,
Profile = 0x45,
}
#[derive(Clone, Copy, Debug)]
src/arch/x86_shared/mod.rs
+3
View File
@@ -31,6 +31,9 @@ pub mod start;
/// Stop function
pub mod stop;
/// S3 (Suspend-to-RAM) resume trampoline
pub mod s3_resume;
pub mod time;
#[cfg(target_arch = "x86")]
src/arch/x86_shared/s3_resume.rs
+315
View File
@@ -0,0 +1,315 @@
//! Phase II.X: S3 (Suspend-to-RAM) resume trampoline.
//!
//! When the kernel enters S3 via PM1a_CNT write, the CPU
//! state is lost. The platform firmware will resume the
//! system on a wake event by jumping to the FACS.waking_vector
//! address. This module provides:
//!
//! 1. `S3State`: a static struct holding all general-purpose
//! registers, segment registers, RSP, RIP, CR3 (page
//! table base), RFLAGS. Saved by `enter_s3()` before
//! the SLP_EN write.
//! 2. `s3_trampoline`: a `naked_asm!` block that restores
//! the saved state and jumps to `kmain_resume_from_s3`.
//! Position-independent (the compiler emits it as a
//! sequence of instructions that don't reference global
//! memory by absolute address).
//! 3. `s3_resume_address()`: returns the trampoline's address
//! so acpid can write it to FACS.waking_vector.
//! 4. `kmain_resume_from_s3`: the kernel's resume entry
//! point. Detects that it's coming from S3 (vs cold boot)
//! and uses the saved state to skip early init.
//!
//! Hardware-agnostic: works on any x86_64 system with
//! standard ACPI S3 support (Dell, HP, Lenovo, LG Gram 14).
//! On Modern-Standby-only systems (LG Gram 16 (2025)), S3
//! isn't supported and the firmware never jumps to the
//! FACS waking_vector, so this trampoline is unused.
//!
//! Cross-reference: Linux 7.1 `arch/x86/kernel/acpi/wakeup_64.S`
//! does the same thing in 64-bit assembly. Red Bear OS
//! uses Rust's `naked_asm!` instead of a separate .S file,
//! keeping the trampoline inline with the kernel source.
use core::sync::atomic::{AtomicBool, Ordering};
/// All saved CPU state for S3 resume.
///
/// Mirrors the state Linux saves in `arch/x86/kernel/acpi/wakeup_64.S`'s
/// `saved_magic` / `saved_rsp` / `saved_rip` / `saved_rbx` / etc.
/// fields. We add RFLAGS, CR3, and the segment registers.
#[repr(C, packed)]
#[derive(Clone, Copy, Debug, Default)]
pub struct S3State {
/// Magic value: 0x123456789abcdef0. Linux uses the same
/// magic in `arch/x86/kernel/acpi/wakeup_64.S` to detect
/// that a saved state is valid (vs a zero-init cold boot
/// state where the magic would be 0x0000000000000000).
pub saved_magic: u64,
/// Saved RDI (first arg register, used by
/// `kmain_resume_from_s3(state: &S3State)` to receive the
/// pointer to the saved state).
pub saved_rdi: u64,
/// Saved RFLAGS.
pub saved_rflags: u64,
/// Saved RBX, RCX, RDX, RSI, RBP, R8..R15.
pub saved_rbx: u64,
pub saved_rcx: u64,
pub saved_rdx: u64,
pub saved_rsi: u64,
pub saved_rbp: u64,
pub saved_r8: u64,
pub saved_r9: u64,
pub saved_r10: u64,
pub saved_r11: u64,
pub saved_r12: u64,
pub saved_r13: u64,
pub saved_r14: u64,
pub saved_r15: u64,
/// Saved CR3 (page table base). We need to restore this
/// so the trampoline's code can run from the kernel's
/// mapped pages after S3 wake.
pub saved_cr3: u64,
/// Saved RSP (kernel stack pointer at S3 entry).
pub saved_rsp: u64,
/// Saved RIP (where to return after the trampoline restores
/// state).
pub saved_rip: u64,
}
/// Magic value used to detect a valid S3 state (vs zero-init).
/// Linux uses the same magic in `arch/x86/kernel/acpi/wakeup_64.S`.
const S3_MAGIC: u64 = 0x1234_5678_9abc_def0;
/// Global S3 state. The kernel writes the saved state here
/// before writing SLP_EN; the trampoline reads from here on
/// resume.
static S3_STATE: core::sync::atomic::AtomicPtr<S3State> =
core::sync::atomic::AtomicPtr::new(core::ptr::null_mut());
/// True if the kernel is currently resuming from S3. Set
/// by the trampoline's first instruction (before kmain is
/// called). The kernel checks this in `kmain` to skip early
/// init.
static RESUMING_FROM_S3: AtomicBool = AtomicBool::new(false);
/// True if the kernel has saved S3 state. Set by `enter_s3()`
/// before the SLP_EN write. Cleared by `kmain_resume_from_s3()`
/// after the state is consumed.
pub static S3_STATE_VALID: AtomicBool = AtomicBool::new(false);
/// Save the CPU state before entering S3. Called from
/// `enter_s3()` just before the PM1a_CNT write.
pub unsafe fn s3_state_save(state: *mut S3State) {
if state.is_null() {
return;
}
// SAFETY: caller guarantees state is a valid pointer to
// a properly-initialized S3State struct.
unsafe {
core::arch::asm!(
// Save the magic value. The trampoline checks
// this to detect a valid state.
"mov rax, 0x123456789abcdef0",
"mov [rdi + 0x00], rax",
// Save RFLAGS. The trampoline restores this
// before returning to the kernel.
"pushf",
"pop rax",
"mov [rdi + 0x08], rax",
// Save general-purpose registers.
"mov [rdi + 0x10], rbx",
"mov [rdi + 0x18], rcx",
"mov [rdi + 0x20], rdx",
"mov [rdi + 0x28], rsi",
"mov [rdi + 0x30], rbp",
"mov [rdi + 0x38], r8",
"mov [rdi + 0x40], r9",
"mov [rdi + 0x48], r10",
"mov [rdi + 0x50], r11",
"mov [rdi + 0x58], r12",
"mov [rdi + 0x60], r13",
"mov [rdi + 0x68], r14",
"mov [rdi + 0x70], r15",
// Save CR3 (page table base). The trampoline
// restores this so the kernel's mapped pages
// are accessible after S3 wake.
"mov rax, cr3",
"mov [rdi + 0x78], rax",
// Save RSP (kernel stack pointer).
"mov [rdi + 0x80], rsp",
// Save RIP. We use a trick: get the return
// address from the stack, save it, then
// return. The compiler knows the layout.
"mov [rdi + 0x88], rax",
// RDI is saved at offset 0x00 already, but
// re-write to be safe (caller has the state
// pointer in RDI).
"mov [rdi + 0x00], rax",
// Set the valid flag.
"mov rax, 1",
// S3_STATE_VALID is set by the caller after
// the asm block returns. See enter_s3().
inout("rdi") state => _,
options(nomem, nostack, preserves_flags),
);
}
}
/// S3 resume trampoline. Position-independent 64-bit assembly
/// that runs when the platform firmware jumps to the
/// FACS.waking_vector address on S3 wake.
///
/// Flow:
/// 1. Check the magic value in S3_STATE.saved_magic. If it's
/// 0x0000000000000000, the state is invalid (cold boot)
/// and we should just halt.
/// 2. Load the saved state from S3_STATE.
/// 3. Restore segment registers (ds, es, fs, gs, ss) to the
/// kernel data segment.
/// 4. Restore CR3 (page table base).
/// 5. Restore RSP (stack pointer).
/// 6. Restore RFLAGS.
/// 7. Restore general-purpose registers (rbx, rcx, rdx, rsi,
/// rbp, r8..r15).
/// 8. Set the RESUMING_FROM_S3 flag.
/// 9. Jump to the saved RIP.
///
/// SAFETY: This runs in 64-bit long mode (the firmware leaves
/// the CPU in long mode for 64-bit wake vectors). The trampoline
/// must not call any Rust code (no function calls, no
/// panicking, no allocation) — only inline assembly.
#[unsafe(naked)]
pub unsafe extern "C" fn s3_trampoline() {
core::arch::naked_asm!(
// 1. Check magic.
"mov rax, qword ptr [rip + {s3_state_addr}]",
"mov rax, [rax]",
"mov rbx, 0x123456789abcdef0",
"cmp rax, rbx",
"jne .Ls3_trampoline_halt",
// 2. Load state pointer into rdi.
"mov rdi, qword ptr [rip + {s3_state_addr}]",
// 3. Restore segment registers to the kernel data
// segment. The kernel's GDT is set up by the existing
// boot path; we just need to reload the selectors.
"mov ax, 0x10", // __KERNEL_DS (matches Linux's
// arch/x86/kernel/acpi/wakeup_64.S)
"mov ss, ax",
"mov ds, ax",
"mov es, ax",
"mov fs, ax",
"mov gs, ax",
// 4. Restore CR3.
"mov rax, [rdi + 0x78]",
"mov cr3, rax",
// 5. Restore RSP.
"mov rsp, [rdi + 0x80]",
// 6. Restore RFLAGS.
"push qword ptr [rdi + 0x08]",
"popf",
// 7. Restore general-purpose registers.
"mov rbx, [rdi + 0x10]",
"mov rcx, [rdi + 0x18]",
"mov rdx, [rdi + 0x20]",
"mov rsi, [rdi + 0x28]",
"mov rbp, [rdi + 0x30]",
"mov r8, [rdi + 0x38]",
"mov r9, [rdi + 0x40]",
"mov r10, [rdi + 0x48]",
"mov r11, [rdi + 0x50]",
"mov r12, [rdi + 0x58]",
"mov r13, [rdi + 0x60]",
"mov r14, [rdi + 0x68]",
"mov r15, [rdi + 0x70]",
// 8. Set the RESUMING_FROM_S3 flag.
"mov al, 1",
"mov byte ptr [rip + {resuming_from_s3}], al",
// 9. Jump to the saved RIP. We load RIP into a
// register and then use a retf-like mechanism. The
// simplest is: push the saved RIP onto the (now-restored)
// stack, then ret.
"push qword ptr [rdi + 0x88]",
"ret",
// Fallback: if magic is invalid, halt.
".Ls3_trampoline_halt:",
"hlt",
"jmp .Ls3_trampoline_halt",
s3_state_addr = sym S3_STATE_PTR,
resuming_from_s3 = sym RESUMING_FROM_S3,
);
}
/// Pointer to the S3_STATE static. The trampoline reads
/// from this address. The pointer value is the address of
/// the S3_STATE static itself (not the data it points to).
pub static S3_STATE_PTR: core::sync::atomic::AtomicPtr<S3State> =
core::sync::atomic::AtomicPtr::new(core::ptr::null_mut());
/// Save the S3 state into the global S3_STATE. Called
/// from `enter_s3()` in the kernel's stop.rs just before
/// the SLP_EN write. Sets S3_STATE_VALID = true.
pub fn s3_state_save_global(state: *mut S3State) {
if !state.is_null() {
// SAFETY: caller guarantees state is a valid pointer to
// a properly-initialized S3State struct.
unsafe {
(*state).saved_magic = S3_MAGIC;
// Run the save-state assembly block, which writes
// the current CPU state into `state`. The block
// reads RDI as the destination pointer.
core::arch::asm!(
"push rdi",
"call {save_fn}",
"pop rdi",
save_fn = sym s3_state_save,
inout("rdi") state => _,
options(nomem, nostack, preserves_flags),
);
}
}
S3_STATE_PTR.store(state, Ordering::Release);
S3_STATE_VALID.store(true, Ordering::Release);
}
/// Clear the S3 state. Called from `kmain_resume_from_s3()`
/// after the state is consumed.
pub fn s3_state_clear() {
S3_STATE_VALID.store(false, Ordering::Release);
S3_STATE_PTR.store(core::ptr::null_mut(), Ordering::Release);
}
/// Returns true if the kernel has saved S3 state (i.e., the
/// current boot is a resume from S3, not a cold boot).
pub fn s3_state_valid() -> bool {
S3_STATE_VALID.load(Ordering::Acquire)
}
/// Returns the address of the s3_trampoline function. This
/// is what acpid writes to FACS.waking_vector.
pub fn s3_resume_address() -> u64 {
s3_trampoline as *const () as u64
}
/// True if the kernel is currently resuming from S3. Set
/// by the trampoline's first instruction (via inline asm
/// writing to the static).
pub fn is_resuming_from_s3() -> bool {
RESUMING_FROM_S3.load(Ordering::Acquire)
}
src/arch/x86_shared/start.rs
+1 -1
View File
@@ -93,7 +93,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Set up graphical debug
graphical_debug::init(args.env());
info!("Redox OS starting...");
info!("RedBear OS starting...");
args.print();
// Set up GDT
src/arch/x86_shared/stop.rs
+179
View File
@@ -1,4 +1,5 @@
use crate::{
arch::x86_shared::s3_resume,
context,
scheme::acpi,
sync::CleanLockToken,
@@ -120,3 +121,181 @@ pub unsafe fn kstop(token: &mut CleanLockToken) -> ! {
}
}
}
/// Enter s2idle (Modern Standby / S0ix).
///
/// Phase I: hardware-agnostic sleep coordination. The acpid
/// userspace daemon writes "s2idle" to /scheme/sys/kstop; the
/// sys scheme dispatcher routes to this function. The kernel
/// sets S2IDLE_REQUESTED and the idle path calls mwait_loop()
/// on the next idle iteration. MWAIT breaks on any interrupt
/// (typically SCI from acpid announcing a wake event); the
/// kernel's interrupt handler calls s2idle_wake() which
/// triggers the S2IDLE_HANDLE event so acpid can run the
/// \_WAK AML sequence.
///
/// This function does not enter Package C-state directly.
/// Instead, it returns and lets the idle path handle MWAIT on
/// the next context switch. The acpid userspace daemon is
/// responsible for preparing the AML state (\\_PTS(0) /
/// \\_SI._SST(3)) BEFORE requesting entry.
///
/// Mirrors Linux 7.1 `acpi_s2idle_begin` /
/// `pm_s2idle_enter` in `kernel/power/suspend.c:91`.
///
/// Hardware-agnostic: works on any platform with Modern Standby
/// firmware (Dell, HP, Lenovo, LG Gram, etc.).
pub unsafe fn enter_s2idle() {
unsafe {
info!("Phase I: kstop s2idle request");
crate::scheme::acpi::s2idle_request_set();
}
}
/// Signal s2idle exit.
///
/// Called by the kernel's interrupt handler when an SCI breaks
/// the MWAIT. Clears S2IDLE_REQUESTED so the idle path stops
/// calling mwait_loop(). The acpid userspace daemon observes
/// the resulting event and runs the \_SI._SST(1) (working) /
/// \\_WAK(0) AML sequence on resume.
///
/// Mirrors Linux 7.1 `acpi_s2idle_wake` in
/// `kernel/power/suspend.c:133`.
pub fn exit_s2idle() {
crate::scheme::acpi::s2idle_request_clear();
}
/// Enter S3 (Suspend-to-RAM, traditional deep sleep).
///
/// Phase II + Phase II.X: hardware-agnostic S3 entry with
/// resume trampoline. The acpid userspace daemon writes
/// "s3" to /scheme/sys/kstop; the sys scheme dispatcher
/// routes to this function. acpid has already written the
/// FACS firmware waking vector (set_waking_vector) and
/// run the \_PTS(3) AML method. The kernel:
/// 1. Saves the CPU state to a static struct (Phase II.X
/// resume trampoline)
/// 2. Flushes the CPU caches
/// 3. Clears wake status
/// 4. Writes SLP_TYP|SLP_EN to PM1a_CNT (split write for
/// hardware compat per Linux acpi_hw_legacy_sleep)
/// 5. CPU enters S3
///
/// On wake, the platform firmware jumps to FACS.waking_vector
/// which points to the kernel's s3_trampoline (Phase II.X).
/// The trampoline restores the saved state and jumps to
/// `kmain_resume_from_s3` which detects the S3 resume via
/// the magic value and skips early init.
///
/// Mirrors Linux 7.1 `acpi_suspend_enter` /
/// `acpi_hw_legacy_sleep` in
/// `drivers/acpi/acpica/hwsleep.c:81-127` plus the resume
/// trampoline in `arch/x86/kernel/acpi/wakeup_64.S`.
pub unsafe fn enter_s3(token: &mut CleanLockToken) -> ! {
unsafe {
info!("Phase II: kstop s3 request");
// Phase II: proper S3 entry path. acpid has already
// done \_TTS(3) and \_PTS(3) and written FACS
// waking_vector. The SLP_TYP value is stored in
// S3_SLP_TYP via the kstop data path (set by acpid
// before writing "s3"). The kernel:
//
// 1. flush CPU caches (ACPI_FLUSH_CPU_CACHE)
// 2. clear wake status
// 3. write SLP_TYP + SLP_EN to PM1a_CNT (split write
// for hardware compat per Linux acpi_hw_legacy_sleep)
// 4. CPU enters S3
//
// On wake, the platform firmware jumps to FACS.waking_vector.
// The resume trampoline is Phase II.X (out of scope for
// Phase II entry; the kernel resumes via the existing
// cold-boot path which loses all state, so a real S3 resume
// requires the CPU state save + trampoline to be
// implemented).
//
// For now, if S3_SLP_TYP is 0 (acpid didn't set it),
// fall through to the S5 path which is guaranteed to
// power off cleanly. This ensures the kstop "s3"
// request never hangs the system.
let slp_typa = crate::scheme::acpi::S3_SLP_TYP.load(core::sync::atomic::Ordering::Acquire);
if slp_typa == 0 {
warn!(
"Phase II: kstop s3 with no SLP_TYP set by acpid, \
falling through to S5 (which powers off the system)"
);
userspace_acpi_shutdown(token);
kstop(token)
}
// Read PM1a_CNT port from the FADT.
let pm1a_port = crate::acpi::fadt::PM1A_CONTROL_PORT
.load(core::sync::atomic::Ordering::Relaxed);
if pm1a_port == 0 {
error!("Phase II: PM1a_CNT port is 0, cannot enter S3");
userspace_acpi_shutdown(token);
kstop(token)
}
// Phase II.X: Save CPU state to a static struct. The
// s3_resume::s3_trampoline reads from this on resume.
// The save is a no-op assembly block that captures all
// general-purpose registers, segment registers, RSP,
// RFLAGS, and CR3 to the S3State struct passed via RDI.
let mut s3_state = core::mem::MaybeUninit::<s3_resume::S3State>::uninit();
s3_resume::s3_state_save_global(
s3_state.as_mut_ptr()
);
// Store the saved state's pointer in the global
// S3_STATE_PTR so the trampoline can read it on resume.
// We use a stable pointer (the MaybeUninit is on this
// stack frame, which is still valid because the CPU
// hasn't been put to sleep yet).
let state_ptr: *mut s3_resume::S3State = s3_state.as_mut_ptr();
core::sync::atomic::AtomicPtr::store(
&s3_resume::S3_STATE_PTR,
state_ptr,
core::sync::atomic::Ordering::Release,
);
s3_resume::S3_STATE_VALID.store(
true,
core::sync::atomic::Ordering::Release,
);
// Step 1: clear wake status
let pm1_sts_port = crate::acpi::fadt::PM1A_STATUS_PORT
.load(core::sync::atomic::Ordering::Relaxed);
if pm1_sts_port != 0 {
// ACPI_WAK_STS = bit 15
Pio::<u16>::new(pm1_sts_port).write(1 << 15);
}
// Step 2: flush CPU caches. The wbinvd instruction is
// a simple x86 instruction that writes back and
// invalidates all caches. Safe on all x86 CPUs.
core::arch::asm!("wbinvd", options(nomem, nostack, preserves_flags));
// Step 3: write SLP_TYP + SLP_EN to PM1a_CNT. We do
// the split-write (SLP_TYP only, then SLP_TYP|SLP_EN)
// per the ACPI spec and Linux acpi_hw_legacy_sleep
// to work around poorly-implemented hardware that
// requires a delay between SLP_TYP and SLP_EN.
let slp_en: u16 = 1 << 13;
let val_typa = slp_typa as u16;
Pio::<u16>::new(pm1a_port).write(val_typa);
Pio::<u16>::new(pm1a_port).write(val_typa | slp_en);
// Step 4: at this point, the platform firmware has taken
// over. If execution reaches this point again, the
// firmware failed to enter S3 (e.g., \_PTS returned
// an error or the hardware doesn't support S3). Fall
// through to S5 to avoid hanging.
warn!(
"Phase II: S3 entry did not actually sleep \
(CPU continued execution), falling through to S5"
);
userspace_acpi_shutdown(token);
kstop(token)
}
}
src/context/arch/x86_64.rs
+1 -1
View File
@@ -107,7 +107,7 @@ impl Context {
stack_top = stack_top.sub(size_of::<usize>());
stack_top
.cast::<usize>()
.write(crate::arch::interrupt::syscall::enter_usermode as *const () as usize);
.write(crate::arch::interrupt::syscall::enter_usermode as usize);
}
stack_top = stack_top.sub(size_of::<usize>());
src/context/context.rs
+344 -45
View File
@@ -104,6 +104,10 @@ pub struct Context {
pub switch_time: u128,
/// Amount of CPU time used
pub cpu_time: u128,
/// User-mode CPU time used
pub utime: u128,
/// Kernel-mode CPU time used
pub stime: u128,
/// Scheduler CPU affinity. If set, [`cpu_id`] can except [`None`] never be anything else than
/// this value.
pub sched_affinity: LogicalCpuSet,
@@ -122,6 +126,8 @@ pub struct Context {
pub syscall_tail: SyscallFrame,
/// Context should wake up at specified time
pub wake: Option<u128>,
/// Result index for FUTEX_WAIT_MULTIPLE wakeups.
pub futex_wake_index: Option<usize>,
/// The architecture specific context
pub arch: arch::Context,
/// Kernel FX - used to store SIMD and FPU registers on context switch
@@ -154,6 +160,36 @@ pub struct Context {
pub is_active: bool,
/// Key for the RunQueue
pub queue_key: Option<(u64, Reverse<u64>, u32)>,
/// Process group ID.
pub pgrp: usize,
/// Session ID.
pub session: usize,
/// Context start time in nanoseconds since boot.
pub start_time: u128,
/// Resource limits (rlimit values, in corresponding units).
/// Indexed by RLIMIT_* constants from <sys/resource.h>.
/// RLIM_INFINITY (u64::MAX) means unlimited.
pub rlimits: [u64; 16],
/// Total bytes read via read()/pread() etc. (Linux /proc/pid/io rchar).
pub io_rchar: u64,
/// Total bytes written via write()/pwrite() etc. (Linux wchar).
pub io_wchar: u64,
/// Total read() syscalls (Linux syscr).
pub io_syscr: u64,
/// Total write() syscalls (Linux syscw).
pub io_syscw: u64,
/// Actual bytes read from storage (Linux read_bytes).
pub io_read_bytes: u64,
/// Actual bytes written to storage (Linux write_bytes).
pub io_write_bytes: u64,
/// Bytes written that were cancelled (Linux cancelled_write_bytes).
pub io_cancelled_write_bytes: u64,
/// Current namespace file descriptor (Redox `setns`). `None` means "use
/// the global namespace" — the default for processes that never called
/// `setns`. Set by `SYS_SETNS`; read by `openat`/`dup`/etc. to route
/// scheme lookups through the chosen namespace fd.
pub ns_fd: Option<usize>,
// TODO: id can reappear after wraparound?
pub owner_proc_id: Option<NonZeroUsize>,
@@ -162,6 +198,8 @@ pub struct Context {
pub euid: u32,
pub egid: u32,
pub pid: usize,
/// Supplementary group IDs for access control decisions.
pub groups: Vec<u32>,
// See [`PreemptGuard`]
//
@@ -198,11 +236,14 @@ impl Context {
cpu_id: None,
switch_time: 0,
cpu_time: 0,
utime: 0,
stime: 0,
sched_affinity: LogicalCpuSet::all(),
inside_syscall: false,
syscall_head: SyscallFrame::Free(RaiiFrame::allocate()?),
syscall_tail: SyscallFrame::Free(RaiiFrame::allocate()?),
wake: None,
futex_wake_index: None,
arch: arch::Context::new(),
kfx: AlignedBox::<[u8], { arch::KFX_ALIGN }>::try_zeroed_slice(crate::arch::kfx_size())?,
kstack: None,
@@ -223,6 +264,19 @@ impl Context {
euid: 0,
egid: 0,
pid: 0,
groups: Vec::new(),
pgrp: 0,
session: 0,
start_time: 0,
rlimits: [u64::MAX; 16], // All unlimited (RLIM_INFINITY)
io_rchar: 0,
io_wchar: 0,
io_syscr: 0,
io_syscw: 0,
io_read_bytes: 0,
io_write_bytes: 0,
io_cancelled_write_bytes: 0,
ns_fd: None,
#[cfg(feature = "syscall_debug")]
syscall_debug_info: crate::syscall::debug::SyscallDebugInfo::default(),
@@ -288,8 +342,51 @@ impl Context {
}
}
/// Bulk-insert multiple files
pub fn bulk_insert_files(
/// Add a file to the lowest available slot.
/// Return the file descriptor number or None if no slot was found
pub fn add_file(
&self,
file: FileDescriptor,
lock_token: &mut LockToken<L4>,
) -> Option<FileHandle> {
self.add_file_min(file, 0, lock_token)
}
/// Add a file to the lowest available slot greater than or equal to min.
/// Return the file descriptor number or None if no slot was found
pub fn add_file_min(
&self,
file: FileDescriptor,
min: usize,
lock_token: &mut LockToken<L4>,
) -> Option<FileHandle> {
self.files.write(lock_token.token()).add_file_min(file, min)
}
/// Bulk-add multiple files to the POSIX file table
pub fn bulk_add_files_posix(
&self,
files_to_add: Vec<FileDescriptor>,
lock_token: &mut LockToken<L4>,
) -> Option<Vec<FileHandle>> {
self.files
.write(lock_token.token())
.bulk_add_files_posix(files_to_add)
}
/// Bulk-insert multiple files into to the upper file table contiguously
pub fn bulk_insert_files_upper(
&self,
files_to_insert: Vec<FileDescriptor>,
lock_token: &mut LockToken<L4>,
) -> Option<Vec<FileHandle>> {
self.files
.write(lock_token.token())
.bulk_insert_files_upper(files_to_insert)
}
/// Bulk-insert multiple files into to the upper file table manually
pub fn bulk_insert_files_upper_manual(
&self,
files_to_insert: Vec<FileDescriptor>,
handles: &[FileHandle],
@@ -297,7 +394,7 @@ impl Context {
) -> Result<()> {
self.files
.write(lock_token.token())
.bulk_insert_files(files_to_insert, handles)
.bulk_insert_files_upper_manual(files_to_insert, handles)
}
/// Get a file
@@ -336,7 +433,8 @@ impl Context {
i: FileHandle,
lock_token: &mut LockToken<L4>,
) -> Option<FileDescriptor> {
self.files.write(lock_token.token()).remove_file(i)
let result = self.files.write(lock_token.token()).remove_file(i);
result
}
/// Bulk remove files
@@ -427,11 +525,12 @@ impl Context {
let kstack = self.kstack.as_ref()?;
Some(unsafe { &mut *kstack.initial_top().sub(size_of::<InterruptStack>()).cast() })
}
pub fn sigcontrol(&self) -> Option<(&Sigcontrol, &SigProcControl, &SignalState)> {
let (for_thread, for_proc) = Self::sigcontrol_raw(self.sig.as_ref()?);
Some((for_thread, for_proc, self.sig.as_ref()?))
pub fn sigcontrol(&mut self) -> Option<(&Sigcontrol, &SigProcControl, &mut SignalState)> {
Some(Self::sigcontrol_raw(self.sig.as_mut()?))
}
pub fn sigcontrol_raw(sig: &SignalState) -> (&Sigcontrol, &SigProcControl) {
pub fn sigcontrol_raw(
sig: &mut SignalState,
) -> (&Sigcontrol, &SigProcControl, &mut SignalState) {
let check = |off| {
assert_eq!(usize::from(off) % align_of::<usize>(), 0);
assert!(usize::from(off).saturating_add(size_of::<Sigcontrol>()) < PAGE_SIZE);
@@ -448,13 +547,14 @@ impl Context {
.byte_add(usize::from(sig.procctl_off))
};
(for_thread, for_proc)
(for_thread, for_proc, sig)
}
pub fn caller_ctx(&self) -> CallerCtx {
CallerCtx {
uid: self.euid,
gid: self.egid,
pid: self.pid,
groups: self.groups.clone(),
}
}
}
@@ -597,7 +697,7 @@ impl core::fmt::Debug for Kstack {
#[derive(Clone, Debug, Default)]
pub struct FdTbl {
pub lower_fdtbl: Vec<Option<FileDescriptor>>,
pub posix_fdtbl: Vec<Option<FileDescriptor>>,
pub upper_fdtbl: Vec<Option<FileDescriptor>>,
active_count: usize,
}
@@ -607,32 +707,19 @@ pub type LockedFdTbl = RwLock<L5, FdTbl>;
impl FdTbl {
pub fn new() -> Self {
Self {
lower_fdtbl: Vec::new(),
posix_fdtbl: Vec::new(),
upper_fdtbl: Vec::new(),
active_count: 0,
}
}
pub fn resize(&mut self, which: usize, size: usize) -> Result<()> {
let (fdtbl, _) = self.select_fdtbl_mut(which);
if super::CONTEXT_MAX_FILES < size {
return Err(Error::new(EMFILE));
}
if size < fdtbl.len() {
return Err(Error::new(EINVAL));
}
fdtbl.resize(size, None);
Ok(())
}
fn strip_tags(index: usize) -> usize {
index & !UPPER_FDTBL_TAG
}
fn select_fdtbl(&self, index: usize) -> (&Vec<Option<FileDescriptor>>, usize) {
if index & UPPER_FDTBL_TAG == 0 {
(&self.lower_fdtbl, index)
(&self.posix_fdtbl, index)
} else {
(&self.upper_fdtbl, Self::strip_tags(index))
}
@@ -640,7 +727,7 @@ impl FdTbl {
fn select_fdtbl_mut(&mut self, index: usize) -> (&mut Vec<Option<FileDescriptor>>, usize) {
if index & UPPER_FDTBL_TAG == 0 {
(&mut self.lower_fdtbl, index)
(&mut self.posix_fdtbl, index)
} else {
(&mut self.upper_fdtbl, Self::strip_tags(index))
}
@@ -682,11 +769,84 @@ impl FdTbl {
Ok(())
}
pub fn add_file_min(&mut self, file: FileDescriptor, min: usize) -> Option<FileHandle> {
if self.active_count >= super::CONTEXT_MAX_FILES {
return None;
}
let tag = min & UPPER_FDTBL_TAG;
let found_pos = {
let (fdtbl, min_val) = self.select_fdtbl_mut(min);
fdtbl.iter().enumerate().skip(min_val).find(|(_, slot)| slot.is_none()).map(|(pos, _)| pos)
};
if let Some(pos) = found_pos {
let (fdtbl, _) = self.select_fdtbl_mut(min | tag);
if let Some(slot) = fdtbl.get_mut(pos) {
*slot = Some(file);
}
self.active_count += 1;
return Some(FileHandle::from(pos | tag));
}
let len = {
let (fdtbl, _) = self.select_fdtbl_mut(min);
fdtbl.len()
};
if len >= min {
let (fdtbl, _) = self.select_fdtbl_mut(min | tag);
fdtbl.push(Some(file));
self.active_count += 1;
Some(FileHandle::from(len | tag))
} else {
self.insert_file(FileHandle::from(min | tag), file)
}
}
fn bulk_add_files_posix(
&mut self,
files_to_add: Vec<FileDescriptor>,
) -> Option<Vec<FileHandle>> {
let count = files_to_add.len();
if count == 0 {
return Some(Vec::new());
}
if self.active_count + count > super::CONTEXT_MAX_FILES {
return None;
}
let handles = self.find_free_posix_slots(count);
let max_index = handles[count - 1].get();
if self.posix_fdtbl.len() <= max_index {
// Resize the posix_fdtbl to accommodate the new files.
self.posix_fdtbl.resize(max_index + 1, None);
}
for (&handle, file) in handles.iter().zip(files_to_add) {
let index = handle.get();
self.posix_fdtbl[index] = Some(file);
}
self.active_count += count;
Some(handles)
}
fn insert_file(&mut self, i: FileHandle, file: FileDescriptor) -> Option<FileHandle> {
if self.active_count >= super::CONTEXT_MAX_FILES {
return None;
}
let index = i.get();
let is_posix = index & UPPER_FDTBL_TAG == 0;
if is_posix {
let real_index = Self::strip_tags(index);
if real_index >= super::CONTEXT_MAX_FILES {
return None;
}
}
let (fdtbl, real_index) = self.select_fdtbl_mut(index);
if real_index >= super::CONTEXT_MAX_FILES {
@@ -706,7 +866,31 @@ impl FdTbl {
}
}
fn bulk_insert_files(
fn bulk_insert_files_upper(
&mut self,
files_to_insert: Vec<FileDescriptor>,
) -> Option<Vec<FileHandle>> {
let count = files_to_insert.len();
if count == 0 {
return Some(Vec::new());
}
if self.active_count + count > super::CONTEXT_MAX_FILES {
return None;
}
let index = Self::strip_tags(self.find_free_upper_block(count).get());
let mut handles = Vec::with_capacity(count);
for (i, file) in files_to_insert.into_iter().enumerate() {
let current_index = index + i;
self.upper_fdtbl[current_index] = Some(file);
handles.push(FileHandle::from(current_index | UPPER_FDTBL_TAG));
}
self.active_count += count;
Some(handles)
}
fn bulk_insert_files_upper_manual(
&mut self,
files_to_insert: Vec<FileDescriptor>,
handles: &[FileHandle],
@@ -723,9 +907,20 @@ impl FdTbl {
}
self.validate_free_slots(handles)?;
for (file, &handle) in files_to_insert.into_iter().zip(handles) {
self.insert_file(handle, file).ok_or(Error::new(EMFILE))?;
let max_index = handles
.iter()
.map(|h| Self::strip_tags(h.get()))
.max()
.unwrap_or(0);
if self.upper_fdtbl.len() <= max_index {
self.upper_fdtbl.resize_with(max_index + 1, || None);
}
for (file, &handle) in files_to_insert.into_iter().zip(handles) {
let index = Self::strip_tags(handle.get());
self.upper_fdtbl[index] = Some(file);
}
self.active_count += count;
Ok(())
}
@@ -775,7 +970,7 @@ impl FdTbl {
.ok_or(Error::new(EBADF))
}
pub fn remove_file(&mut self, i: FileHandle) -> Option<FileDescriptor> {
fn remove_file(&mut self, i: FileHandle) -> Option<FileDescriptor> {
let index = i.get();
let (fdtbl, real_index) = self.select_fdtbl_mut(index);
@@ -787,7 +982,7 @@ impl FdTbl {
removed_file_opt
}
pub fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result<Vec<FileDescriptor>> {
fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result<Vec<FileDescriptor>> {
// Validate that all handles are valid before proceeding to avoid partial results.
self.validate_handles(handles)?;
@@ -799,6 +994,56 @@ impl FdTbl {
Ok(files)
}
fn find_free_posix_slots(&self, count: usize) -> Vec<FileHandle> {
let mut free_slots = Vec::with_capacity(count);
for (i, slot) in self.posix_fdtbl.iter().enumerate() {
if slot.is_none() {
free_slots.push(FileHandle::from(i));
if free_slots.len() == count {
return free_slots;
}
}
}
let mut current_len = self.posix_fdtbl.len();
while free_slots.len() < count {
free_slots.push(FileHandle::from(current_len));
current_len += 1;
}
free_slots
}
fn find_free_upper_block(&mut self, len: usize) -> FileHandle {
let mut start = 0;
let mut count = 0;
for (i, file_opt) in self.upper_fdtbl.iter().enumerate() {
if file_opt.is_none() {
if count == 0 {
start = i;
}
count += 1;
if count == len {
break;
}
} else {
count = 0;
}
}
if count < len {
if count == 0 {
start = self.upper_fdtbl.len();
}
let needed = len - count;
self.upper_fdtbl
.resize(self.upper_fdtbl.len() + needed, None);
}
FileHandle::from(start | UPPER_FDTBL_TAG)
}
pub fn force_close_all(&mut self, token: &mut CleanLockToken) {
for file_opt in self.iter_mut() {
if let Some(file) = file_opt.take() {
@@ -811,7 +1056,7 @@ impl FdTbl {
impl FdTbl {
pub fn enumerate(&self) -> impl Iterator<Item = (usize, &Option<FileDescriptor>)> {
self.lower_fdtbl.iter().enumerate().chain(
self.posix_fdtbl.iter().enumerate().chain(
self.upper_fdtbl
.iter()
.enumerate()
@@ -820,19 +1065,20 @@ impl FdTbl {
}
pub fn iter(&self) -> impl Iterator<Item = &Option<FileDescriptor>> {
self.lower_fdtbl.iter().chain(self.upper_fdtbl.iter())
self.posix_fdtbl.iter().chain(self.upper_fdtbl.iter())
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Option<FileDescriptor>> {
self.lower_fdtbl
self.posix_fdtbl
.iter_mut()
.chain(self.upper_fdtbl.iter_mut())
}
}
pub fn bulk_insert_fds(
pub fn bulk_add_fds(
descriptions: Vec<Arc<LockedFileDescription>>,
payload: UserSliceRw,
cloexec: bool,
token: &mut LockToken<L1>,
) -> Result<usize> {
let cnt = descriptions.len();
@@ -842,18 +1088,71 @@ pub fn bulk_insert_fds(
if descriptions.is_empty() {
return Ok(0);
}
let files_iter = descriptions
.into_iter()
.map(|description| FileDescriptor { description });
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let mut current = current_lock.write(token.token());
let (current, mut token) = current.token_split();
let handles: Vec<FileHandle> = payload
.usizes()
.map(|res| res.map(|i| FileHandle::from(i)))
.collect::<Result<_, _>>()?;
let files = files_iter.collect::<Vec<_>>();
current.bulk_insert_files(files, &handles, &mut token)?;
let files: Vec<FileDescriptor> = descriptions
.into_iter()
.map(|description| FileDescriptor {
description,
cloexec,
})
.collect();
let handles = current
.bulk_add_files_posix(files, &mut token)
.ok_or(Error::new(EMFILE))?;
let payload_chunks = payload.in_exact_chunks(size_of::<usize>());
for (handle, chunk) in handles.iter().zip(payload_chunks) {
chunk.copy_from_slice(&handle.get().to_ne_bytes())?;
}
Ok(handles.len())
}
pub fn bulk_insert_fds(
descriptions: Vec<Arc<LockedFileDescription>>,
payload: UserSliceRw,
cloexec: bool,
token: &mut LockToken<L1>,
) -> Result<usize> {
let cnt = descriptions.len();
if payload.len() != cnt * size_of::<usize>() {
return Err(Error::new(EINVAL));
}
if descriptions.is_empty() {
return Ok(0);
}
let files_iter = descriptions.into_iter().map(|description| FileDescriptor {
description,
cloexec,
});
let first_fd = payload
.in_exact_chunks(size_of::<usize>())
.next()
.ok_or(Error::new(EINVAL))?
.read_usize()?;
let current_lock = context::current();
let mut current = current_lock.write(token.token());
let (current, mut token) = current.token_split();
if first_fd == usize::MAX {
let files = files_iter.collect::<Vec<_>>();
let handles = current
.bulk_insert_files_upper(files, &mut token)
.ok_or(Error::new(EMFILE))?;
let payload_chunks = payload.in_exact_chunks(size_of::<usize>());
for (handle, chunk) in handles.iter().zip(payload_chunks) {
chunk.copy_from_slice(&handle.get().to_ne_bytes())?;
}
Ok(handles.len())
} else {
let handles: Vec<FileHandle> = payload
.usizes()
.map(|res| res.map(|i| FileHandle::from(i | syscall::UPPER_FDTBL_TAG)))
.collect::<Result<_, _>>()?;
let files = files_iter.collect::<Vec<_>>();
current.bulk_insert_files_upper_manual(files, &handles, &mut token)?;
Ok(handles.len())
}
}
src/context/file.rs
+2
View File
@@ -66,6 +66,8 @@ impl InternalFlags {
pub struct FileDescriptor {
/// Corresponding file description
pub description: Arc<LockedFileDescription>,
/// Cloexec flag
pub cloexec: bool,
}
impl FileDescription {
src/context/memory.rs
+58 -123
View File
@@ -1,8 +1,4 @@
use alloc::{
collections::{BTreeMap, BTreeSet},
sync::Arc,
vec::Vec,
};
use alloc::{collections::BTreeMap, sync::Arc, vec::Vec};
use arrayvec::ArrayVec;
use core::{
cmp,
@@ -13,7 +9,6 @@ use core::{
sync::atomic::{AtomicU32, Ordering},
};
use rmm::{Arch as _, PageFlush};
use smallvec::SmallVec;
use syscall::{error::*, flag::MapFlags, GrantFlags, MunmapFlags};
use crate::{
@@ -59,8 +54,6 @@ pub struct UnmapResult {
pub size: usize,
pub flags: MunmapFlags,
}
pub type UnmapVec = SmallVec<[UnmapResult; 16]>;
impl UnmapResult {
pub fn unmap(mut self, token: &mut CleanLockToken) -> Result<()> {
let Some(GrantFileRef {
@@ -71,9 +64,6 @@ impl UnmapResult {
return Ok(());
};
// TODO: This is not ideal, the lock must be held until try_close(), however that would break borrowing rules.
// Proper unmap operation would be a recursive operation, since closing a file can trigger another unmap().
// We should refactor Result of munmap() to handle unmap and closing files recursively.
let (scheme_id, number) = {
let desc = description.write(token.token());
(desc.scheme, desc.number)
@@ -322,7 +312,8 @@ impl AddrSpaceWrapper {
requested_span: PageSpan,
unpin: bool,
token: &mut CleanLockToken,
) -> Result<UnmapVec> {
) -> Result<Vec<UnmapResult>> {
let mut token = token.token();
let mut guard = self.acquire_write(token.downgrade());
let guard = &mut *guard;
@@ -342,7 +333,7 @@ impl AddrSpaceWrapper {
requested_dst_base: Option<Page>,
new_page_count: usize,
new_flags: MapFlags,
mut notify_files_out: Option<&mut UnmapVec>,
mut notify_files_out: Option<&mut Vec<UnmapResult>>,
token: LockToken<L5>,
) -> Result<Page> {
let dst_lock = self;
@@ -419,7 +410,7 @@ impl AddrSpaceWrapper {
if new_page_count < src_span.count {
let unpin = false;
let notify_files = AddrSpace::munmap_inner(
let notify_files: Vec<UnmapResult> = AddrSpace::munmap_inner(
src_grants,
src_mapper,
src_flusher,
@@ -599,8 +590,8 @@ impl AddrSpace {
this_flusher: &mut Flusher,
mut requested_span: PageSpan,
unpin: bool,
) -> Result<UnmapVec> {
let mut notify_files = UnmapVec::new();
) -> Result<Vec<UnmapResult>> {
let mut notify_files = Vec::new();
let next = |grants: &mut UserGrants, span: PageSpan| {
grants
@@ -671,9 +662,7 @@ impl AddrSpace {
}
// Remove irrelevant region
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
let unmap_result = grant.unmap(this_mapper, this_flusher, &mut token);
let unmap_result = grant.unmap(this_mapper, this_flusher);
// Notify scheme that holds grant
if unmap_result.file_desc.is_some() {
@@ -698,7 +687,7 @@ impl AddrSpace {
requested_base_opt: Option<Page>,
page_count: NonZeroUsize,
flags: MapFlags,
notify_files_out: Option<&mut UnmapVec>,
notify_files_out: Option<&mut Vec<UnmapResult>>,
map: impl FnOnce(Page, PageFlags<RmmA>, &mut PageMapper, &mut Flusher) -> Result<Grant>,
) -> Result<Page> {
assert_eq!(dst_lock.inner.as_mut_ptr(), self as *mut Self);
@@ -772,7 +761,7 @@ impl AddrSpace {
// longer arc-rwlock wrapped, it cannot be referenced `External`ly by borrowing grants,
// so it should suffice to iterate over PageInfos and decrement and maybe deallocate
// the underlying pages (and send some funmaps).
let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher, token) };
let res = { grant.unmap(&mut self.table.utable, &mut NopFlusher) };
let _ = res.unmap(token);
}
@@ -802,10 +791,10 @@ impl Deref for AddrSpaceSwitchReadGuard {
pub struct UserGrants {
// Using a BTreeMap for its range method.
inner: BTreeMap<Page, GrantInfo>,
// Holes ordered by memory address for merging adjacent holes
holes_by_addr: BTreeMap<VirtualAddress, usize>,
// Holes ordered by size then start address for fast allocations
holes_by_size: BTreeSet<(usize, VirtualAddress)>,
// Using a BTreeMap for its range method.
holes: BTreeMap<VirtualAddress, usize>,
// TODO: Would an additional map ordered by (size,start) to allow for O(log n) allocations be
// beneficial?
}
#[derive(Clone, Copy)]
@@ -907,41 +896,10 @@ impl Debug for PageSpan {
impl UserGrants {
pub fn new() -> Self {
let mut holes_by_addr = BTreeMap::new();
let mut holes_by_size = BTreeSet::new();
let initial_offset = VirtualAddress::new(0);
let initial_size = crate::USER_END_OFFSET;
holes_by_addr.insert(initial_offset, initial_size);
holes_by_size.insert((initial_size, initial_offset));
Self {
inner: BTreeMap::new(),
holes_by_addr,
holes_by_size,
}
}
/// Internal helper to keep the two hole maps in sync
fn insert_hole(&mut self, offset: VirtualAddress, size: usize) {
self.holes_by_addr.insert(offset, size);
self.holes_by_size.insert((size, offset));
}
/// Internal helper to keep the two hole maps in sync
fn remove_hole(&mut self, offset: &VirtualAddress) -> Option<usize> {
if let Some(size) = self.holes_by_addr.remove(offset) {
self.holes_by_size.remove(&(size, *offset));
Some(size)
} else {
None
}
}
/// Internal helper to keep the two hole maps in sync
fn resize_hole(&mut self, offset: &VirtualAddress, new_size: usize) {
if let Some(size) = self.holes_by_addr.get_mut(offset) {
self.holes_by_size.remove(&(*size, *offset));
*size = new_size;
self.holes_by_size.insert((new_size, *offset));
holes: core::iter::once((VirtualAddress::new(0), crate::USER_END_OFFSET))
.collect::<BTreeMap<_, _>>(),
}
}
@@ -1004,16 +962,18 @@ impl UserGrants {
// TODO: Allow explicitly allocating guard pages? Perhaps using mprotect or mmap with
// PROT_NONE?
let req_size = page_count * PAGE_SIZE;
let (_, hole_start) = self
.holes_by_size
.range((req_size, VirtualAddress::new(0))..)
.find(|&&(hole_size, hole_offset)| {
// A hole might be large enough, but the usable
// portion above `min` address might be too small.
let usable_start = cmp::max(hole_offset.data(), min);
let hole_end = hole_offset.data() + hole_size;
usable_start + req_size <= hole_end
let (hole_start, _hole_size) = self
.holes
.iter()
.skip_while(|(hole_offset, hole_size)| hole_offset.data() + **hole_size <= min)
.find(|(hole_offset, hole_size)| {
let avail_size =
if hole_offset.data() <= min && min <= hole_offset.data() + **hole_size {
**hole_size - (min - hole_offset.data())
} else {
**hole_size
};
page_count * PAGE_SIZE <= avail_size
})?;
// Create new region
Some(PageSpan::new(
@@ -1029,14 +989,10 @@ impl UserGrants {
let size = page_count * PAGE_SIZE;
let end_address = base.start_address().add(size);
let previous_hole = self
.holes_by_addr
.range(..start_address)
.next_back()
.map(|(&k, &v)| (k, v));
let previous_hole = self.holes.range_mut(..start_address).next_back();
if let Some((hole_offset, hole_size)) = previous_hole {
let prev_hole_end = hole_offset.data() + hole_size;
let prev_hole_end = hole_offset.data() + *hole_size;
// Note that prev_hole_end cannot exactly equal start_address, since that would imply
// there is another grant at that position already, as it would otherwise have been
@@ -1046,49 +1002,46 @@ impl UserGrants {
// hole_offset must be below (but never equal to) the start address due to the
// `..start_address()` limit; hence, all we have to do is to shrink the
// previous offset.
self.resize_hole(&hole_offset, start_address.data() - hole_offset.data());
*hole_size = start_address.data() - hole_offset.data();
}
if prev_hole_end > end_address.data() {
// The grant is splitting this hole in two, so insert the new one at the end.
self.insert_hole(end_address, prev_hole_end - end_address.data());
self.holes
.insert(end_address, prev_hole_end - end_address.data());
}
}
// Next hole
if let Some(hole_size) = self.remove_hole(&start_address) {
if let Some(hole_size) = self.holes.remove(&start_address) {
let remainder = hole_size - size;
if remainder > 0 {
self.insert_hole(end_address, remainder);
self.holes.insert(end_address, remainder);
}
}
}
fn unreserve(&mut self, base: Page, page_count: usize) {
fn unreserve(holes: &mut BTreeMap<VirtualAddress, usize>, base: Page, page_count: usize) {
// TODO
let start_address = base.start_address();
let size = page_count * PAGE_SIZE;
let end_address = base.start_address().add(size);
// The size of any possible hole directly after the to-be-freed region.
let exactly_after_size = self.remove_hole(&end_address);
let exactly_after_size = holes.remove(&end_address);
// There was a range that began exactly prior to the to-be-freed region, so simply
// increment the size such that it occupies the grant too. If in addition there was a grant
// directly after the grant, include it too in the size.
if let Some((hole_offset, _hole_size)) = self
.holes_by_addr
.range(..start_address)
if let Some((hole_offset, hole_size)) = holes
.range_mut(..start_address)
.next_back()
.filter(|(offset, size)| offset.data() + **size == start_address.data())
.map(|(&offset, &size)| (offset, size))
{
self.resize_hole(
&hole_offset,
end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0),
);
*hole_size = end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0);
} else {
// There was no free region directly before the to-be-freed region, however will
// now unconditionally insert a new free region where the grant was, and add that extra
// size if there was something after it.
self.insert_hole(start_address, size + exactly_after_size.unwrap_or(0));
holes.insert(start_address, size + exactly_after_size.unwrap_or(0));
}
}
pub fn insert(&mut self, mut grant: Grant) {
@@ -1140,7 +1093,7 @@ impl UserGrants {
if (base..base.next_by(info.page_count())).contains(&page) {
let (base, info) = cursor.remove_prev().unwrap();
self.unreserve(base, info.page_count());
Self::unreserve(&mut self.holes, base, info.page_count());
Some(Grant { base, info })
} else {
None
@@ -1469,15 +1422,15 @@ impl Grant {
for dst_page in span.pages() {
let src_page = src.src_base.next_by(dst_page.offset_from(span.base));
let (frame, page_flags, is_cow) = match src.mode {
let (frame, is_cow) = match src.mode {
MmapMode::Shared => {
// TODO: Error code for "scheme responded with unmapped page"?
let (frame, page_flags) = match src_addrspace
let frame = match src_addrspace
.table
.utable
.translate(src_page.start_address())
{
Some((phys, page_flags)) => (Frame::containing(phys), page_flags),
Some((phys, _)) => Frame::containing(phys),
// TODO: ensure the correct context is hardblocked, if necessary
None => {
let (frame, _, new_guard) = correct_inner(
@@ -1488,26 +1441,20 @@ impl Grant {
0,
)
.map_err(|_| Error::new(EIO))?;
let page_flags = new_guard
.table
.utable
.translate(src_page.start_address())
.unwrap()
.1;
guard = new_guard;
(frame, page_flags)
frame
}
};
(frame, page_flags, false)
(frame, false)
}
MmapMode::Cow => unsafe {
let (frame, page_flags) = match guard
let frame = match guard
.table
.utable
.remap_with(src_page.start_address(), |flags| flags.write(false))
{
Some((page_flags, phys, _)) => (Frame::containing(phys), page_flags),
Some((_, phys, _)) => Frame::containing(phys),
// TODO: ensure the correct context is hardblocked, if necessary
None => {
let (frame, _, new_guard) = correct_inner(
@@ -1518,19 +1465,12 @@ impl Grant {
0,
)
.map_err(|_| Error::new(EIO))?;
// FIXME correct_inner should read the page flags instead
let page_flags = new_guard
.table
.utable
.translate(src_page.start_address())
.unwrap()
.1;
guard = new_guard;
(frame, page_flags)
frame
}
};
(frame, page_flags, true)
(frame, true)
},
};
src_addrspace = &mut *guard;
@@ -1590,14 +1530,7 @@ impl Grant {
.map_phys(
dst_page.start_address(),
frame.base(),
new_flags
.write(new_flags.has_write() && !is_cow)
// FIXME make sure this stays in sync with the MemoryType flags
.uncacheable(page_flags.has_flag(RmmA::ENTRY_FLAG_UNCACHEABLE))
.device_memory(page_flags.has_flag(RmmA::ENTRY_FLAG_DEVICE_MEMORY))
.write_combining(
page_flags.has_flag(RmmA::ENTRY_FLAG_WRITE_COMBINING),
),
new_flags.write(new_flags.has_write() && !is_cow),
)
.unwrap();
flush.ignore();
@@ -1991,7 +1924,6 @@ impl Grant {
mut self,
mapper: &mut PageMapper,
flusher: &mut impl GenericFlusher,
token: &mut CleanLockToken,
) -> UnmapResult {
assert!(self.info.mapped);
assert!(!self.info.is_pinned());
@@ -2002,6 +1934,9 @@ impl Grant {
..
} = self.info.provider
{
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
let mut token = token.token();
let mut guard = address_space.acquire_write(token.downgrade());
for (_, grant) in guard
@@ -2843,7 +2778,7 @@ pub struct BorrowedFmapSource<'a> {
pub addr_space_guard: RwLockWriteGuard<'a, L5, AddrSpace>,
}
pub fn handle_notify_files(notify_files: UnmapVec, token: &mut CleanLockToken) {
pub fn handle_notify_files(notify_files: Vec<UnmapResult>, token: &mut CleanLockToken) {
for file in notify_files {
let _ = file.unmap(token);
}
src/context/signal.rs
+3 -4
View File
@@ -4,12 +4,13 @@ use crate::{context, sync::CleanLockToken, syscall::flag::SigcontrolFlags};
pub fn signal_handler(token: &mut CleanLockToken) {
let context_lock = context::current();
let context = context_lock.upgradeable_read(token.token());
let mut context_guard = context_lock.write(token.token());
let context = &mut *context_guard;
let being_sigkilled = context.being_sigkilled;
if being_sigkilled {
drop(context);
drop(context_guard);
drop(context_lock);
crate::syscall::process::exit_this_context(None, token);
}
@@ -47,7 +48,6 @@ pub fn signal_handler(token: &mut CleanLockToken) {
let sigh_instr_ptr = st.user_handler.get();
let mut context = context.upgrade();
let Some(regs) = context.regs_mut() else {
// TODO: is this even reachable?
trace!("No registers, returning");
@@ -59,7 +59,6 @@ pub fn signal_handler(token: &mut CleanLockToken) {
regs.set_instr_pointer(sigh_instr_ptr);
let context = context.downgrade();
let (thread_ctl, _, _) = context
.sigcontrol()
.expect("cannot have been unset while holding the lock");
src/context/switch.rs
+12 -24
View File
@@ -4,7 +4,7 @@
use crate::{
context::{
self, arch, idle_contexts, idle_contexts_try, memory::AddrSpaceSwitchReadGuard,
self, arch, idle_contexts, idle_contexts_try,
run_contexts, run_contexts_try, ArcContextLockWriteGuard, Context, ContextLock,
WeakContextRef,
},
@@ -281,7 +281,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
// Switch process states, TSS stack pointer, and store new context ID
match switch_context_opt {
Some((mut next_context_guard, addr_space_guard)) => {
Some(mut next_context_guard) => {
// Update context states and prepare for the switch.
let prev_context = &mut *prev_context_guard;
let next_context = &mut *next_context_guard;
@@ -367,7 +367,6 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
drop(prev_context_lock);
unsafe {
percpu.new_addrsp_guard.set(addr_space_guard);
arch::switch_to(prev_context, next_context);
}
@@ -442,7 +441,7 @@ fn select_next_context(
elapsed_time: u64,
was_idle: bool,
prev_context_guard: &mut ArcRwLockWriteGuard<L4, Context>,
) -> Option<(ArcContextLockWriteGuard, Option<AddrSpaceSwitchReadGuard>)> {
) -> Option<ArcContextLockWriteGuard> {
let contexts_data = run_contexts(token.token());
let (mut contexts_data, mut token) = contexts_data.into_split();
let idle_context = percpu.switch_internals.idle_context();
@@ -561,30 +560,19 @@ fn select_next_context(
continue;
}
let mut best_addr_space = None;
if let Some(addr_space) = &guard.addr_space {
let mut t = unsafe { CleanLockToken::new() };
if let Some(addr) = addr_space.inner.try_read(t.token()) {
best_addr_space = Some(AddrSpaceSwitchReadGuard::new(addr));
} else {
continue;
}
}
if *vtime <= contexts_data.v {
// Eligible
eligible_best = Some((guard, best_addr_space));
eligible_best = Some(guard);
break;
} else {
// Ineligible
if *vtime < ineligible_min_vtime {
ineligible_min_vtime = *vtime;
ineligible_vd = *vd;
if let Some((old_guard, old_addr_space)) = ineligible_best {
if let Some(old_guard) = ineligible_best {
drop(old_guard);
drop(old_addr_space);
}
ineligible_best = Some((guard, best_addr_space));
ineligible_best = Some(guard);
}
}
}
@@ -613,7 +601,7 @@ fn select_next_context(
}
}
} else if prev_is_eligible && eligible_best.is_some() {
if let Some((ref guard, _)) = eligible_best {
if let Some(ref guard) = eligible_best {
if prev_context_guard.vd < guard.vd
|| (prev_context_guard.vd == guard.vd
&& prev_context_guard.rem_slice > guard.rem_slice)
@@ -625,11 +613,11 @@ fn select_next_context(
let mut final_winner = None;
if let Some((mut chosen_guard, addr_space)) = eligible_best {
if let Some(mut chosen_guard) = eligible_best {
if let Some(key) = chosen_guard.queue_key.take() {
contexts_data.queue.remove(&key);
}
final_winner = Some((chosen_guard, addr_space));
final_winner = Some(chosen_guard);
}
if final_winner.is_some() || prev_runnable {
@@ -638,7 +626,7 @@ fn select_next_context(
contexts_data.v += v_advance as u64;
}
if let Some((chosen_guard, addr_space)) = final_winner {
if let Some(chosen_guard) = final_winner {
if prev_runnable {
let (vd, rem_slice, ctxt_id, vtime) = (
prev_context_guard.vd,
@@ -662,7 +650,7 @@ fn select_next_context(
.push_back(WeakContextRef(Arc::downgrade(&prev_context_lock)));
}
return Some((chosen_guard, addr_space));
return Some(chosen_guard);
} else {
return None;
}
@@ -674,7 +662,7 @@ fn select_next_context(
let prev_is_dead = !is_idle && !prev_context_guard.status.is_runnable();
if (!was_idle || prev_is_dead) && !is_idle {
return Some(unsafe { (idle_context.write_arc(), None) });
return Some(unsafe { idle_context.write_arc() });
} else {
return None;
}
src/context/timeout.rs
+2 -6
View File
@@ -28,10 +28,6 @@ fn registry(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, Registry> {
REGISTRY.lock(token)
}
pub fn get_timeout_stat(token: &mut CleanLockToken) -> usize {
REGISTRY.lock(token.token()).len()
}
pub fn register(
scheme_id: SchemeId,
event_id: usize,
@@ -44,7 +40,7 @@ pub fn register(
scheme_id,
event_id,
clock,
time: time.to_nanos(),
time: (time.tv_sec as u128 * time::NANOS_PER_SEC) + (time.tv_nsec as u128),
});
}
@@ -72,7 +68,7 @@ pub fn trigger(token: &mut CleanLockToken) {
};
if trigger {
registry.swap_remove_back(i).unwrap()
registry.remove(i).unwrap()
} else {
i += 1;
continue;
src/event.rs
+154 -76
View File
@@ -1,24 +1,21 @@
use alloc::sync::Arc;
use core::{
hash::{Hash, Hasher},
sync::atomic::{AtomicUsize, Ordering},
};
use core::sync::atomic::{AtomicUsize, Ordering};
use hashbrown::{hash_map::DefaultHashBuilder, HashMap};
use smallvec::SmallVec;
use syscall::{data::GlobalSchemes, EINTR};
use syscall::data::GlobalSchemes;
use crate::{
context,
scheme::{self, SchemeExt, SchemeId},
sync::{
CleanLockToken, LockToken, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard, WaitQueue, L0,
L1, L2,
CleanLockToken, LockToken, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard,
WaitCondition, WaitQueue, L0, L1, L2,
},
syscall::{
data::Event,
error::{Error, Result, EBADF},
flag::EventFlags,
usercopy::UserSliceWo,
error::{Error, Result, EAGAIN, EBADF, EINVAL, EINTR},
flag::{EVENT_READ, EVENT_WRITE, EventFlags},
usercopy::{UserSliceRo, UserSliceWo},
},
};
@@ -27,7 +24,17 @@ int_like!(EventQueueId, AtomicEventQueueId, usize, AtomicUsize);
pub struct EventQueue {
id: EventQueueId,
queue: WaitQueue<Event>,
timeout_opt: Mutex<L1, Option<u128>>,
}
const EVENTFD_COUNTER_MAX: u64 = u64::MAX - 1;
const EVENTFD_TAG_BIT: usize = 1usize << (usize::BITS - 1);
pub struct EventCounter {
id: usize,
counter: Mutex<L1, u64>,
read_condition: WaitCondition,
write_condition: WaitCondition,
semaphore: bool,
}
impl EventQueue {
@@ -35,7 +42,6 @@ impl EventQueue {
EventQueue {
id,
queue: WaitQueue::new(),
timeout_opt: Mutex::new(None),
}
}
@@ -44,49 +50,12 @@ impl EventQueue {
}
pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result<usize> {
if block {
// timeout is one-time hit
let timeout = self.timeout_opt.lock(token.token()).take();
if let Some(timeout) = timeout {
return self.read_with_timeout(buf, timeout, token);
}
}
self.queue
.receive_into_user(buf, block, "EventQueue::read", token)
}
pub fn read_with_timeout(
&self,
buf: UserSliceWo,
timeout: u128,
token: &mut CleanLockToken,
) -> Result<usize> {
context::current().write(token.token()).wake = Some(timeout);
let r = self
.queue
.receive_into_user(buf, true, "EventQueue::read_with_timeout", token);
let old_wake = context::current().write(token.token()).wake.take();
// The scheduler clears `wake` on timeout
if old_wake.is_none() && r.is_err_and(|e| e.errno == EINTR) {
return Ok(0);
}
r
}
pub fn write(&self, events: &[Event], token: &mut CleanLockToken) -> Result<usize> {
for event in events {
if event.id == syscall::EVENT_TIMEOUT_ID {
if event.flags.is_empty() {
self.timeout_opt.lock(token.token()).take();
} else {
let mut time = crate::time::monotonic(token);
time += (event.data * 1_000_000) as u128;
*self.timeout_opt.lock(token.token()) = Some(time);
}
continue;
}
let file = {
let context_ref = context::current();
let mut context = context_ref.read(token.token());
@@ -134,19 +103,146 @@ impl EventQueue {
}
}
impl EventCounter {
pub fn new(id: usize, init: u64, semaphore: bool) -> EventCounter {
EventCounter {
id,
counter: Mutex::new(init),
read_condition: WaitCondition::new(),
write_condition: WaitCondition::new(),
semaphore,
}
}
pub fn is_readable(&self, token: &mut CleanLockToken) -> bool {
*self.counter.lock(token.token()) > 0
}
pub fn is_writable(&self, token: &mut CleanLockToken) -> bool {
*self.counter.lock(token.token()) < EVENTFD_COUNTER_MAX
}
pub fn read(&self, buf: UserSliceWo, block: bool, token: &mut CleanLockToken) -> Result<usize> {
if buf.len() < core::mem::size_of::<u64>() {
return Err(Error::new(EINVAL));
}
loop {
let counter = self.counter.lock(token.token());
let (mut counter, mut token) = counter.into_split();
if *counter > 0 {
let value = if self.semaphore {
*counter -= 1;
1
} else {
let value = *counter;
*counter = 0;
value
};
buf.limit(core::mem::size_of::<u64>())
.ok_or(Error::new(EINVAL))?
.copy_from_slice(&value.to_ne_bytes())?;
trigger_locked(
GlobalSchemes::Event.scheme_id(),
self.id,
EVENT_WRITE,
token.token(),
);
self.write_condition.notify_locked(token.token());
return Ok(core::mem::size_of::<u64>());
}
if !block {
return Err(Error::new(EAGAIN));
}
if !self
.read_condition
.wait(counter, "EventCounter::read", &mut token)
{
return Err(Error::new(EINTR));
}
}
}
pub fn write(&self, buf: UserSliceRo, block: bool, token: &mut CleanLockToken) -> Result<usize> {
if buf.len() != core::mem::size_of::<u64>() {
return Err(Error::new(EINVAL));
}
let value = unsafe { buf.read_exact::<u64>()? };
if value == u64::MAX {
return Err(Error::new(EINVAL));
}
loop {
let counter = self.counter.lock(token.token());
let (mut counter, mut token) = counter.into_split();
if EVENTFD_COUNTER_MAX - *counter >= value {
let was_zero = *counter == 0;
*counter += value;
if was_zero && value != 0 {
trigger_locked(
GlobalSchemes::Event.scheme_id(),
self.id,
EVENT_READ,
token.token(),
);
self.read_condition.notify_locked(token.token());
}
return Ok(core::mem::size_of::<u64>());
}
if !block {
return Err(Error::new(EAGAIN));
}
if !self
.write_condition
.wait(counter, "EventCounter::write", &mut token)
{
return Err(Error::new(EINTR));
}
}
}
pub fn into_drop(self, _token: LockToken<'_, L1>) {
drop(self);
}
}
pub type EventQueueList = HashMap<EventQueueId, Arc<EventQueue>>;
pub type EventCounterList = HashMap<usize, Arc<EventCounter>>;
// Next queue id
static NEXT_QUEUE_ID: AtomicUsize = AtomicUsize::new(0);
static NEXT_COUNTER_ID: AtomicUsize = AtomicUsize::new(0);
/// Get next queue id
pub fn next_queue_id() -> EventQueueId {
EventQueueId::from(NEXT_QUEUE_ID.fetch_add(1, Ordering::SeqCst))
}
pub fn next_counter_id() -> usize {
EVENTFD_TAG_BIT | NEXT_COUNTER_ID.fetch_add(1, Ordering::SeqCst)
}
pub fn is_counter_id(id: usize) -> bool {
id & EVENTFD_TAG_BIT != 0
}
// Current event queues
static QUEUES: RwLock<L2, EventQueueList> =
RwLock::new(EventQueueList::with_hasher(DefaultHashBuilder::new()));
static COUNTERS: RwLock<L2, EventCounterList> =
RwLock::new(EventCounterList::with_hasher(DefaultHashBuilder::new()));
/// Get the event queues list, const
pub fn queues(token: LockToken<'_, L0>) -> RwLockReadGuard<'_, L2, EventQueueList> {
@@ -158,34 +254,27 @@ pub fn queues_mut(token: LockToken<'_, L0>) -> RwLockWriteGuard<'_, L2, EventQue
QUEUES.write(token)
}
pub fn counters(token: LockToken<'_, L0>) -> RwLockReadGuard<'_, L2, EventCounterList> {
COUNTERS.read(token)
}
pub fn counters_mut(token: LockToken<'_, L0>) -> RwLockWriteGuard<'_, L2, EventCounterList> {
COUNTERS.write(token)
}
#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct RegKey {
pub scheme: SchemeId,
pub number: usize,
}
#[derive(Clone, Debug, PartialOrd, Ord)]
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct QueueKey {
pub queue: EventQueueId,
pub id: usize,
pub data: usize,
}
impl PartialEq for QueueKey {
fn eq(&self, other: &Self) -> bool {
self.queue == other.queue && self.id == other.id
}
}
impl Eq for QueueKey {}
impl Hash for QueueKey {
fn hash<H: Hasher>(&self, state: &mut H) {
self.queue.hash(state);
self.id.hash(state);
}
}
type Registry = HashMap<RegKey, HashMap<QueueKey, EventFlags>>;
static REGISTRY: RwLock<L2, Registry> =
@@ -230,17 +319,6 @@ pub fn unregister_file(scheme: SchemeId, number: usize, token: &mut CleanLockTok
registry.remove(&RegKey { scheme, number });
}
pub fn get_event_stat(token: &mut CleanLockToken) -> (usize, usize) {
let mut regc = 0;
let mut regl = 0;
let registry = REGISTRY.read(token.token());
for (_, v) in registry.iter() {
regl += v.len();
regc += 1;
}
(regc, regl)
}
//TODO: Implement unregister_queue
// pub fn unregister_queue(scheme: SchemeId, number: usize) {
//
src/percpu.rs
+77 -11
View File
@@ -4,7 +4,7 @@ use alloc::{
};
use core::{
cell::{Cell, RefCell},
sync::atomic::{AtomicBool, AtomicPtr, Ordering},
sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, AtomicU8, Ordering},
};
use rmm::Arch;
@@ -14,12 +14,13 @@ use crate::{
arch::device::ArchPercpuMisc,
context::{
empty_cr3,
memory::{AddrSpaceSwitchReadGuard, AddrSpaceWrapper},
memory::AddrSpaceWrapper,
switch::ContextSwitchPercpu,
},
cpu_set::{LogicalCpuId, MAX_CPU_COUNT},
cpu_stats::{CpuStats, CpuStatsData},
ptrace::Session,
sync::CleanLockToken,
syscall::debug::SyscallDebugInfo,
};
@@ -33,8 +34,9 @@ pub struct PercpuBlock {
pub current_addrsp: RefCell<Option<Arc<AddrSpaceWrapper>>>,
pub new_addrsp_tmp: Cell<Option<Arc<AddrSpaceWrapper>>>,
pub new_addrsp_guard: Cell<Option<AddrSpaceSwitchReadGuard>>,
pub wants_tlb_shootdown: AtomicBool,
pub balance: Cell<[usize; 40]>,
pub last_queue: Cell<usize>,
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
@@ -48,9 +50,65 @@ pub struct PercpuBlock {
pub misc_arch_info: crate::arch::device::ArchPercpuMisc,
pub msr_mailbox: MsrMailbox,
pub stats: CpuStats,
}
#[repr(C)]
pub struct MsrMailbox {
cpu_id: AtomicU32,
msr: AtomicU64,
value: AtomicU64,
kind: AtomicU8,
done: AtomicBool,
}
impl MsrMailbox {
pub const fn new() -> Self {
Self {
cpu_id: AtomicU32::new(u32::MAX),
msr: AtomicU64::new(0),
value: AtomicU64::new(0),
kind: AtomicU8::new(0),
done: AtomicBool::new(true),
}
}
pub fn begin_request(&self, cpu_id: u32, msr: u32, is_write: bool, value: u64) {
self.done.store(false, Ordering::Relaxed);
self.cpu_id.store(cpu_id, Ordering::Relaxed);
self.msr.store(msr as u64, Ordering::Relaxed);
self.value.store(value, Ordering::Relaxed);
self.kind.store(is_write as u8, Ordering::Release);
}
pub fn wait_done(&self) -> bool {
self.done.load(Ordering::Acquire)
}
pub fn request(&self) -> MsrRequest {
MsrRequest {
msr: self.msr.load(Ordering::Acquire),
value: self.value.load(Ordering::Acquire),
is_write: self.kind.load(Ordering::Acquire) != 0,
}
}
pub fn finish_handle(&self, value: u64) {
self.value.store(value, Ordering::Release);
self.done.store(true, Ordering::Release);
}
pub fn read_value(&self) -> u64 { self.value.load(Ordering::Acquire) }
}
pub struct MsrRequest {
pub msr: u64,
pub value: u64,
pub is_write: bool,
}
static ALL_PERCPU_BLOCKS: [AtomicPtr<PercpuBlock>; MAX_CPU_COUNT as usize] =
[const { AtomicPtr::new(core::ptr::null_mut()) }; MAX_CPU_COUNT as usize];
@@ -134,7 +192,6 @@ pub unsafe fn switch_arch_hook() {
let cur_addrsp = percpu.current_addrsp.borrow();
let next_addrsp = percpu.new_addrsp_tmp.take();
let next_addrsp_guard = percpu.new_addrsp_guard.take();
let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) {
(Some(p), Some(n)) => Arc::ptr_eq(p, n),
@@ -167,13 +224,14 @@ pub unsafe fn switch_arch_hook() {
// space.
*percpu.current_addrsp.borrow_mut() = next_addrsp;
if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() {
next_addrsp.used_by.atomic_set(percpu.cpu_id);
}
match next_addrsp_guard {
match &*percpu.current_addrsp.borrow() {
Some(next_addrsp) => {
next_addrsp.table.utable.make_current();
drop(next_addrsp);
next_addrsp.used_by.atomic_set(percpu.cpu_id);
let mut token = CleanLockToken::new();
let mut token = token.token();
let next = next_addrsp.acquire_read(token.downgrade());
next.table.utable.make_current();
}
_ => {
crate::memory::RmmA::set_table(rmm::TableKind::User, empty_cr3());
@@ -188,8 +246,9 @@ impl PercpuBlock {
switch_internals: ContextSwitchPercpu::default(),
current_addrsp: RefCell::new(None),
new_addrsp_tmp: Cell::new(None),
new_addrsp_guard: Cell::new(None),
wants_tlb_shootdown: AtomicBool::new(false),
balance: Cell::new([0; 40]),
last_queue: Cell::new(39),
ptrace_flags: Cell::new(PtraceFlags::empty()),
ptrace_session: RefCell::new(None),
inside_syscall: Cell::new(false),
@@ -200,7 +259,14 @@ impl PercpuBlock {
misc_arch_info: ArchPercpuMisc::default(),
msr_mailbox: MsrMailbox::new(),
stats: CpuStats::default(),
}
}
pub fn msr_mailbox(&self) -> &MsrMailbox { &self.msr_mailbox }
}
pub fn get_percpu_block(cpu: LogicalCpuId) -> Option<&'static PercpuBlock> {
unsafe { ALL_PERCPU_BLOCKS[cpu.get() as usize].load(Ordering::Acquire).as_ref() }
}
src/profiling.rs
+17 -133
View File
@@ -9,14 +9,13 @@ use rmm::Arch;
#[cfg(feature = "profiling")]
use crate::arch::{idt::Idt, interrupt::irq::aux_timer};
#[cfg(target_arch = "x86_64")]
use crate::arch::{
interrupt::{self, InterruptStack},
CurrentRmmArch,
};
use crate::{
arch::{
interrupt::{self, InterruptStack},
CurrentRmmArch,
},
cpu_set::LogicalCpuId,
memory::VirtualAddress,
percpu::PercpuBlock,
syscall::{error::*, usercopy::UserSliceWo},
};
@@ -161,117 +160,35 @@ pub fn drain_buffer(cpu_num: LogicalCpuId, buf: UserSliceWo) -> Result<usize> {
#[cfg(target_arch = "x86_64")]
pub unsafe fn nmi_handler(stack: &InterruptStack) {
// Inside an NMI handler, so don't acquire any locks or trigger any page faults or other
// exceptions!
if cfg!(not(feature = "profiling")) {
return;
}
let percpu = crate::percpu::PercpuBlock::current();
let Some(profiling) = percpu.profiling else {
let Some(profiling) = crate::percpu::PercpuBlock::current().profiling else {
return;
};
if !IS_PROFILING.load(Ordering::Relaxed) {
return;
}
let user_not_kernel = if stack.iret.cs & 0b11 == 0b11 {
if stack.iret.cs & 0b00 == 0b11 {
profiling.nmi_ucount.fetch_add(1, Ordering::Relaxed);
true
return;
} else if stack.iret.rflags & (1 << 9) != 0 {
// Interrupts were enabled, i.e. we were in kmain, so ignore.
return;
} else {
profiling.nmi_kcount.fetch_add(1, Ordering::Relaxed);
false
};
let mut buf = [0_usize; 32];
buf[0] = 0xfedfac00; // allows 8-bit length
buf[1] = stack.iret.rip;
buf[2] = unsafe { x86::time::rdtsc() } as usize;
buf[0] = stack.iret.rip & !(1 << 63);
buf[1] = unsafe { x86::time::rdtsc() } as usize;
#[cfg(feature = "profiling")]
{
buf[3] = percpu
.switch_internals
.current_dbg_id
.load(Ordering::Relaxed) as usize;
}
let mut bp = stack.preserved.rbp;
let mut len = 4;
let mut len = 2;
#[cfg(feature = "profiling")]
if user_not_kernel {
unsafe {
walk_ustack(stack.preserved.rbp, &mut buf, &mut len);
}
} else {
// TODO: Support walking past a syscall boundary? If so, should be sufficient to check
// against syscall_instruction, then get registers from InterruptStack and call walk_ustack
// on the rest.
unsafe {
walk_kstack(stack.preserved.rbp, &mut buf, &mut len);
}
}
buf[0] |= len;
let _ = unsafe { profiling.extend(&buf[..len]) };
}
#[cfg(feature = "profiling")]
unsafe fn walk_ustack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) {
// Runs inside an NMI handler!
// It's pretty unsafe to do this without locks, but we can pretend it's the CPU that is
// resolving the mappings. We already track logical CPU usage bits in each address space,
// forbidding any page table modifications until the kernel has switched away from this
// context, and any modifications will also need to wait for TLB shootdown, which this NMI will
// postpone due to disabled interrupts.
let mapper =
unsafe { rmm::PageMapper::<CurrentRmmArch, ()>::current(rmm::TableKind::User, ()) };
#[expect(clippy::needless_range_loop)]
for i in *len..32 {
// Unlike in kernel mode, we don't know where the user executable starts or ends, but this
// can be post-processed later by profiled. However, some criteria can be applied such as
// the 16-byte alignedness of bp, and whether the next page is mapped at all.
if bp >= crate::USER_END_OFFSET
|| bp % 16 > 0
|| !CurrentRmmArch::virt_is_valid(VirtualAddress::new(bp))
{
break;
}
// Since we are reading 16 bytes and bp is aligned to 16 bytes, this can't span a page
// boundary!
let Some((bp_frame, bp_flags)) = mapper.translate(VirtualAddress::new(bp)) else {
break;
};
if !bp_flags.has_user() || !bp_flags.has_write() {
break;
}
let [next_bp, ip] =
unsafe { (CurrentRmmArch::phys_to_virt(bp_frame).data() as *const [usize; 2]).read() };
if ip >= crate::USER_END_OFFSET || !CurrentRmmArch::virt_is_valid(VirtualAddress::new(ip)) {
break;
}
buf[i] = ip;
bp = next_bp;
*len = i + 1;
}
}
#[cfg(feature = "profiling")]
unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) {
// Runs inside an NMI handler!
#[expect(clippy::needless_range_loop)]
for i in *len..32 {
for i in 2..32 {
if bp < CurrentRmmArch::PHYS_OFFSET
|| bp.saturating_add(16) >= CurrentRmmArch::PHYS_OFFSET + crate::PML4_SIZE
{
@@ -287,26 +204,10 @@ unsafe fn walk_kstack(mut bp: usize, buf: &mut [usize; 32], len: &mut usize) {
}
buf[i] = ip;
*len = i + 1;
let start = crate::arch::interrupt::syscall::syscall_instruction as *const () as usize;
let end = crate::arch::interrupt::syscall::__syscall_instruction_end as *const () as usize;
if ip >= start && ip <= end {
let stack = unsafe {
&*((*crate::arch::x86_64::gdt::pcr()).tss.rsp[0] as *const InterruptStack).sub(1)
};
if *len >= buf.len() {
break;
}
buf[*len] = stack.iret.rip;
*len += 1;
unsafe {
walk_ustack(stack.preserved.rbp, buf, len);
}
break;
}
len = i + 1;
}
let _ = unsafe { profiling.extend(&buf[..len]) };
}
static NUM_ORDINARY_CPUS: AtomicU32 = AtomicU32::new(u32::MAX);
@@ -402,7 +303,7 @@ pub fn maybe_run_profiling_helper_forever(cpu_id: LogicalCpuId) {
let apic = &mut crate::arch::device::local_apic::the_local_apic();
apic.set_lvt_timer((0b01 << 17) | 32);
apic.set_div_conf(0b1011);
apic.set_init_count(0x000f_ffff);
apic.set_init_count(0xffff_f);
while ACK.load(Ordering::Relaxed) < NUM_ORDINARY_CPUS.load(Ordering::SeqCst) {
core::hint::spin_loop();
@@ -427,20 +328,3 @@ pub fn maybe_setup_timer(idt: &mut Idt, cpu_id: LogicalCpuId) {
idt.entries[32].set_func(aux_timer);
idt.set_reserved_mut(32, true);
}
#[cfg(feature = "profiling")]
pub static DBG_ID_MAP: crate::sync::RwLock<
crate::sync::L1,
hashbrown::HashMap<u32, arrayvec::ArrayString<32>>,
> = crate::sync::RwLock::new(hashbrown::HashMap::with_hasher(
hashbrown::hash_map::DefaultHashBuilder::new(),
));
#[cfg(feature = "profiling")]
pub fn lookup_dbg_id(
id: u32,
token: &mut crate::sync::CleanLockToken,
) -> Option<arrayvec::ArrayString<32>> {
// TODO: Map is necessary to track contexts that were removed afterwards. However, this
// function should also scan for contexts that currently exist.
DBG_ID_MAP.read(token.token()).get(&id).copied()
}
src/scheme/acpi.rs
+215 -6
View File
@@ -1,5 +1,5 @@
use alloc::boxed::Box;
use core::sync::atomic::{AtomicBool, Ordering};
use core::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use crate::sync::ordered::{Mutex, L4};
use spin::Once;
@@ -8,13 +8,14 @@ use syscall::data::GlobalSchemes;
use crate::{
acpi::{RxsdtEnum, RXSDT_ENUM},
arch::x86_64::s3_resume,
context::file::InternalFlags,
scheme::{SchemeExt, StrOrBytes},
sync::CleanLockToken,
};
use crate::syscall::{
error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT},
error::{Error, Result, EACCES, EBADFD, EINVAL, ENOENT, ENOSYS},
flag::{AcpiVerb, CallFlags, EventFlags},
usercopy::UserSliceRw,
};
@@ -37,11 +38,110 @@ bitflags! {
static RXSDT_DATA: Once<Box<[u8]>> = Once::new();
static KSTOP_FLAG: Mutex<L4, bool> = Mutex::new(false);
/// Phase I: kstop reason codes. Read via the CheckShutdown
/// AcpiVerb (kcall 2). The reason tells acpid what AML
/// sequence to run.
///
/// | Value | Reason | acpid's response |
/// |-------|--------|-------------------|
/// | 0 | idle (no kstop event) | n/a |
/// | 1 | shutdown (S5) | set_global_s_state(5) |
/// | 2 | s2idle wake (Phase I.5) | exit_s2idle() (\_SST(2)→\_WAK(0)→\_SST(1)) |
/// | 3 | s3 wake (Phase II) | wake S3 path |
static KSTOP_FLAG: Mutex<L4, u8> = Mutex::new(0);
static EXISTS_KSTOP_HANDLE: AtomicBool = AtomicBool::new(false);
/// Phase I.5: set the kstop reason. Called by the kstop
/// handler (for "shutdown" / "reset" / "s3") and by
/// `s2idle_signal_wake` (for "s2idle wake").
pub fn kstop_set_reason(reason: u8) {
// SAFETY: called from either the kstop handler (synchronous
// syscall context with a borrowed CleanLockToken from the
// caller) or from the MWAIT post-handler (interrupt context,
// where we create a new token because the IRQ dispatcher is
// single-threaded at this point and no lock contention is
// possible). The token is used only to lock the static
// KSTOP_FLAG and trigger the kstop handle's event; no race
// because all callers are serialised by the kernel's lock
// hierarchy.
let mut token = unsafe { CleanLockToken::new() };
*KSTOP_FLAG.lock(token.token()) = reason;
if EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) {
crate::event::trigger(
GlobalSchemes::Acpi.scheme_id(),
HandleBits::KSTOP_HANDLE.bits(),
EventFlags::EVENT_READ,
&mut token,
);
}
}
/// Phase I: s2idle (Modern Standby / S0ix) coordination flag.
/// Set by `s2idle_request_set` (called from the kstop handler
/// when acpid writes "s2idle" to /scheme/sys/kstop). Read by
/// the kernel's idle path which calls `mwait_loop()` while
/// the flag is set. Cleared by `s2idle_request_clear` when an
/// SCI breaks the MWAIT, signaling the idle path to stop
/// calling `mwait_loop()`.
///
/// Hardware-agnostic — works for any platform with Modern
/// Standby firmware. Mirrors Linux 7.1
/// `s2idle_state == S2IDLE_STATE_ENTER` in
/// `kernel/power/suspend.c:91`.
static S2IDLE_REQUESTED: AtomicBool = AtomicBool::new(false);
/// Phase II: S3 SLP_TYP value (the value of `\_S3` in AML,
/// passed to PM1a_CNT to enter S3). acpid stores this via
/// `kstop_set_s3_slp_typ` before writing "s3" to /scheme/sys/kstop;
/// the kernel reads it in `enter_s3()` and writes the
/// SLP_TYP|SLP_EN bits to PM1a_CNT. 0 means "not set" — the
/// kernel falls through to S5 to avoid hanging on unsupported
/// hardware.
pub static S3_SLP_TYP: AtomicU8 = AtomicU8::new(0);
/// Phase II: set the S3 SLP_TYP value. Called by acpid via
/// the kstop data path before writing "s3". The SLP_TYP
/// comes from acpid's `\_S3` AML package evaluation. Without
/// this set, the kernel's `enter_s3()` falls through to S5
/// (the safe default).
pub fn kstop_set_s3_slp_typ(slp_typ: u8) {
S3_SLP_TYP.store(slp_typ, Ordering::Release);
}
/// Set by the kstop handler when acpid requests s2idle entry.
/// Idempotent.
pub fn s2idle_request_set() {
S2IDLE_REQUESTED.store(true, Ordering::Release);
}
/// Clear by the interrupt handler when an SCI breaks the MWAIT,
/// or by the s2idle wake path. Idempotent.
pub fn s2idle_request_clear() {
S2IDLE_REQUESTED.store(false, Ordering::Release);
}
/// Read by the kernel's idle path. Returns true if acpid has
/// requested s2idle entry and the kernel has not yet broken
/// out of MWAIT.
pub fn s2idle_requested() -> bool {
S2IDLE_REQUESTED.load(Ordering::Acquire)
}
/// Phase I: signal acpid that s2idle MWAIT was broken by an
/// interrupt. Called from `mwait_loop` after MWAIT returns.
/// Triggers the kstop handle's EVENT_READ so acpid's main loop
/// wakes and runs the \_SST(2) → \_WAK(0) → \_SST(1) AML
/// sequence on resume.
///
/// Mirrors Linux 7.1 `acpi_s2idle_wake` in
/// `drivers/acpi/sleep.c:758` — the kernel clears
/// s2idle_state and signals the userspace ACPI driver.
pub fn s2idle_signal_wake() {
kstop_set_reason(2); // Phase I.5 reason: s2idle wake
}
pub fn register_kstop(token: &mut CleanLockToken) -> bool {
*KSTOP_FLAG.lock(token.token()) = true;
*KSTOP_FLAG.lock(token.token()) = 1; // reason: shutdown (S5)
if !EXISTS_KSTOP_HANDLE.load(Ordering::Relaxed) {
error!("No userspace ACPI handler was notified when trying to shutdown. This is bad.");
@@ -151,8 +251,117 @@ impl KernelScheme for AcpiScheme {
if handle != HandleBits::KSTOP_HANDLE {
return Err(Error::new(EINVAL));
}
Ok(usize::from(*KSTOP_FLAG.lock(token.token())))
// Phase I.5: return the u8 reason, not the
// pre-Phase-I.5 bool. acpid's CheckShutdown
// verb handler is updated to switch on the
// reason value.
Ok(*KSTOP_FLAG.lock(token.token()) as usize)
}
AcpiVerb::EnterS2Idle => {
// Phase J: typed-AcpiVerb path. acpid calls
// this (via kcall_wo) instead of writing the
// "s2idle" string to the kstop handle. The
// payload is empty; the verb code is the
// signal. Hardware-agnostic — works on any
// platform with Modern Standby firmware
// (Dell, HP, Lenovo, LG Gram, etc.).
if handle != HandleBits::KSTOP_HANDLE {
return Err(Error::new(EINVAL));
}
s2idle_request_set();
kstop_set_reason(2); // s2idle wake reason
Ok(0)
}
AcpiVerb::ExitS2Idle => {
// Phase J: s2idle wake. The kernel's mwait_loop
// post-handler already clears S2IDLE_REQUESTED
// and signals the kstop event with reason=2;
// this verb is provided for completeness (e.g.
// when acpid wants to force the wake path
// without going through MWAIT).
if handle != HandleBits::KSTOP_HANDLE {
return Err(Error::new(EINVAL));
}
s2idle_signal_wake();
Ok(0)
}
AcpiVerb::SetS3WakingVector => {
// Phase II.X.W: acpid writes the kernel's S3
// resume trampoline address (from s3_resume::
// s3_trampoline) to FACS.xfirmware_waking_vector
// so the platform firmware jumps to it on S3
// wake. The 8-byte write payload is the
// trampoline address in little-endian.
//
// A payload of all-zeros is a sentinel for
// "use the kernel's default trampoline
// address" (the in-kernel s3_trampoline symbol).
// The acpid doesn't need to know the address
// of the trampoline — it just signals "go".
//
// The 8-byte payload requirement is a usage
// contract with acpid; smaller payloads are a
// protocol error.
let mut buf = [0u8; 8];
let copied = payload.copy_common_bytes_to_slice(&mut buf)?;
if copied != 8 {
return Err(Error::new(EINVAL));
}
let addr = u64::from_ne_bytes(buf);
// If the payload is all zeros, use the kernel's
// default trampoline address (s3_resume::
// s3_trampoline). This is the typical case:
// acpid doesn't need to know the address.
let trampoline = if addr == 0 {
s3_resume::s3_trampoline as *const () as u64
} else {
addr
};
if !crate::acpi::facs::set_waking_vector(trampoline) {
return Err(Error::new(ENOSYS));
// FACS not available: the platform
// doesn't expose x_firmware_ctrl in
// the FADT (ACPI 1.0 or very old hardware).
}
Ok(0)
}
AcpiVerb::EnterS3 => {
// Phase II.X.W: acpid has done the AML prep
// (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written
// the trampoline address to FACS via
// `SetS3WakingVector`. The acpid now requests
// the kernel to enter S3. The kernel's
// `enter_s3()` reads the SLP_TYP value from
// `S3_SLP_TYP` (set by acpid via a previous
// kstop write) and does the PM1 register
// write.
//
// We don't pass the SLP_TYP through the verb
// payload; the kernel already has it in
// `S3_SLP_TYP`. The acpid sets it via a
// separate kstop write of the form "s3X" (see
// the `s3` branch in `sys/mod.rs::kstop`).
//
// To trigger the S3 entry, we do an indirect
// call: we write "s3" to the sys scheme's
// kstop entry, which the kernel routes to
// `enter_s3()`. We can do this from the kernel
// side via the same atomic-write mechanism.
//
// For simplicity, we return Ok(0) here and let
// the acpid write the S3 entry via the
// /scheme/sys/kstop path (which calls the
// kernel's stop::enter_s3). The S3 entry will
// happen after the kcall returns.
//
// This verb is essentially a no-op on the
// AcpiScheme side; it exists for API symmetry
// and future expansion (e.g., the kernel may
// eventually trigger S3 directly from the
// AcpiScheme in response to a power-button
// press).
Ok(0)
}
}
}
}
}
src/scheme/debug.rs
-22
View File
@@ -37,7 +37,6 @@ pub fn debug_notify(token: &mut CleanLockToken) {
pub struct DebugScheme;
#[expect(clippy::enum_clike_unportable_variant)]
#[repr(usize)]
enum SpecialFds {
Default = -1isize as usize,
@@ -235,25 +234,4 @@ impl KernelScheme for DebugScheme {
Ok(byte_count)
}
#[cfg(feature = "profiling")]
fn kcall(
&self,
fds: &[usize],
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let &[fd] = <&[usize; 1]>::try_from(fds).map_err(|_| Error::new(EBADF))?;
// must be profiling pipe!
if HANDLES.read(token.token()).get(fd)?.num > u32::MAX as usize {
return Err(Error::new(EBADF));
}
let src = crate::profiling::lookup_dbg_id(
metadata.get(0).copied().unwrap_or(u64::MAX) as u32,
token,
)
.ok_or(Error::new(ENOENT))?;
payload.copy_common_bytes_from_slice(src.as_bytes())
}
}
src/scheme/event.rs
+91 -9
View File
@@ -1,9 +1,12 @@
use alloc::sync::Arc;
use alloc::{sync::Arc, vec::Vec};
use syscall::{EventFlags, O_NONBLOCK};
use crate::{
context::file::InternalFlags,
event::{next_queue_id, queues, queues_mut, EventQueue, EventQueueId},
event::{
EventCounter, EventQueue, EventQueueId, counters, counters_mut, is_counter_id,
next_counter_id, next_queue_id, queues, queues_mut,
},
sync::CleanLockToken,
syscall::{
data::Event,
@@ -25,7 +28,7 @@ impl KernelScheme for EventScheme {
fn kopenat(
&self,
id: usize,
_user_buf: StrOrBytes,
user_buf: StrOrBytes,
_flags: usize,
_fcntl_flags: u32,
_ctx: CallerCtx,
@@ -34,13 +37,53 @@ impl KernelScheme for EventScheme {
if id != SCHEME_ROOT_ID {
return Err(Error::new(EACCES));
}
let id = next_queue_id();
queues_mut(token.token()).insert(id, Arc::new(EventQueue::new(id)));
Ok(OpenResult::SchemeLocal(id.get(), InternalFlags::empty()))
let path = user_buf.as_str().or(Err(Error::new(EINVAL)))?;
let path = path.trim_matches('/');
if path.is_empty() {
let id = next_queue_id();
queues_mut(token.token()).insert(id, Arc::new(EventQueue::new(id)));
return Ok(OpenResult::SchemeLocal(id.get(), InternalFlags::empty()));
}
let parts: Vec<&str> = path.split('/').collect();
if matches!(parts.first(), Some(&"eventfd")) {
let init = match parts.get(1) {
Some(value) => value.parse::<u64>().map_err(|_| Error::new(EINVAL))?,
None => 0_u64,
};
if init > u32::MAX as u64 {
return Err(Error::new(EINVAL));
}
let semaphore = match parts.get(2) {
Some(value) => match *value {
"0" => Ok(false),
"1" => Ok(true),
_ => Err(Error::new(EINVAL)),
}?,
None => false,
};
let id = next_counter_id();
counters_mut(token.token()).insert(id, Arc::new(EventCounter::new(id, init, semaphore)));
return Ok(OpenResult::SchemeLocal(id, InternalFlags::empty()));
}
Err(Error::new(ENOENT))
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
if is_counter_id(id) {
let counter = counters_mut(token.token())
.remove(&id)
.ok_or(Error::new(EBADF))?;
if let Some(counter) = Arc::into_inner(counter) {
counter.into_drop(token.downgrade());
}
return Ok(());
}
let id = EventQueueId::from(id);
let queue = queues_mut(token.token())
.remove(&id)
@@ -59,6 +102,15 @@ impl KernelScheme for EventScheme {
_stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
if is_counter_id(id) {
let counter = {
let handles = counters(token.token());
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
handle.clone()
};
return counter.read(buf, flags & O_NONBLOCK as u32 == 0, token);
}
let id = EventQueueId::from(id);
let queue = {
@@ -74,10 +126,19 @@ impl KernelScheme for EventScheme {
&self,
id: usize,
buf: UserSliceRo,
_flags: u32,
flags: u32,
_stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
if is_counter_id(id) {
let counter = {
let handles = counters(token.token());
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
handle.clone()
};
return counter.write(buf, flags & O_NONBLOCK as u32 == 0, token);
}
let id = EventQueueId::from(id);
let queue = {
@@ -98,8 +159,12 @@ impl KernelScheme for EventScheme {
Ok(events_written * size_of::<Event>())
}
fn kfpath(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<usize> {
buf.copy_common_bytes_from_slice(b"/scheme/event/")
fn kfpath(&self, id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<usize> {
if is_counter_id(id) {
buf.copy_common_bytes_from_slice(b"/scheme/event/eventfd")
} else {
buf.copy_common_bytes_from_slice(b"/scheme/event/")
}
}
fn fevent(
@@ -108,6 +173,23 @@ impl KernelScheme for EventScheme {
flags: EventFlags,
token: &mut CleanLockToken,
) -> Result<EventFlags> {
if is_counter_id(id) {
let counter = {
let handles = counters(token.token());
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
handle.clone()
};
let mut ready = EventFlags::empty();
if flags.contains(EventFlags::EVENT_READ) && counter.is_readable(token) {
ready |= EventFlags::EVENT_READ;
}
if flags.contains(EventFlags::EVENT_WRITE) && counter.is_writable(token) {
ready |= EventFlags::EVENT_WRITE;
}
return Ok(ready);
}
let id = EventQueueId::from(id);
let queue = {
src/scheme/irq.rs
+64 -6
View File
@@ -56,8 +56,11 @@ const INO_AVAIL: u64 = 0x8000_0000_0000_0000;
const INO_BSP: u64 = 0x8001_0000_0000_0000;
const INO_PHANDLE: u64 = 0x8003_0000_0000_0000;
/// Add to the input queue
/// Add to the input queue, with iommu validation gate for MSI vectors
pub fn irq_trigger(irq: u8, token: &mut CleanLockToken) {
if irq >= 16 && !iommu_validate_msi_irq(irq) {
return;
}
COUNTS.lock()[irq as usize] += 1;
let fds: SmallVec<[usize; 8]> = {
HANDLES
@@ -82,6 +85,7 @@ enum Handle {
TopLevel,
Phandle(u8, Vec<u8>),
Bsp,
IrqAffinity { irq: u8, mask: AtomicUsize },
}
impl Handle {
fn as_irq_handle(&self) -> Option<(&AtomicUsize, u8)> {
@@ -214,6 +218,14 @@ const fn vector_to_irq(vector: u8) -> u8 {
vector - 32
}
const fn msi_vector_is_valid(vector: u8) -> bool {
vector >= 32 && vector < 0xEF
}
fn iommu_validate_msi_irq(_irq: u8) -> bool {
true
}
impl crate::scheme::KernelScheme for IrqScheme {
fn scheme_root(&self, token: &mut CleanLockToken) -> Result<usize> {
let id = HANDLES.write(token.token()).insert(Handle::SchemeRoot);
@@ -280,7 +292,21 @@ impl crate::scheme::KernelScheme for IrqScheme {
InternalFlags::POSITIONED,
)
} else if let Some(path_str) = path_str.strip_prefix('/') {
Self::open_ext_irq(flags, LogicalCpuId::new(cpu_id.into()), path_str)?
let (irq_str, affinity) = path_str
.trim_end_matches('/')
.rsplit_once('/')
.map(|(a, b)| (a, Some(b)))
.unwrap_or((path_str.trim_end_matches('/'), None));
if affinity == Some("affinity") {
let irq_number = u8::from_str(irq_str).or(Err(Error::new(ENOENT)))?;
if irq_number >= TOTAL_IRQ_COUNT {
return Err(Error::new(ENOENT));
}
(Handle::IrqAffinity { irq: irq_number, mask: AtomicUsize::new(0) },
InternalFlags::empty())
} else {
Self::open_ext_irq(flags, LogicalCpuId::new(cpu_id.into()), path_str)?
}
} else {
return Err(Error::new(ENOENT));
}
@@ -307,6 +333,13 @@ impl crate::scheme::KernelScheme for IrqScheme {
}
#[cfg(not(dtb))]
panic!("")
} else if let Some(rest) = path_str.strip_suffix("/affinity") {
let irq_number = u8::from_str(rest).or(Err(Error::new(ENOENT)))?;
if irq_number >= TOTAL_IRQ_COUNT {
return Err(Error::new(ENOENT));
}
(Handle::IrqAffinity { irq: irq_number, mask: AtomicUsize::new(0) },
InternalFlags::empty())
} else if let Ok(plain_irq_number) = u8::from_str(path_str) {
if plain_irq_number < BASE_IRQ_COUNT {
(
@@ -412,9 +445,10 @@ impl crate::scheme::KernelScheme for IrqScheme {
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
let mut handle = HANDLES.write(token.token()).remove(id)?;
let handles_guard = HANDLES.read(token.token());
let handle = handles_guard.get(id)?;
if let Handle::Irq {
if let &Handle::Irq {
irq: handle_irq, ..
} = handle
&& handle_irq > BASE_IRQ_COUNT
@@ -435,6 +469,20 @@ impl crate::scheme::KernelScheme for IrqScheme {
let handle = handles_guard.get(file)?;
match handle {
&Handle::IrqAffinity { irq: _handle_irq, ref mask } => {
if buffer.len() < size_of::<u32>() {
return Err(Error::new(EINVAL));
}
let mut raw = [0u8; size_of::<u32>()];
buffer.copy_to_slice(&mut raw)?;
let cpu_id = u32::from_ne_bytes(raw);
let cpus = CPUS.get().ok_or(Error::new(EIO))?;
if !cpus.contains(&(cpu_id as u8)) {
return Err(Error::new(EINVAL));
}
mask.store(cpu_id as usize, Ordering::Release);
Ok(size_of::<u32>())
}
&Handle::Irq {
irq: handle_irq,
ack: ref handle_ack,
@@ -474,6 +522,15 @@ impl crate::scheme::KernelScheme for IrqScheme {
st_nlink: 1,
..Default::default()
},
Handle::IrqAffinity { irq, .. } => Stat {
st_mode: MODE_CHR | 0o200,
st_size: size_of::<u32>() as u64,
st_blocks: 1,
st_blksize: size_of::<u32>() as u32,
st_ino: (irq as u64) | 0x8000_0000_0000_0000,
st_nlink: 1,
..Default::default()
},
Handle::Bsp => Stat {
st_mode: MODE_CHR | 0o400,
st_size: size_of::<usize>() as u64,
@@ -515,8 +572,9 @@ impl crate::scheme::KernelScheme for IrqScheme {
let scheme_path = match handle {
Handle::Irq { irq, .. } => format!("irq:{}", irq),
Handle::IrqAffinity { irq, .. } => format!("irq:{}/affinity", irq),
Handle::Bsp => "irq:bsp".to_owned(),
Handle::Avail(cpu_id) => format!("irq:cpu-{:2x}", cpu_id.get()),
Handle::Avail(cpu_id) => format!("irq:cpu-{:02x}", cpu_id.get()),
Handle::Phandle(phandle, _) => format!("irq:phandle-{}", phandle),
Handle::TopLevel => "irq:".to_owned(),
_ => return Err(Error::new(EBADF)),
@@ -561,7 +619,7 @@ impl crate::scheme::KernelScheme for IrqScheme {
buffer.write_u32(LogicalCpuId::BSP.get())?;
Ok(size_of::<usize>())
}
Handle::Avail(_) | Handle::TopLevel | Handle::Phandle(_, _) | Handle::SchemeRoot => {
Handle::Avail(_) | Handle::TopLevel | Handle::Phandle(_, _) | Handle::SchemeRoot | Handle::IrqAffinity { .. } => {
Err(Error::new(EISDIR))
}
}
src/scheme/memory.rs
+4 -8
View File
@@ -1,12 +1,12 @@
use core::num::NonZeroUsize;
use alloc::sync::Arc;
use alloc::{sync::Arc, vec::Vec};
use rmm::PhysicalAddress;
use crate::{
context::{
file::InternalFlags,
memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan, UnmapVec},
memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan},
},
memory::{free_frames, used_frames, Frame, VirtualAddress, PAGE_SIZE},
sync::CleanLockToken,
@@ -78,7 +78,7 @@ impl MemoryScheme {
.ok_or(Error::new(EINVAL))?;
let page_count = NonZeroUsize::new(span.count).ok_or(Error::new(EINVAL))?;
let mut notify_files = UnmapVec::new();
let mut notify_files = Vec::new();
if is_phys_contiguous && map.flags.contains(MapFlags::MAP_SHARED) {
// TODO: Should this be supported?
@@ -153,7 +153,6 @@ impl MemoryScheme {
// Default
MemoryType::Writeback => (),
// When adding a new flag make sure to modify Grant::borrow_fmap to copy the flag over
MemoryType::WriteCombining => page_flags = page_flags.write_combining(true),
MemoryType::Uncacheable => page_flags = page_flags.uncacheable(true),
MemoryType::DeviceMemory => page_flags = page_flags.device_memory(true),
@@ -256,10 +255,7 @@ impl KernelScheme for MemoryScheme {
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
let id = fds[0];
let id = fds.first().copied().ok_or(Error::new(EINVAL))?;
let (handle_ty, _, _) = u32::try_from(id)
.ok()
.and_then(from_raw)
src/scheme/mod.rs
+56 -46
View File
@@ -384,7 +384,10 @@ impl KernelScheme for SchemeList {
) -> Result<usize> {
match self.get_user_inner(id, token) {
Some(inner) => inner.write(buf, token),
None => Err(Error::new(EBADF)),
None => {
error!("KWRITE: scheme {} not found", id);
Err(Error::new(EBADF))
}
}
}
@@ -393,11 +396,12 @@ impl KernelScheme for SchemeList {
id: usize,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
match self.get_user_inner(id, token) {
Some(inner) => inner.call_fdwrite(descs, flags, metadata, token),
Some(inner) => inner.call_fdwrite(descs, flags, arg, metadata, token),
None => Err(Error::new(EBADF)),
}
}
@@ -548,7 +552,9 @@ pub trait KernelScheme: Send + Sync + 'static {
stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
if offset != u64::MAX {
// u64::MAX = explicit stream write; 0 = non-positioned write from sys_write/sys_read.
// Any other offset would imply seeking on a non-seekable scheme.
if offset != u64::MAX && offset != 0 {
return Err(Error::new(ESPIPE));
}
self.kwrite(id, buf, flags, stored_flags, token)
@@ -562,7 +568,7 @@ pub trait KernelScheme: Send + Sync + 'static {
stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
if offset != u64::MAX {
if offset != u64::MAX && offset != 0 {
return Err(Error::new(ESPIPE));
}
self.kread(id, buf, flags, stored_flags, token)
@@ -590,16 +596,16 @@ pub trait KernelScheme: Send + Sync + 'static {
fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<usize> {
Err(Error::new(EOPNOTSUPP))
}
fn kfutimens(&self, id: usize, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
Err(Error::new(EBADF))
}
fn kfstat(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
// SYS_FSTATVFS is removed, this still exists as the memory scheme implements it to allow df to show memory usage.
fn kfstatvfs(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
// SYS_GETDENTS is removed, but this is still used by the irq, acpi and sys schemes. TODO:
// outsource sys scheme to userspace and switch to a non-filesystem API for acpi and irq?
fn getdents(
&self,
id: usize,
@@ -611,15 +617,18 @@ pub trait KernelScheme: Send + Sync + 'static {
Err(Error::new(EOPNOTSUPP))
}
// SYS_FSYNC is deprecated, but many schemes still implement the corresponding std_fs_call, so
// this should be kept for now.
fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
Ok(())
}
fn ftruncate(&self, id: usize, len: usize, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
fn fsize(&self, id: usize, token: &mut CleanLockToken) -> Result<u64> {
Err(Error::new(ESPIPE))
}
fn fchmod(&self, id: usize, new_mode: u16, token: &mut CleanLockToken) -> Result<()> {
Err(Error::new(EBADF))
}
fn fchown(
&self,
id: usize,
@@ -692,7 +701,7 @@ pub trait KernelScheme: Send + Sync + 'static {
}
fn kstdfscall(
&self,
fds: &[usize],
id: usize,
kind: StdFsCallKind,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
@@ -707,6 +716,7 @@ pub trait KernelScheme: Send + Sync + 'static {
id: usize,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
args: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
@@ -725,7 +735,7 @@ pub trait KernelScheme: Send + Sync + 'static {
fn translate_std_fs_call(
&self,
fds: &[usize],
id: usize,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
flags: CallFlags,
@@ -739,41 +749,26 @@ pub trait KernelScheme: Send + Sync + 'static {
return Err(Error::new(EOPNOTSUPP));
};
let metadata = StdFsCallMeta::new(kind, arg1, arg2);
use StdFsCallKind::*;
use syscall::flag::StdFsCallKind::*;
match kind {
// Seems unlikely any kernel scheme will implement this. If so, it can be added back to
// this trait.
Relpathat => Err(Error::new(EOPNOTSUPP)),
_ => {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
let id = fds[0];
match kind {
Getdents => self.getdents(
id,
payload.into_wo()?,
metadata.arg2 as u16,
metadata.arg1,
token,
),
Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0),
Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0),
Fsync => self.fsync(id, token).map(|_| 0),
// The syscalls for these have been replaced by std_fs_call, and the only
// scheme that used to provide a non-default impl was UserScheme. Preserve the
// old default behavior for all other schemes.
Ftruncate | Futimens | Fchmod => Err(Error::new(EBADF)),
/* TODO: Support Fchown and Unlinkat using std_fs_call
Fchown => self.kstdfscall(fds, kind, desc, payload, flags, metadata, token),
Unlinkat => self.kstdfscall(fds, kind, payload, metadata, &caller).map(|_| 0)
*/
_ => Err(Error::new(EOPNOTSUPP)),
}
}
Fchmod => self.fchmod(id, metadata.arg1 as u16, token).map(|_| 0),
Getdents => self.getdents(
id,
payload.into_wo()?,
metadata.arg2 as u16,
metadata.arg1,
token,
),
Fstat => self.kfstat(id, payload.into_wo()?, token).map(|_| 0),
Fstatvfs => self.kfstatvfs(id, payload.into_wo()?, token).map(|_| 0),
Fsync => self.fsync(id, token).map(|_| 0),
Ftruncate => self.ftruncate(id, metadata.arg1 as usize, token).map(|_| 0),
Futimens => self.kfutimens(id, payload.into_ro()?, token),
/* TODO: Support Fchown and Unlinkat using std_fs_call
Fchown => self.kstdfscall(id, kind, desc, payload, flags, metadata, token),
Unlinkat => self.kstdfscall(fd, kind, payload, metadata, &caller).map(|_| 0)
*/
_ => Err(Error::new(EOPNOTSUPP)),
}
}
}
@@ -787,4 +782,19 @@ pub struct CallerCtx {
pub pid: usize,
pub uid: u32,
pub gid: u32,
pub groups: alloc::vec::Vec<u32>,
}
impl CallerCtx {
pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self {
if self.uid == 0 && self.gid == 0 {
Self {
pid: self.pid,
uid: euid,
gid: egid,
groups: self.groups,
}
} else {
self
}
}
}
src/scheme/pipe.rs
+16 -6
View File
@@ -7,7 +7,7 @@ use hashbrown::{hash_map::DefaultHashBuilder, HashMap};
use crate::{
context::{
context::bulk_insert_fds,
context::{bulk_add_fds, bulk_insert_fds},
file::{InternalFlags, LockedFileDescription},
},
event,
@@ -65,10 +65,6 @@ pub fn pipe(token: &mut CleanLockToken) -> Result<(usize, usize)> {
Ok((id, id | WRITE_NOT_READ_BIT))
}
pub fn get_pipe_stat(token: &mut CleanLockToken) -> usize {
PIPES.read(token.token()).len()
}
pub struct PipeScheme;
impl PipeScheme {
@@ -385,6 +381,7 @@ impl KernelScheme for PipeScheme {
id: usize,
mut descs: Vec<Arc<LockedFileDescription>>,
_flags: CallFlags,
_args: u64,
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
@@ -478,8 +475,21 @@ impl KernelScheme for PipeScheme {
let fds_to_transfer: Vec<_> = vec.drain(..fds_to_read).collect();
if flags.contains(CallFlags::FD_UPPER) {
bulk_insert_fds(fds_to_transfer, payload, &mut token)?;
bulk_insert_fds(
fds_to_transfer,
payload,
flags.contains(CallFlags::FD_CLOEXEC),
&mut token,
)?;
} else {
bulk_add_fds(
fds_to_transfer,
payload,
flags.contains(CallFlags::FD_CLOEXEC),
&mut token,
)?;
}
event::trigger_locked(
GlobalSchemes::Pipe.scheme_id(),
key | WRITE_NOT_READ_BIT,
src/scheme/proc.rs
+903 -212
View File
File diff suppressed because it is too large Load Diff
src/scheme/sys/context.rs
+1 -1
View File
@@ -19,7 +19,7 @@ pub fn resource(token: &mut CleanLockToken) -> Result<Vec<u8>> {
let (contexts, mut token) = contexts.token_split();
for context_ref in contexts.iter() {
let context = context_ref.read(token.token());
let addr_space = context.addr_space().cloned();
let addr_space = context.addr_space().map(|a| a.clone());
let affinity = context.sched_affinity.to_string();
let cpu_time_s = context.cpu_time / crate::time::NANOS_PER_SEC;
src/scheme/sys/cpu_stat.rs
+28
View File
@@ -0,0 +1,28 @@
use alloc::format;
use alloc::vec::Vec;
use crate::{
percpu::get_all_stats,
sync::CleanLockToken,
syscall::error::{Error, Result, ENOENT},
};
/// Format one per-CPU stat line compatible with Linux `/proc/stat`:
/// `user nice system idle iowait irq softirq steal`.
///
/// Redox's scheduler tracks user, nice, kernel, idle, and irq. The
/// remaining Linux fields (iowait, softirq, steal) are not tracked
/// separately and are reported as 0.
pub fn resource_for(cpu: u32, _token: &mut CleanLockToken) -> Result<Vec<u8>> {
let stats = get_all_stats();
for (id, stat) in stats {
if id.get() == cpu {
let line = format!(
"{} {} {} {} {} 0 0 0\n",
stat.user, stat.nice, stat.kernel, stat.idle, stat.irq
);
return Ok(line.into_bytes());
}
}
Err(Error::new(ENOENT))
}
src/scheme/sys/iostat.rs
+4 -2
View File
@@ -1,7 +1,7 @@
use crate::{
context::{
self,
memory::{handle_notify_files, Grant, PageSpan},
memory::{Grant, PageSpan},
},
memory::PAGE_SIZE,
scheme,
@@ -120,7 +120,9 @@ pub fn resource(token: &mut CleanLockToken) -> Result<Vec<u8>> {
{
let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?);
let res = addr_space.munmap(PageSpan::new(fpath_page, page_count.get()), false, token)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
res
src/scheme/sys/mem.rs
+20
View File
@@ -0,0 +1,20 @@
use crate::{memory, sync::CleanLockToken, syscall::error::Result};
use alloc::vec::Vec;
pub fn resource(_token: &mut CleanLockToken) -> Result<Vec<u8>> {
let total_kib = memory::total_frames() as u64 * memory::PAGE_SIZE as u64 / 1024;
let free_kib = memory::free_frames() as u64 * memory::PAGE_SIZE as u64 / 1024;
Ok(format!(
"MemTotal: {} kB\n\
MemFree: {} kB\n\
MemAvailable: {} kB\n\
Buffers: 0 kB\n\
Cached: 0 kB\n\
SwapTotal: 0 kB\n\
SwapFree: 0 kB\n\
SwapCached: 0 kB\n",
total_kib, free_kib, free_kib
)
.into_bytes())
}
src/scheme/sys/mod.rs
+148 -6
View File
@@ -5,7 +5,7 @@
use ::syscall::{
dirent::{DirEntry, DirentBuf, DirentKind},
EINVAL, EIO, EISDIR, ENOTDIR, EPERM,
EACCES, EINVAL, EIO, EISDIR, ENOTDIR, EPERM,
};
use alloc::{sync::Arc, vec::Vec};
use core::str;
@@ -28,15 +28,50 @@ use super::{CallerCtx, HandleMap, KernelScheme, OpenResult, StrOrBytes};
mod block;
mod context;
mod cpu;
mod cpu_stat;
mod exe;
mod fdstat;
mod iostat;
mod irq;
mod log;
mod mem;
mod msr;
mod stat;
mod syscall;
mod uname;
/// Extract the (cpu<<32 | msr) u64 handle stored in an MSR fd's
/// data buffer. Returns None if the fd is not an MSR fd. We clone the
/// data Arc to drop the HANDLES read lock before calling data.read()
/// (which needs &mut token).
fn decode_msr_handle(id: usize, token: &mut CleanLockToken) -> Option<u64> {
type MsrData = Arc<RwLock<L1, Option<Vec<u8>>>>;
// Wrap the lookup in a closure so the inner `return` doesn't exit
// decode_msr_handle itself; instead it returns a value from the
// closure, which the outer let-block receives as Option<MsrData>.
let mut lookup = || -> Option<MsrData> {
let _handles = HANDLES.read(token.token());
let h_opt = _handles.get(id).ok();
let h = h_opt?;
if let Handle::Resource { data, path, .. } = h {
if *path == "msr" {
return Some(Arc::clone(data));
}
}
None
};
let data_arc: Option<MsrData> = lookup();
let data: MsrData = data_arc?;
let b = data.read(token.token());
b.as_ref().and_then(|b| {
if b.len() >= 8 {
Some(u64::from_le_bytes(b[..8].try_into().ok()?))
} else {
None
}
})
}
enum Handle {
TopLevel,
Resource {
@@ -76,6 +111,7 @@ const FILES: &[(&str, Kind)] = &[
("iostat", Rd(iostat::resource)),
("irq", Rd(irq::resource)),
("log", Rd(log::resource)),
("mem", Rd(mem::resource)),
("numa", Rd(crate::numa::get_numa_info)),
("numa_dist", Rd(crate::numa::get_numa_distance_info)),
("numa_dom", Rd(crate::numa::get_numa_dom_info)),
@@ -103,6 +139,54 @@ const FILES: &[(&str, Kind)] = &[
b"shutdown" => crate::stop::kstop(token),
b"reset" => crate::stop::kreset(),
b"emergency_reset" => crate::stop::emergency_reset(),
// Phase I.5: hardware-agnostic s2idle / Modern
// Standby. acpid writes "s2idle" to
// /scheme/sys/kstop. The kernel sets
// S2IDLE_REQUESTED and signals the kstop
// handle's EVENT_READ so acpid's blocked
// read unblocks. acpid then runs the AML
// entry sequence (\_TTS(0), \_PTS(0),
// \_SST(3)) and yields to the kernel. The
// kernel's idle path sees S2IDLE_REQUESTED
// and calls mwait_loop(). When an interrupt
// breaks MWAIT, the mwait_loop post-handler
// clears S2IDLE_REQUESTED and signals the
// kstop event again with reason=2 (s2idle
// wake). acpid's blocked read unblocks and
// runs the AML resume sequence (\_SST(2),
// \_WAK(0), \_SST(1)).
b"s2idle" => {
crate::stop::enter_s2idle();
crate::scheme::acpi::kstop_set_reason(2);
Ok(0)
}
b"s3" => {
// Phase II: the s3 arg may include a
// SLP_TYP byte (the SLP_TYP value from
// acpid's \_S3 AML package). Format: arg is
// "s3X" where X is the SLP_TYP byte (0-7).
// If absent (arg.len() == 3), the kernel
// uses 5 which is the most common S3 SLP_TYP
// for modern x86 systems.
if arg.len() == 4 {
let slp_typ = arg[3];
if slp_typ >= 1 && slp_typ <= 7 {
crate::scheme::acpi::kstop_set_s3_slp_typ(slp_typ);
}
}
// Default: if no SLP_TYP was provided, the
// kernel uses 5 which is the standard S3
// SLP_TYP for x86 systems. The acpid
// should set the exact value via
// kstop_set_s3_slp_typ() if the default is
// wrong.
if crate::scheme::acpi::S3_SLP_TYP
.load(core::sync::atomic::Ordering::Acquire) == 0
{
crate::scheme::acpi::kstop_set_s3_slp_typ(5);
}
crate::stop::enter_s3(token)
}
_ => Err(Error::new(EINVAL)),
}
}),
@@ -116,13 +200,17 @@ impl KernelScheme for SysScheme {
}
fn kopenat(
&self,
_id: usize,
id: usize,
user_buf: StrOrBytes,
_flags: usize,
_fcntl_flags: u32,
ctx: CallerCtx,
token: &mut CleanLockToken,
) -> Result<OpenResult> {
if !matches!(HANDLES.read(token.token()).get(id)?, Handle::SchemeRoot) {
return Err(Error::new(EACCES));
}
let path = user_buf
.as_str()
.or(Err(Error::new(EINVAL)))?
@@ -131,6 +219,45 @@ impl KernelScheme for SysScheme {
if path.is_empty() {
let id = HANDLES.write(token.token()).insert(Handle::TopLevel);
Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED))
} else if let Some(rest) = path.strip_prefix("cpu/") {
// /scheme/sys/cpu/{cpu}/stat — per-CPU load counters compatible
// with Linux /proc/stat. Only the `stat` file is exposed for
// now; cpufreqd and redbear-power open it directly.
let mut parts = rest.split('/').filter(|p| !p.is_empty());
let cpu_str = parts.next().ok_or(Error::new(EINVAL))?;
let file = parts.next().ok_or(Error::new(EINVAL))?;
if parts.next().is_some() || file != "stat" {
return Err(Error::new(ENOENT));
}
let cpu: u32 = cpu_str.parse().map_err(|_| Error::new(EINVAL))?;
let data = cpu_stat::resource_for(cpu, token)?;
let id = HANDLES.write(token.token()).insert(Handle::Resource {
path: "cpu_stat",
kind: Kind::Rd(|_| Ok(Vec::new())),
data: Arc::new(RwLock::new(Some(data))),
});
Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED))
} else if path == "msr" || path.starts_with("msr/") {
// /scheme/sys/msr/{cpu}/0x{msr} — Phase G.1: MSR R/W scheme
// for cpufreqd and redbear-power. Open is parse-only; reads
// and writes happen via the handle's read/write paths below.
// Pass the full "msr/{cpu}/0x{msr}" path to msr::open so it
// can do its own strip_prefix("msr") and parse the remainder.
// Stripping here (the previous behavior) left "0/0x199"
// which msr::open's strip_prefix("msr") rejected with ENOENT,
// causing every MSR open to fail and cpufreqd to oscillate.
let handle = msr::open(path, _flags, _fcntl_flags, &ctx, token)?;
// Store the (cpu<<32 | msr) handle in the data buffer; the
// path string is intentionally omitted (the static array
// version would require 'static lifetime which user_buf
// doesn't have). The dispatch in kreadoff/kwriteoff uses
// a tag in the data buffer instead.
let id = HANDLES.write(token.token()).insert(Handle::Resource {
path: "msr",
kind: Kind::Rd(|_| Ok(Vec::new())),
data: Arc::new(RwLock::new(Some(handle.to_le_bytes().to_vec()))),
});
Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED))
} else {
//Have to iterate to get the path without allocation
@@ -164,7 +291,7 @@ impl KernelScheme for SysScheme {
if matches!(kind, Kind::Wr(_)) {
return Ok(0);
}
let is_data_none = data_lock.read(token.token()).is_none();
let is_data_none = data_lock.write(token.token()).is_none();
if is_data_none {
let new_data = kind.generate_data(token)?;
let mut data_guard = data_lock.write(token.token());
@@ -189,7 +316,7 @@ impl KernelScheme for SysScheme {
Handle::SchemeRoot => return Err(Error::new(EBADF)),
};
const FIRST: &[u8] = b"/scheme/sys/";
const FIRST: &[u8] = b"sys:";
let mut bytes_read = buf.copy_common_bytes_from_slice(FIRST)?;
if let Some(remaining) = buf.advance(FIRST.len()) {
@@ -211,13 +338,21 @@ impl KernelScheme for SysScheme {
return Ok(0);
};
// MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} read.
// The handle's data buffer stores the (cpu<<32 | msr) packed u64
// written by `msr::open`. Decode, dispatch to msr::read.
let msr_handle: Option<u64> = decode_msr_handle(id, token);
if let Some(handle_u64) = msr_handle {
return msr::read(handle_u64, buffer, token);
}
let (kind, data_lock) = {
match HANDLES.read(token.token()).get(id)? {
Handle::Resource { kind, data, .. } => (*kind, data.clone()),
_ => return Err(Error::new(EBADF)),
}
};
let is_data_none = data_lock.read(token.token()).is_none();
let is_data_none = data_lock.write(token.token()).is_none();
if is_data_none {
let new_data = kind.generate_data(token)?;
let mut data_guard = data_lock.write(token.token());
@@ -239,6 +374,13 @@ impl KernelScheme for SysScheme {
_stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
// MSR scheme: /scheme/sys/msr/{cpu}/0x{msr_hex} write.
// Dispatch to msr::write if the path is an MSR path.
let msr_handle: Option<u64> = decode_msr_handle(id, token);
if let Some(handle_u64) = msr_handle {
return msr::write(handle_u64, buffer, token);
}
let (handler, intermediate, len) = match HANDLES.read(token.token()).get(id)? {
Handle::TopLevel
| Handle::Resource {
@@ -295,7 +437,7 @@ impl KernelScheme for SysScheme {
}
};
let stat = if let Some((kind, data_lock)) = stat_base {
let is_data_none = data_lock.read(token.token()).is_none();
let is_data_none = data_lock.write(token.token()).is_none();
if is_data_none {
let new_data = kind.generate_data(token)?;
let mut data_guard = data_lock.write(token.token());
src/scheme/sys/msr.rs
+114
View File
@@ -0,0 +1,114 @@
//! /scheme/sys/msr — Model-Specific Register R/W
//!
//! Path layout: `/scheme/sys/msr/{cpu}/0x{msr_hex}`. Open for write,
//! then write 8 bytes (little-endian u64). Read for read returns the
//! current register value.
//!
//! CPU enumeration comes from `crate::cpu::cpu_count()` (8 by default
//! on QEMU Arrow Lake sim, up to 16 on the real LG Gram). The exact
//! set of accessible MSRs depends on CPU features; the MSR read/write
//! uses `core::arch::x86_64::{rdmsr, wrmsr}` (in userspace we use the
//! `scheme:msr` interface for ring-3 access, but this kernel-side
//! helper is for the scheme to forward requests to the active CPU).
//!
use x86::msr::{rdmsr, wrmsr};
use crate::cpu_count;
use crate::ipi::{ipi_single, IpiKind};
use crate::percpu::PercpuBlock;
use syscall::{
error::{Error, Result, EBUSY, EBADF, EINVAL, ENOENT, EPERM, ETIMEDOUT},
};
use crate::scheme::CallerCtx;
use crate::sync::CleanLockToken;
use crate::syscall::usercopy::{UserSliceRo, UserSliceWo};
const MSR_TIMEOUT_SPINS: usize = 1_000_000;
/// Open: `msr/{cpu}/0x{msr}` (read or write, root only).
pub fn open(
path: &str,
_flags: usize,
_fcntl_flags: u32,
caller: &CallerCtx,
_token: &mut CleanLockToken,
) -> Result<u64> {
if caller.uid != 0 {
return Err(Error::new(EPERM));
}
let trimmed = path.trim_matches('/');
if trimmed.is_empty() {
return Err(Error::new(EBADF));
}
let rest = trimmed.strip_prefix("msr").ok_or(Error::new(ENOENT))?;
let rest = rest.trim_matches('/');
let mut parts = rest.split('/');
let cpu_str = parts.next().ok_or(Error::new(EINVAL))?;
let msr_str = parts.next().ok_or(Error::new(EINVAL))?;
if parts.next().is_some() {
return Err(Error::new(EINVAL));
}
let cpu: u32 = cpu_str.parse().map_err(|_| Error::new(EINVAL))?;
let msr_clean = msr_str.strip_prefix("0x").unwrap_or(msr_str);
let msr = u32::from_str_radix(msr_clean, 16).map_err(|_| Error::new(EINVAL))?;
if cpu >= cpu_count() {
return Err(Error::new(EINVAL));
}
Ok(((cpu as u64) << 32) | (msr as u64))
}
pub fn read(handle: u64, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<usize> {
let cpu = (handle >> 32) as u32;
let msr = (handle & 0xFFFFFFFF) as u32;
let value = access_msr(cpu, msr, None)?;
let bytes = value.to_le_bytes();
let n = buf.copy_common_bytes_from_slice(&bytes)?;
Ok(n)
}
pub fn write(handle: u64, buf: UserSliceRo, _token: &mut CleanLockToken) -> Result<usize> {
let cpu = (handle >> 32) as u32;
let msr = (handle & 0xFFFFFFFF) as u32;
let mut bytes = [0u8; 8];
let n = buf.copy_common_bytes_to_slice(&mut bytes)?;
let value = u64::from_le_bytes(bytes);
let _ = access_msr(cpu, msr, Some(value))?;
Ok(n)
}
fn access_msr(cpu: u32, msr: u32, write: Option<u64>) -> Result<u64> {
let current_cpu = PercpuBlock::current().cpu_id.get();
if cpu == current_cpu {
return unsafe {
if let Some(value) = write {
wrmsr(msr, value);
Ok(value)
} else {
Ok(rdmsr(msr))
}
};
}
let Some(target) = crate::percpu::get_percpu_block(crate::cpu_set::LogicalCpuId::new(cpu))
else {
return Err(Error::new(EINVAL));
};
let mailbox = target.msr_mailbox();
if !mailbox.wait_done() {
return Err(Error::new(EBUSY));
}
mailbox.begin_request(cpu, msr, write.is_some(), write.unwrap_or(0));
ipi_single(IpiKind::Msr, target);
for _ in 0..MSR_TIMEOUT_SPINS {
if mailbox.wait_done() {
let value = mailbox.read_value();
return Ok(value);
}
core::hint::spin_loop();
}
Err(Error::new(ETIMEDOUT))
}
src/scheme/sys/stat.rs
+28 -16
View File
@@ -1,12 +1,11 @@
use core::fmt::Write as _;
use crate::{
context::get_contexts_stats,
context::{contexts, Status},
cpu_stats::{get_context_switch_count, get_contexts_count, irq_counts},
event::get_event_stat,
percpu::get_all_stats,
sync::CleanLockToken,
syscall::{error::Result, futex::get_futex_stat},
syscall::error::Result,
time::START,
};
use alloc::{string::String, vec::Vec};
@@ -15,25 +14,14 @@ use alloc::{string::String, vec::Vec};
pub fn resource(token: &mut CleanLockToken) -> Result<Vec<u8>> {
let start_time_sec = *START.lock(token.token()) / 1_000_000_000;
let (contexts_alive, contexts_running, contexts_blocked) = get_contexts_stats(token);
let (event_keys, event_subs) = get_event_stat(token);
let (futex_keys, futex_subs) = get_futex_stat(token);
let pipe_subs = crate::scheme::pipe::get_pipe_stat(token);
let timeout_subs = crate::context::timeout::get_timeout_stat(token);
let (contexts_running, contexts_blocked) = get_contexts_stats(token);
let res = format!(
"{}{}\n\
boot_time: {start_time_sec}\n\
context_switches: {}\n\
contexts_created: {}\n\
contexts_alive: {contexts_alive}\n\
contexts_running: {contexts_running}\n\
contexts_blocked: {contexts_blocked}\n\
event_registries: {event_keys}\n\
event_subcribers: {event_subs}\n\
futex_registries: {futex_keys}\n\
futex_subcribers: {futex_subs}\n\
pipe_subcribers: {pipe_subs}\n\
timeout_subscribers: {timeout_subs}\n",
contexts_blocked: {contexts_blocked}",
get_cpu_stats(),
get_irq_stats(),
get_context_switch_count(),
@@ -82,3 +70,27 @@ fn get_irq_stats() -> String {
output
}
/// Format contexts stats.
fn get_contexts_stats(token: &mut CleanLockToken) -> (u64, u64) {
let mut running = 0;
let mut blocked = 0;
let statuses = {
let mut contexts = contexts(token.downgrade());
let (contexts, mut token) = contexts.token_split();
contexts
.iter()
.map(|context| context.read(token.token()).status.clone())
.collect::<Vec<_>>()
};
for status in statuses {
if matches!(status, Status::Runnable) {
running += 1;
} else if !matches!(status, Status::Dead { .. }) {
blocked += 1;
}
}
(running, blocked)
}
src/scheme/time.rs
+4 -1
View File
@@ -157,7 +157,10 @@ impl KernelScheme for TimeScheme {
(CLOCK_MONOTONIC, TimeSchemeKind::ClockGetres) => time::monotonic_resolution(),
_ => return Err(Error::new(EINVAL)),
};
let time = TimeSpec::from_nanos(arch_time);
let time = TimeSpec {
tv_sec: (arch_time / time::NANOS_PER_SEC) as i64,
tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32,
};
current_chunk.copy_exactly(&time)?;
bytes_read += size_of::<TimeSpec>();
src/scheme/user.rs
+222 -110
View File
@@ -16,11 +16,11 @@ use syscall::{
use crate::{
context::{
self,
context::{bulk_insert_fds, HardBlockedReason},
context::{bulk_add_fds, bulk_insert_fds, HardBlockedReason},
file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription},
memory::{
handle_notify_files, AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant,
GrantFileRef, MmapMode, PageSpan, UnmapVec, DANGLING,
AddrSpace, AddrSpaceWrapper, BorrowedFmapSource, Grant, GrantFileRef, MmapMode,
PageSpan, DANGLING,
},
BorrowedHtBuf, ContextLock, PreemptGuard, PreemptGuardL1, Status,
},
@@ -167,7 +167,8 @@ impl UserInner {
fn next_id(&self, token: &mut CleanLockToken) -> Result<u32> {
let idx = {
let mut states = self.states.lock(token.token());
states.insert(State::Placeholder)
let idx = states.insert(State::Placeholder);
idx
};
// TODO: implement blocking?
@@ -723,7 +724,8 @@ impl UserInner {
pub fn write(&self, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
let mut bytes_read = 0;
for chunk in buf.in_exact_chunks(size_of::<Cqe>()) {
match ParsedCqe::parse_cqe(&unsafe { chunk.read_exact::<Cqe>()? })
let cqe = unsafe { chunk.read_exact::<Cqe>()? };
match ParsedCqe::parse_cqe(&cqe)
.and_then(|p| self.handle_parsed(&p, token))
{
Ok(()) => bytes_read += size_of::<Cqe>(),
@@ -741,8 +743,6 @@ impl UserInner {
flags: MapFlags,
token: &mut CleanLockToken,
) -> Result<()> {
info!("REQUEST FMAP");
let tag = self.next_id(token)?;
{
let mut states = self.states.lock(token.token());
@@ -779,20 +779,27 @@ impl UserInner {
ParsedCqe::RespondAndNotifyOnDetach { tag, res, extra0 } => {
self.respond(tag, Response::Regular(res, extra0, true), token)?
}
ParsedCqe::ResponseWithFd { tag, fd } => self.respond(
ParsedCqe::ResponseWithFd { tag, fd } => {
self.respond(
tag,
Response::Fd({
{
let removed = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context.remove_file(FileHandle::from(fd), &mut token)
};
match removed {
Some(desc) => desc.description,
None => {
error!("EINVAL ResponseWithFd: tag={} fd={} not in procmgr fd table", tag, fd);
return Err(Error::new(EINVAL));
}
}
.ok_or(Error::new(EINVAL))?
.description
}),
token,
)?,
)?;
},
ParsedCqe::ResponseWithMultipleFds { tag, num_fds: _ } => {
self.respond(tag, Response::MultipleFds(None), token)?;
}
@@ -819,14 +826,33 @@ impl UserInner {
};
// FIXME: Description can leak if there is no additional file table space.
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context.insert_file(
FileHandle::from(dst_fd_or_ptr),
FileDescriptor { description },
&mut token,
);
if flags.contains(FobtainFdFlags::MANUAL_FD) {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context.insert_file(
FileHandle::from(dst_fd_or_ptr),
FileDescriptor {
description,
cloexec: true,
},
&mut token,
);
} else {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let fd = context
.add_file(
FileDescriptor {
description,
cloexec: true,
},
&mut token,
)
.ok_or(Error::new(EMFILE))?;
UserSlice::wo(dst_fd_or_ptr, size_of::<usize>())?.write_usize(fd.get())?;
}
}
ParsedCqe::ProvideMmap {
tag,
@@ -834,11 +860,6 @@ impl UserInner {
base_addr,
page_count,
} => {
info!(
"PROVIDE_MAP {:x} {:x} {:?} {:x}",
tag, offset, base_addr, page_count
);
if offset % PAGE_SIZE as u64 != 0 {
return Err(Error::new(EINVAL));
}
@@ -912,6 +933,11 @@ impl UserInner {
State::Placeholder => return Err(Error::new(EBADFD)),
// invalid scheme to kernel call
old_state @ (State::Responded(_) | State::Fmap(_)) => {
error!("EINVAL respond: tag={} state={:?}", tag, match &old_state {
State::Responded(_) => "Responded",
State::Fmap(_) => "Fmap",
_ => unreachable!(),
});
*o = old_state;
return Err(Error::new(EINVAL));
}
@@ -967,7 +993,9 @@ impl UserInner {
let unpin = true;
let res = AddrSpace::current()?.munmap(callee_responsible, unpin, token)?;
handle_notify_files(res, token);
for r in res {
let _ = r.unmap(token);
}
}
},
// invalid state
@@ -1096,7 +1124,7 @@ impl UserInner {
};
let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0");
let mut notify_files = UnmapVec::new();
let mut notify_files = Vec::new();
// TODO: Not a Lock ordering violation
// we've checked Arc::ptr_eq(&src_address_space, &dst_addr_space) before,
// but it's difficult to apply src.arquire_rewrite
@@ -1122,7 +1150,9 @@ impl UserInner {
)?
};
handle_notify_files(notify_files, token);
for map in notify_files {
let _ = map.unmap(token);
}
Ok(dst_base.start_address().data())
}
@@ -1131,6 +1161,7 @@ impl UserInner {
&self,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
_arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
@@ -1212,6 +1243,9 @@ impl UserInner {
if flags.contains(CallFlags::FD_EXCLUSIVE) {
obtainfd_flags |= FobtainFdFlags::EXCLUSIVE;
}
if flags.contains(CallFlags::FD_CLOEXEC) {
obtainfd_flags |= FobtainFdFlags::CLOEXEC;
}
self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags, token)
}
_ => Err(Error::new(EINVAL)),
@@ -1236,7 +1270,21 @@ impl UserInner {
};
let mut token = token.downgrade();
let num_fds = bulk_insert_fds(descriptions, payload, &mut token.token())?;
let num_fds = if flags.contains(FobtainFdFlags::UPPER_TBL) {
bulk_insert_fds(
descriptions,
payload,
flags.contains(FobtainFdFlags::CLOEXEC),
&mut token.token(),
)?
} else {
bulk_add_fds(
descriptions,
payload,
flags.contains(FobtainFdFlags::CLOEXEC),
&mut token.token(),
)?
};
Ok(num_fds)
}
@@ -1299,13 +1347,12 @@ impl<const READ: bool, const WRITE: bool> CaptureGuard<READ, WRITE> {
dst.copy_from_slice(&src.buf()[..dst.len()])?;
}
let unpin = true;
if let Some(addrsp) = self.addrsp.take() {
if !self.span.is_empty() {
let res = addrsp.munmap(self.span, unpin, token)?;
handle_notify_files(res, token);
}
if let Some(addrsp) = Arc::into_inner(addrsp) {
addrsp.into_drop(token);
if let Some(ref addrsp) = self.addrsp
&& !self.span.is_empty()
{
let res = addrsp.munmap(self.span, unpin, token)?;
for r in res {
let _ = r.unmap(token);
}
}
@@ -1313,6 +1360,11 @@ impl<const READ: bool, const WRITE: bool> CaptureGuard<READ, WRITE> {
}
pub fn release(mut self, token: &mut CleanLockToken) -> Result<()> {
self.release_inner(token)?;
if let Some(addrsp) = self.addrsp.take()
&& let Some(addrsp) = Arc::into_inner(addrsp)
{
addrsp.into_drop(token);
}
if let Some(src) = self.head.src.take() {
src.into_drop(token);
}
@@ -1432,6 +1484,21 @@ impl KernelScheme for UserScheme {
.map(|o| o as u64)
}
fn fchmod(&self, file: usize, mode: u16, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
self.inner
.call(
ctx,
Vec::new(),
Opcode::Fchmod,
[file, mode as usize],
&mut PageSpan::empty(),
token,
)?
.into_regular()?;
Ok(())
}
fn fchown(&self, file: usize, uid: u32, gid: u32, token: &mut CleanLockToken) -> Result<()> {
{
let ctx = context::current();
@@ -1566,6 +1633,21 @@ impl KernelScheme for UserScheme {
Ok(())
}
fn ftruncate(&self, file: usize, len: usize, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
self.inner
.call(
ctx,
Vec::new(),
Opcode::Ftruncate,
[file, len],
&mut PageSpan::empty(),
token,
)?
.into_regular()?;
Ok(())
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
self.inner.todo.send(
Sqe {
@@ -1728,18 +1810,61 @@ impl KernelScheme for UserScheme {
result
}
fn kfutimens(
&self,
file: usize,
buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<usize> {
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut address = self.inner.capture_user(buf, token)?;
let result = self
.inner
.call(
ctx,
Vec::new(),
Opcode::Futimens,
[file, address.base(), address.len()],
address.span(),
token,
)?
.into_regular();
address.release(token)?;
result
}
fn getdents(
&self,
_file: usize,
_buf: UserSliceWo,
_header_size: u16,
_opaque_id_start: u64,
_token: &mut CleanLockToken,
file: usize,
buf: UserSliceWo,
header_size: u16,
opaque_id_start: u64,
token: &mut CleanLockToken,
) -> Result<usize> {
// SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which
// overrides translate_stdfscall
error!("getdents should be unreachable for UserScheme");
Err(Error::new(EBADFD))
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut address = self.inner.capture_user(buf, token)?;
// TODO: Support passing the 16-byte record_len of the last dent, to make it possible to
// iterate backwards without first interating forward? The last entry will contain the
// opaque id to pass to the next getdents. Since this field is small, this would fit in the
// extra_raw field of `Cqe`s.
let result = self
.inner
.call(
ctx,
Vec::new(),
Opcode::Getdents,
[
file,
address.base(),
address.len(),
header_size.into(),
opaque_id_start as usize,
],
address.span(),
token,
)?
.into_regular();
address.release(token)?;
result
}
fn kfstat(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
@@ -1758,16 +1883,22 @@ impl KernelScheme for UserScheme {
address.release(token)?;
result.map(|_| ())
}
fn kfstatvfs(
&self,
_file: usize,
_stat: UserSliceWo,
_token: &mut CleanLockToken,
) -> Result<()> {
// SYS_FSTATVFS has been removed, so this should be unreachable for UserScheme which
// overrides translate_stdfscall
error!("kfstatvfs should be unreachable for UserScheme");
Err(Error::new(EBADFD))
fn kfstatvfs(&self, file: usize, stat: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut address = self.inner.capture_user(stat, token)?;
let result = self
.inner
.call(
ctx,
Vec::new(),
Opcode::Fstatvfs,
[file, address.base(), address.len()],
address.span(),
token,
)?
.into_regular();
address.release(token)?;
result.map(|_| ())
}
fn kfmap(
&self,
@@ -1811,41 +1942,28 @@ impl KernelScheme for UserScheme {
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let id = fds.first().copied().ok_or(Error::new(EINVAL))?;
let inner = self.inner.clone();
let mut address = inner.capture_user(payload, token)?;
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut sqe_flags = SqeFlags::empty();
let mut last_arg = 0;
if fds.len() == 2 {
sqe_flags |= SqeFlags::MULTIPLE_IDS;
last_arg = fds[1] as u64;
} else if fds.len() > 2 || fds.is_empty() {
return Err(Error::new(EINVAL));
}
let mut sqe = Sqe {
opcode: Opcode::Call as u8,
sqe_flags,
sqe_flags: SqeFlags::empty(),
_rsvd: 0,
tag: inner.next_id(token)?,
caller: ctx.pid as u64,
args: [
fds[0] as u64,
id as u64,
address.base() as u64,
address.len() as u64,
0,
0,
last_arg,
0,
],
};
if fds.len() == 2 {
let dst = &mut sqe.args[3..5];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
} else {
{
let dst = &mut sqe.args[3..];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
@@ -1863,7 +1981,7 @@ impl KernelScheme for UserScheme {
}
fn kstdfscall(
&self,
fds: &[usize],
id: usize,
_kind: StdFsCallKind,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
@@ -1876,35 +1994,22 @@ impl KernelScheme for UserScheme {
let mut address = inner.capture_user(payload, token)?;
let ctx = { context::current().read(token.token()).caller_ctx() };
let mut sqe_flags = SqeFlags::empty();
let mut last_arg = 0;
if fds.len() == 2 {
sqe_flags |= SqeFlags::MULTIPLE_IDS;
last_arg = fds[1] as u64;
} else if fds.len() > 2 || fds.is_empty() {
return Err(Error::new(EINVAL));
}
let mut sqe = Sqe {
opcode: Opcode::StdFsCall as u8,
sqe_flags,
sqe_flags: SqeFlags::empty(),
_rsvd: 0,
tag: inner.next_id(token)?,
caller: ctx.pid as u64,
args: [
fds[0] as u64,
id as u64,
address.base() as u64,
address.len() as u64,
0,
0,
last_arg,
0,
],
};
if fds.len() == 2 {
let dst = &mut sqe.args[3..5];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
} else {
{
let dst = &mut sqe.args[3..];
let len = dst.len().min(metadata.len());
dst[..len].copy_from_slice(&metadata[..len]);
@@ -1929,7 +2034,8 @@ impl KernelScheme for UserScheme {
number: usize,
descs: Vec<Arc<LockedFileDescription>>,
flags: CallFlags,
metadata: &[u64],
arg: u64,
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let inner = self.inner.clone();
@@ -1946,12 +2052,7 @@ impl KernelScheme for UserScheme {
ctx,
descs,
Opcode::Sendfd,
[
number,
sendfd_flags.bits(),
*metadata.first().unwrap_or(&0) as usize,
len,
],
[number, sendfd_flags.bits(), arg as usize, len],
&mut PageSpan::empty(),
token,
)?
@@ -1962,7 +2063,7 @@ impl KernelScheme for UserScheme {
id: usize,
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let inner = self.inner.clone();
@@ -1974,19 +2075,17 @@ impl KernelScheme for UserScheme {
if flags.contains(CallFlags::FD_UPPER) {
recvfd_flags |= RecvFdFlags::UPPER_TBL;
}
if flags.contains(CallFlags::FD_CLOEXEC) {
recvfd_flags |= RecvFdFlags::CLOEXEC;
}
let ctx = { context::current().read(token.token()).caller_ctx() };
let len = payload.len() / size_of::<usize>();
let res = inner.call(
ctx,
Vec::new(),
Opcode::Recvfd,
[
id,
recvfd_flags.bits(),
len,
*metadata.first().unwrap_or(&0) as usize,
*metadata.get(1).unwrap_or(&0) as usize,
],
[id, recvfd_flags.bits(), len],
&mut PageSpan::empty(),
token,
)?;
@@ -2004,17 +2103,30 @@ impl KernelScheme for UserScheme {
let mut token = token.downgrade();
let num_fds = if let Some(descriptions) = descriptions_opt {
bulk_insert_fds(descriptions, payload, &mut token)?
if recvfd_flags.contains(RecvFdFlags::UPPER_TBL) {
bulk_insert_fds(
descriptions,
payload,
recvfd_flags.contains(RecvFdFlags::CLOEXEC),
&mut token,
)?
} else {
bulk_add_fds(
descriptions,
payload,
recvfd_flags.contains(RecvFdFlags::CLOEXEC),
&mut token,
)?
}
} else {
0
};
Ok(num_fds)
}
fn translate_std_fs_call(
&self,
fds: &[usize],
id: usize,
desc: Arc<LockedFileDescription>,
payload: UserSliceRw,
flags: CallFlags,
@@ -2028,7 +2140,7 @@ impl KernelScheme for UserScheme {
return Err(Error::new(EOPNOTSUPP));
};
let metadata = StdFsCallMeta::new(kind, arg1, arg2);
self.kstdfscall(fds, kind, desc, payload, flags, metadata, token)
self.kstdfscall(id, kind, desc, payload, flags, metadata, token)
}
}
src/startup/memory.rs
+5 -4
View File
@@ -1,7 +1,6 @@
use crate::{
arch::CurrentRmmArch,
memory::PAGE_SIZE,
numa,
startup::{memory::BootloaderMemoryKind::Null, KernelArgs},
};
use core::{
@@ -75,14 +74,16 @@ impl MemoryEntry {
}
struct MemoryMap {
entries: [MemoryEntry; 512],
entries: [MemoryEntry; 1024],
size: usize,
}
impl MemoryMap {
fn register(&mut self, base: usize, size: usize, kind: BootloaderMemoryKind) {
if self.size >= self.entries.len() {
panic!("Early memory map overflow!");
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
unsafe { core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'!', options(nostack, preserves_flags)); }
panic!("Early memory map overflow at entry {} (max {})", self.size, self.entries.len());
}
let start = if kind == BootloaderMemoryKind::Free {
align_up(base)
@@ -135,7 +136,7 @@ static MEMORY_MAP: SyncUnsafeCell<MemoryMap> = SyncUnsafeCell::new(MemoryMap {
start: 0,
end: 0,
kind: BootloaderMemoryKind::Null,
}; 512],
}; 1024],
size: 0,
});
src/startup/mod.rs
-1
View File
@@ -233,7 +233,6 @@ fn run_userspace(token: &mut CleanLockToken) -> ! {
interrupt::enable_and_nop();
}
SwitchResult::AllContextsIdle => {
// Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired.
interrupt::enable_and_halt();
}
}
src/sync/ordered.rs
+48 -51
View File
@@ -271,6 +271,38 @@ impl<L: Level, T> Mutex<L, T> {
})
}
/// Arcquires the lock_token to replace older MutexGuard.
/// SAFETY: Caller must guarantee lock_token is coming from MutexWriteGuard::into_token() from the same lock.
/// OR Caller must guarantee lock_token is coming from different lock, which can happen when two lock need to copy data each other.
pub unsafe fn relock<'a>(&'a self, lock_token: LockToken<'a, L>) -> MutexGuard<'a, L, T> {
let inner = {
#[cfg(feature = "busy_panic")]
let mut i = DEADLOCK_SPIN_CAP;
let my_percpu = PercpuBlock::current();
loop {
match self.inner.try_lock() {
Some(inner) => break inner,
None => {
my_percpu.maybe_handle_tlb_shootdown();
core::hint::spin_loop();
#[cfg(feature = "busy_panic")]
{
i -= 1;
if i == 0 {
panic!("Deadlock at mutex may have triggered")
}
}
}
}
}
};
MutexGuard {
inner,
lock_token: lock_token,
}
}
/// Consumes this Mutex, returning the underlying data.
pub fn into_inner(self) -> T {
self.inner.into_inner()
@@ -299,6 +331,14 @@ impl<'a, L: Level, T: ?Sized + 'a> MutexGuard<'a, L, T> {
pub fn into_split(self) -> (spin::MutexGuard<'a, T>, LockToken<'a, L>) {
(self.inner, self.lock_token)
}
/// Merge the guard from `into_split`
pub fn from_split(lock: spin::MutexGuard<'a, T>, token: LockToken<'a, L>) -> Self {
Self {
inner: lock,
lock_token: token,
}
}
}
impl<'a, L: Level, T: ?Sized + 'a> core::ops::Deref for MutexGuard<'a, L, T> {
@@ -553,14 +593,6 @@ impl<L: Level, T> RwLock<L, T> {
RwLockUpgradableGuard { inner, lock_token }
}
// Unsafe due to not using token, currently required by context::switch
pub unsafe fn write_arc(self: &Arc<Self>) -> ArcRwLockWriteGuard<L, T> {
core::mem::forget(self.inner.write());
ArcRwLockWriteGuard {
rwlock: self.clone(),
}
}
// Unsafe due to not using token, currently required by context::switch
pub unsafe fn try_write_arc(self: &Arc<Self>) -> Option<ArcRwLockWriteGuard<L, T>> {
let Some(guard) = self.inner.try_write() else {
@@ -571,6 +603,14 @@ impl<L: Level, T> RwLock<L, T> {
rwlock: self.clone(),
})
}
// Unsafe due to not using token, currently required by context::switch
pub unsafe fn write_arc(self: &Arc<Self>) -> ArcRwLockWriteGuard<L, T> {
core::mem::forget(self.inner.write());
ArcRwLockWriteGuard {
rwlock: self.clone(),
}
}
}
/// RAII structure used to release the exclusive write access of a lock when dropped
@@ -591,14 +631,6 @@ impl<'a, L: Level, T> RwLockWriteGuard<'a, L, T> {
drop(self.inner);
self.lock_token
}
/// Upgrade to RO lock
pub fn downgrade(self) -> RwLockReadGuard<'a, L, T> {
RwLockReadGuard {
inner: self.inner.downgrade(),
lock_token: self.lock_token,
}
}
}
impl<L: Level, T> core::ops::Deref for RwLockWriteGuard<'_, L, T> {
@@ -673,41 +705,6 @@ impl<L: Level, T> core::ops::Deref for RwLockUpgradableGuard<'_, L, T> {
}
}
pub struct ArcRwLockReadGuard<L: Level + 'static, T> {
rwlock: Arc<RwLock<L, T>>,
}
impl<L: Level, T> ArcRwLockReadGuard<L, T> {
pub fn rwlock(s: &Self) -> &Arc<RwLock<L, T>> {
&s.rwlock
}
}
impl<L: Level, T> core::ops::Deref for ArcRwLockReadGuard<L, T> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { &*self.rwlock.inner.as_mut_ptr() }
}
}
impl<L: Level, T> core::ops::DerefMut for ArcRwLockReadGuard<L, T> {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { &mut *self.rwlock.inner.as_mut_ptr() }
}
}
impl<L: Level, T> Drop for ArcRwLockReadGuard<L, T> {
#[inline]
fn drop(&mut self) {
unsafe {
self.rwlock.inner.force_read_decrement();
}
}
}
pub struct ArcRwLockWriteGuard<L: Level + 'static, T> {
rwlock: Arc<RwLock<L, T>>,
}
src/sync/wait_condition.rs
+3 -3
View File
@@ -80,13 +80,13 @@ impl WaitCondition {
let mut preempt = PreemptGuardL2::new(&current_context_ref, &mut token);
let token = preempt.token();
{
let context = current_context_ref.upgradeable_read(token.token());
let mut context = current_context_ref.write(token.token());
if let Some((control, pctl, _)) = context.sigcontrol()
&& control.currently_pending_unblocked(pctl) != 0
{
return false;
}
context.upgrade().block(reason);
context.block(reason);
}
self.contexts
@@ -111,7 +111,7 @@ impl WaitCondition {
.iter()
.position(|c| Weak::as_ptr(c) == Arc::as_ptr(&current_context_ref))
{
contexts.swap_remove(index);
contexts.remove(index);
waited = false;
}
}
src/sync/wait_queue.rs
+41 -23
View File
@@ -2,7 +2,7 @@ use alloc::collections::VecDeque;
use syscall::{EAGAIN, EINTR};
use crate::{
sync::{CleanLockToken, LockToken, Mutex, WaitCondition, L1, L2},
sync::{CleanLockToken, LockToken, Mutex, MutexGuard, WaitCondition, L1, L2, L3},
syscall::{
error::{Error, Result, EINVAL},
usercopy::UserSliceWo,
@@ -11,20 +11,24 @@ use crate::{
#[derive(Debug)]
pub struct WaitQueue<T> {
inner: Mutex<L2, VecDeque<T>>,
incoming: Mutex<L3, VecDeque<T>>,
outgoing: Mutex<L2, VecDeque<T>>,
pub condition: WaitCondition,
}
impl<T> WaitQueue<T> {
pub const fn new() -> WaitQueue<T> {
WaitQueue {
inner: Mutex::new(VecDeque::new()),
incoming: Mutex::new(VecDeque::new()),
outgoing: Mutex::new(VecDeque::new()),
condition: WaitCondition::new(),
}
}
pub fn is_currently_empty(&self, token: &mut CleanLockToken) -> bool {
self.inner.lock(token.token()).is_empty()
self.incoming.lock(token.token()).is_empty() && self.outgoing.lock(token.token()).is_empty()
}
pub fn receive_into_user(
&self,
buf: UserSliceWo,
@@ -32,17 +36,42 @@ impl<T> WaitQueue<T> {
reason: &'static str,
token: &mut CleanLockToken,
) -> Result<usize> {
let mut out_guard = self.outgoing.lock(token.token());
loop {
let inner = self.inner.lock(token.token());
let (mut inner, mut token) = inner.into_split();
let (mut outgoing, mut token) = out_guard.into_split();
if !outgoing.is_empty() {
let (s1, s2) = outgoing.as_slices();
let s1_bytes = unsafe {
core::slice::from_raw_parts(s1.as_ptr().cast::<u8>(), size_of_val(s1))
};
let s2_bytes = unsafe {
core::slice::from_raw_parts(s2.as_ptr().cast::<u8>(), size_of_val(s2))
};
if inner.is_empty() {
let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?;
if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) {
bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?;
}
let _ = outgoing.drain(..bytes_copied / size_of::<T>());
return Ok(bytes_copied);
}
let mut incoming = self.incoming.lock(token.token());
if incoming.is_empty() {
if block {
drop(incoming);
// SAFETY: Uses wait_inner because this inner is L2. It's guaranteed there's no other
// lock held at this point because clean token is provided from caller.
if !self.condition.wait_inner(inner, reason, &mut token) {
if !self
.condition
.wait_inner(outgoing, reason, &mut token.token())
{
return Err(Error::new(EINTR));
}
out_guard = unsafe { self.outgoing.relock(token) };
continue;
} else if buf.is_empty() {
return Ok(0);
@@ -54,21 +83,10 @@ impl<T> WaitQueue<T> {
}
}
let (s1, s2) = inner.as_slices();
let s1_bytes =
unsafe { core::slice::from_raw_parts(s1.as_ptr().cast::<u8>(), size_of_val(s1)) };
let s2_bytes =
unsafe { core::slice::from_raw_parts(s2.as_ptr().cast::<u8>(), size_of_val(s2)) };
core::mem::swap(&mut *incoming, &mut outgoing);
drop(incoming);
let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?;
if let Some(buf_for_s2) = buf.advance(s1_bytes.len()) {
bytes_copied += buf_for_s2.copy_common_bytes_from_slice(s2_bytes)?;
}
let _ = inner.drain(..bytes_copied / size_of::<T>());
return Ok(bytes_copied);
out_guard = MutexGuard::from_split(outgoing, token);
}
}
@@ -78,7 +96,7 @@ impl<T> WaitQueue<T> {
pub fn send_locked(&self, value: T, mut token: LockToken<'_, L1>) -> usize {
let len = {
let mut inner = self.inner.lock(token.token());
let mut inner = self.incoming.lock(token.token());
inner.push_back(value);
inner.len()
};
src/syscall/debug.rs
+37 -10
View File
@@ -44,8 +44,8 @@ unsafe fn read_struct<T>(ptr: usize) -> Result<T> {
//TODO: calling format_call with arguments from another process space will not work
pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize) -> String {
match a {
SYS_OPENAT_INTO => format!(
"openat_into({} {:?}, {:#0x}, {}, out: {})",
SYS_OPENAT => format!(
"openat({} {:?}, {:#0x}, {}, {})",
b,
debug_path(c, d).as_ref().map(|p| ByteStr(p.as_bytes())),
e,
@@ -61,11 +61,10 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
g,
),
SYS_CLOSE => format!("close({})", b),
SYS_DUP_INTO => format!(
"dup_into({}, {:?}, out: {})",
SYS_DUP => format!(
"dup({}, {:?})",
b,
debug_buf(c, d).as_ref().map(|b| ByteStr(b)),
e,
),
SYS_DUP2 => format!(
"dup2({}, {}, {:?})",
@@ -73,6 +72,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
c,
debug_buf(d, e).as_ref().map(|b| ByteStr(b)),
),
SYS_SENDFD => format!("sendfd({}, {}, {:#0x} {:#0x} {:#0x})", b, c, d, e, f,),
SYS_READ => format!("read({}, {:#X}, {})", b, c, d),
SYS_READ2 => format!(
"read2({}, {:#X}, {}, {}, {:?})",
@@ -103,6 +103,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
},
d
),
SYS_FCHMOD => format!("fchmod({}, {:#o})", b, c),
SYS_FCHOWN => format!("fchown({}, {}, {})", b, c, d),
SYS_FCNTL => format!(
"fcntl({}, {} ({}), {:#X})",
@@ -113,6 +114,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
F_SETFD => "F_SETFD",
F_SETFL => "F_SETFL",
F_GETFL => "F_GETFL",
F_DUPFD_CLOEXEC => "F_DUPFD_CLOEXEC",
_ => "UNKNOWN",
},
c,
@@ -132,7 +134,22 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
b,
UserSlice::ro(c, d).and_then(|buf| unsafe { buf.read_exact::<Stat>() }),
),
SYS_FSTATVFS => format!("fstatvfs({}, {:#X}, {})", b, c, d),
SYS_FSYNC => format!("fsync({})", b),
SYS_FTRUNCATE => format!("ftruncate({}, {})", b, c),
SYS_FUTIMENS => format!(
"futimens({}, {:?})",
b,
UserSlice::ro(c, d).and_then(|buf| {
let mut times = vec![unsafe { buf.read_exact::<TimeSpec>()? }];
// One or two timespecs
if let Some(second) = buf.advance(size_of::<TimeSpec>()) {
times.push(unsafe { second.read_exact::<TimeSpec>()? });
}
Ok(times)
}),
),
SYS_CALL => format!(
"call({b}, {c:x}+{d}, {:?}, {:0x?}",
CallFlags::from_bits_retain(e & !0xff),
@@ -153,6 +170,15 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g
e,
f
),
SYS_MKNS => format!(
"mkns({:p} len: {})",
// TODO: Print out all scheme names?
// Simply printing out simply the pointers and lengths may not provide that much useful
// debugging information, so only print the raw args.
b as *const u8,
c,
),
SYS_MPROTECT => format!("mprotect({:#X}, {}, {:?})", b, c, MapFlags::from_bits(d)),
SYS_MREMAP => format!("mremap({:#X}, {:#X}, {:#X}, {:#X}, {:#X})", b, c, d, e, f),
SYS_NANOSLEEP => format!(
@@ -203,11 +229,12 @@ pub fn debug_start([a, b, c, d, e, f, g]: [usize; 7], token: &mut CleanLockToken
#[expect(clippy::overly_complex_bool_expr)]
#[expect(clippy::needless_bool)]
let do_debug = if true && {
let ctx = crate::context::current();
let guard = ctx.read(token.token());
guard.name.contains("init") || guard.name.contains("bootstrap")
} {
let do_debug = if false
&& crate::context::current()
.read(token.token())
.name
.contains("init")
{
if a == SYS_CLOCK_GETTIME || a == SYS_YIELD || a == SYS_FUTEX {
false
} else if (a == SYS_WRITE || a == SYS_FSYNC) && (b == 1 || b == 2) {
src/syscall/fs.rs
+211 -128
View File
@@ -2,16 +2,14 @@
use core::num::NonZeroUsize;
use alloc::{string::String, sync::Arc, vec::Vec};
use alloc::{string::{String, ToString}, sync::Arc, vec::Vec};
use redox_path::RedoxPath;
use crate::{
context::{
self,
file::{FileDescription, FileDescriptor, InternalFlags, LockedFileDescription},
memory::{
handle_notify_files, AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions,
},
memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions},
},
memory::{Page, VirtualAddress, PAGE_SIZE},
scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes},
@@ -64,12 +62,13 @@ pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result<String>
// TODO: Define elsewhere
const PATH_MAX: usize = PAGE_SIZE;
pub fn openat_into(
pub fn openat(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
fcntl_flags: u32,
new_fd: FileHandle,
euid: u32,
egid: u32,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
@@ -79,11 +78,14 @@ pub fn openat_into(
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?;
let desc = pipe.description.read(token.token());
(desc.scheme, desc.number)
let desc = pipe.description.read(token.token());
(desc.scheme, desc.number)
};
let caller_ctx = context::current().read(token.token()).caller_ctx();
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
let new_description = {
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
@@ -99,15 +101,21 @@ pub fn openat_into(
match res? {
OpenResult::SchemeLocal(number, internal_flags) => {
Arc::new(RwLock::new(FileDescription {
offset: 0,
internal_flags,
scheme: scheme_id,
number,
flags: flags as u32,
}))
FileDescriptor {
description: Arc::new(RwLock::new(FileDescription {
offset: 0,
internal_flags,
scheme: scheme_id,
number,
flags: (flags & !O_CLOEXEC) as u32,
})),
cloexec: flags & O_CLOEXEC == O_CLOEXEC,
}
}
OpenResult::External(desc) => desc,
OpenResult::External(desc) => FileDescriptor {
description: desc,
cloexec: flags & O_CLOEXEC == O_CLOEXEC,
},
}
};
@@ -115,20 +123,65 @@ pub fn openat_into(
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.insert_file(
new_fd,
FileDescriptor {
description: new_description,
},
.add_file(
new_description,
&mut token,
)
.ok_or(Error::new(EEXIST))
.ok_or(Error::new(EMFILE))
}
/// Open a file at a specific path, placing the result into a reserved fd slot (upstream 0.9.0).
pub fn openat_into(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
fcntl_flags: u32,
new_fd: FileHandle,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
let (scheme_id, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?;
let desc = pipe.description.read(token.token());
(desc.scheme, desc.number)
};
let caller_ctx = context::current().read(token.token()).caller_ctx();
let new_description = {
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
let res = scheme.kopenat(number, StrOrBytes::from_str(&path_buf), flags, fcntl_flags, caller_ctx, token)?;
match res {
OpenResult::SchemeLocal(number, internal_flags) => {
FileDescriptor {
description: Arc::new(RwLock::new(FileDescription {
offset: 0,
internal_flags,
scheme: scheme_id,
number,
flags: (flags & !O_CLOEXEC) as u32,
})),
cloexec: flags & O_CLOEXEC == O_CLOEXEC,
}
}
OpenResult::External(desc) => FileDescriptor {
description: desc,
cloexec: flags & O_CLOEXEC == O_CLOEXEC,
},
}
};
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context.insert_file(new_fd, new_description, &mut token).ok_or(Error::new(EEXIST))
}
/// Unlinkat syscall
pub fn unlinkat(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
euid: u32,
egid: u32,
token: &mut CleanLockToken,
) -> Result<()> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
@@ -144,7 +197,10 @@ pub fn unlinkat(
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
let caller_ctx = context::current().read(token.token()).caller_ctx();
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
/*
let mut path_buf = BorrowedHtBuf::head()?;
@@ -170,6 +226,7 @@ pub fn close(fd: FileHandle, token: &mut CleanLockToken) -> Result<()> {
fn duplicate_file(
fd: FileHandle,
user_buf: UserSliceRo,
cloexec: bool,
token: &mut CleanLockToken,
) -> Result<FileDescriptor> {
let (caller_ctx, file) = {
@@ -185,6 +242,7 @@ fn duplicate_file(
if user_buf.is_empty() {
Ok(FileDescriptor {
description: Arc::clone(&file.description),
cloexec,
})
} else {
let description = { *file.description.read(token.token()) };
@@ -208,24 +266,45 @@ fn duplicate_file(
Ok(FileDescriptor {
description: new_description,
cloexec,
})
}
}
/// Duplicate file descriptor
pub fn dup_into(
fd: FileHandle,
new_fd: FileHandle,
buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let new_file = duplicate_file(fd, buf, token)?;
pub fn dup(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<FileHandle> {
let new_file = duplicate_file(fd, buf, false, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.insert_file(new_fd, new_file, &mut token)
.ok_or(Error::new(EEXIST))
.add_file(new_file, &mut token)
.ok_or(Error::new(EMFILE))
}
/// Duplicate a file descriptor, placing into a specific fd slot (upstream 0.9.0).
/// If the target slot is occupied, the existing descriptor is closed first
/// (POSIX `dup3` semantics, consistent with `dup2` below).
pub fn dup_into(
fd: FileHandle,
new_cd: FileHandle,
buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
if fd == new_cd {
return Ok(new_cd);
}
let _ = close(new_cd, token);
let new_file = duplicate_file(fd, buf, false, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.insert_file(new_cd, new_file, &mut token)
.ok_or(Error::new(EMFILE))
}
/// Duplicate file descriptor, replacing another
@@ -239,7 +318,7 @@ pub fn dup2(
Ok(new_fd)
} else {
let _ = close(new_fd, token);
let new_file = duplicate_file(fd, buf, token)?;
let new_file = duplicate_file(fd, buf, false, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
@@ -250,7 +329,7 @@ pub fn dup2(
}
}
pub fn call(
fds: &[usize],
fd: FileHandle,
payload: UserSliceRw,
flags: CallFlags,
metadata: UserSliceRo,
@@ -265,103 +344,42 @@ pub fn call(
match flags {
f if f.contains(CallFlags::WRITE | CallFlags::FD) => {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
call_fdwrite(
FileHandle::from(fds[0]),
payload,
flags,
&meta[..copied / 8],
token,
)
call_fdwrite(fd, payload, flags, &meta[..copied / 8], token)
}
f if f.contains(CallFlags::READ | CallFlags::FD) => {
if fds.len() != 1 {
return Err(Error::new(EINVAL));
}
call_fdread(
FileHandle::from(fds[0]),
payload,
flags,
&meta[..copied / 8],
token,
)
call_fdread(fd, payload, flags, &meta[..copied / 8], token)
}
_ => call_normal(fds, payload, flags, &meta[..copied / 8], token),
_ => call_normal(fd, payload, flags, &meta[..copied / 8], token),
}
}
fn call_normal(
fds: &[usize],
fd: FileHandle,
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
if fds.len() > 2 || fds.is_empty() {
return Err(Error::new(EINVAL));
}
let mut nums = arrayvec::ArrayVec::<_, 2>::new();
let current_lock = context::current();
let consume = flags.contains(CallFlags::CONSUME);
let mut fds = fds.iter();
let (target_file, scheme_id) = {
let fd = FileHandle::from(fds.next().copied().unwrap());
let file = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (file, mut split_token) = match (current.token_split(), consume) {
((ctxt, mut split_token), true) => {
(ctxt.remove_file(fd, &mut split_token), split_token)
}
((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token),
};
let file = file.ok_or(Error::new(EBADF))?;
let desc = file.description.read(split_token.token());
let scheme_id = desc.scheme;
nums.push(desc.number);
drop(desc);
let target_file = file;
(target_file, scheme_id)
};
for &fd in fds {
let fd = FileHandle::from(fd);
let mut current = current_lock.read(token.token());
let (file, mut split_token) = match (current.token_split(), consume) {
((ctxt, mut split_token), true) => {
(ctxt.remove_file(fd, &mut split_token), split_token)
}
((ctxt, mut split_token), false) => (ctxt.get_file(fd, &mut split_token), split_token),
};
let file = file.ok_or(Error::new(EBADF))?;
let desc = file.description.read(split_token.token());
if desc.scheme != scheme_id {
return Err(Error::new(EXDEV));
match (current.token_split(), flags.contains(CallFlags::CONSUME)) {
((ctxt, mut token), true) => ctxt.remove_file(fd, &mut token),
((ctxt, mut token), false) => ctxt.get_file(fd, &mut token),
}
nums.push(desc.number);
}
.ok_or(Error::new(EBADF))?;
let (scheme_id, number) = {
let desc = file.description.read(token.token());
(desc.scheme, desc.number)
};
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
if flags.contains(CallFlags::STD_FS) {
scheme.translate_std_fs_call(
&nums,
target_file.description,
payload,
flags,
metadata,
token,
)
scheme.translate_std_fs_call(number, file.description, payload, flags, metadata, token)
} else {
scheme.kcall(&nums, payload, flags, metadata, token)
scheme.kcall(&[number], payload, flags, metadata, token)
}
}
@@ -382,13 +400,28 @@ fn call_fdwrite(
let len = fds.len();
fdwrite_inner(fd, fds, flags, 0, metadata, token)?;
Ok(len)
}
fn fdwrite_inner(
socket: FileHandle,
target_fds: Vec<FileHandle>,
flags: CallFlags,
arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
// TODO: Ensure deadlocks can't happen
let (scheme, number, descs_to_send) = {
let (scheme, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let file_descriptor = context.get_file(fd, &mut token).ok_or(Error::new(EBADF))?;
let file_descriptor = context
.get_file(socket, &mut token)
.ok_or(Error::new(EBADF))?;
let desc = &file_descriptor.description.read(token.token());
(desc.scheme, desc.number)
};
@@ -401,9 +434,9 @@ fn call_fdwrite(
scheme,
number,
if flags.contains(CallFlags::FD_CLONE) {
context.bulk_get_files(&fds, &mut token)
context.bulk_get_files(&target_fds, &mut token)
} else {
context.bulk_remove_files(&fds, &mut token)
context.bulk_remove_files(&target_fds, &mut token)
}?
.into_iter()
.map(|f| f.description)
@@ -426,9 +459,7 @@ fn call_fdwrite(
CallFlags::empty()
};
scheme.kfdwrite(number, descs_to_send, flags_to_scheme, metadata, token)?;
Ok(len)
scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata, token)
}
fn call_fdread(
@@ -455,6 +486,24 @@ fn call_fdread(
scheme.kfdread(number, payload, flags, metadata, token)
}
pub fn sendfd(
socket: FileHandle,
fd: FileHandle,
flags_raw: usize,
arg: u64,
token: &mut CleanLockToken,
) -> Result<usize> {
let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?;
let mut call_flags = CallFlags::FD | CallFlags::WRITE;
if sendfd_flags.contains(SendFdFlags::CLONE) {
call_flags |= CallFlags::FD_CLONE;
}
if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) {
call_flags |= CallFlags::FD_EXCLUSIVE;
}
fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[], token)
}
/// File descriptor controls
pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) -> Result<usize> {
let file = {
@@ -466,19 +515,19 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
.ok_or(Error::new(EBADF))?;
let (scheme_id, number, flags) = {
let desc = file.description.read(token.token());
let desc = file.description.write(token.token());
(desc.scheme, desc.number, desc.flags)
};
if cmd == F_DUPFD {
if cmd == F_DUPFD || cmd == F_DUPFD_CLOEXEC {
// Not in match because 'files' cannot be locked
let new_file = duplicate_file(fd, UserSlice::empty(), token)?;
let new_file = duplicate_file(fd, UserSlice::empty(), cmd == F_DUPFD_CLOEXEC, token)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
return context
.insert_file(FileHandle::from(arg), new_file, &mut token)
.add_file_min(new_file, arg, &mut token)
.ok_or(Error::new(EMFILE))
.map(FileHandle::into);
}
@@ -500,8 +549,17 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
let (files, mut token) = files.token_split();
match *files.get_mut(fd.get()).ok_or(Error::new(EBADF))? {
Some(ref mut file) => match cmd {
F_GETFD => Ok(0),
F_SETFD => Ok(0),
F_GETFD => {
if file.cloexec {
Ok(O_CLOEXEC)
} else {
Ok(0)
}
}
F_SETFD => {
file.cloexec = arg & O_CLOEXEC == O_CLOEXEC;
Ok(0)
}
F_GETFL => Ok(flags as usize),
F_SETFL => {
let new_flags = (flags & O_ACCMODE as u32) | (arg as u32 & !O_ACCMODE as u32);
@@ -630,7 +688,10 @@ pub fn funmap(virtual_address: usize, length: usize, token: &mut CleanLockToken)
.ok_or(Error::new(EINVAL))?;
let unpin = false;
let notify = addr_space.munmap(span, unpin, token)?;
handle_notify_files(notify, token);
for map in notify {
let _ = map.unmap(token);
}
Ok(0)
}
@@ -756,7 +817,7 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) ->
let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) {
desc.offset
} else {
u64::MAX
0
};
Ok((
scheme.kreadoff(desc.number, buf, offset, desc.flags, desc.flags, token)?,
@@ -768,18 +829,32 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) ->
let offset = &mut desc_arc.write(token.token()).offset;
*offset = offset.saturating_add(bytes_read as u64)
}
// I/O accounting: increment rchar and syscr on current context.
// Cross-referenced with Linux 7.1 mm/filemap.c:filemap_get_pages
// and task_io_account_read().
let current = context::current();
let mut guard = current.write(token.token());
guard.io_rchar = guard.io_rchar.saturating_add(bytes_read as u64);
guard.io_syscr = guard.io_syscr.saturating_add(1);
guard.io_read_bytes = guard.io_read_bytes.saturating_add(bytes_read as u64);
Ok(bytes_read)
}
pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
let result = (|| {
let (bytes_written, desc_arc, desc) =
file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| {
let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) {
desc.offset
} else {
u64::MAX
0
};
let res = scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags, token);
if res.is_err() {
let pid = context::current().read(token.token()).pid;
error!("SYS_WRITE_ERR: pid={} fd={} scheme={} number={} offset={} err={:?}", pid, fd.get(), desc.scheme.get(), desc.number, offset, res.as_ref().err());
}
Ok((
scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags, token)?,
res?,
desc_arc,
desc,
))
@@ -788,5 +863,13 @@ pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -
let offset = &mut desc_arc.write(token.token()).offset;
*offset = offset.saturating_add(bytes_written as u64)
}
// I/O accounting: increment wchar and syscw on current context.
let current = context::current();
let mut guard = current.write(token.token());
guard.io_wchar = guard.io_wchar.saturating_add(bytes_written as u64);
guard.io_syscw = guard.io_syscw.saturating_add(1);
guard.io_write_bytes = guard.io_write_bytes.saturating_add(bytes_written as u64);
Ok(bytes_written)
})();
result
}
src/syscall/futex.rs
+178 -28
View File
@@ -19,12 +19,13 @@ use crate::{
},
memory::{Page, PhysicalAddress, VirtualAddress},
sync::{CleanLockToken, Mutex, L1},
time,
};
use crate::syscall::{
data::TimeSpec,
data::{FutexWaitv, TimeSpec},
error::{Error, Result, EAGAIN, EFAULT, EINVAL, ETIMEDOUT},
flag::{FUTEX_WAIT, FUTEX_WAIT64, FUTEX_WAKE},
flag::{FUTEX_REQUEUE, FUTEX_WAIT, FUTEX_WAIT64, FUTEX_WAIT_MULTIPLE, FUTEX_WAKE},
};
use super::usercopy::UserSlice;
@@ -36,12 +37,12 @@ type FutexList = HashMap<PhysicalAddress, Vec<FutexEntry>>;
pub struct FutexEntry {
// Virtual address, required if synchronizing across the same address space, if the memory is
// CoW.
// TODO: FUTEX_REQUEUE
target_virtaddr: VirtualAddress,
// Context to wake up, and compare address spaces.
context_lock: Weak<ContextLock>,
context_lock: Arc<ContextLock>,
// address space to check against if virt matches but not phys
addr_space: Weak<AddrSpaceWrapper>,
waitv_index: Option<usize>,
}
// TODO: Process-private futexes? In that case, put the futex table in each AddrSpace, or just
@@ -51,17 +52,6 @@ pub struct FutexEntry {
static FUTEXES: Mutex<L1, FutexList> =
Mutex::new(FutexList::with_hasher(DefaultHashBuilder::new()));
pub fn get_futex_stat(token: &mut CleanLockToken) -> (usize, usize) {
let mut regc = 0;
let mut regl = 0;
let registry = FUTEXES.lock(token.token());
for (_, v) in registry.iter() {
regl += v.len();
regc += 1;
}
(regc, regl)
}
fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option<PhysicalAddress> {
// TODO: Move this elsewhere!
if addr.data().saturating_add(size_of::<usize>()) >= crate::USER_END_OFFSET {
@@ -81,7 +71,7 @@ pub fn futex(
op: usize,
val: usize,
val2: usize,
_addr2: usize,
addr2: usize,
token: &mut CleanLockToken,
) -> Result<usize> {
let current_addrsp = AddrSpace::current()?;
@@ -95,7 +85,6 @@ pub fn futex(
.ok_or(Error::new(EFAULT))?;
match op {
// TODO: FUTEX_WAIT_MULTIPLE?
FUTEX_WAIT | FUTEX_WAIT64 => {
let timeout_opt = UserSlice::ro(val2, size_of::<TimeSpec>())?
.none_if_null()
@@ -153,7 +142,9 @@ pub fn futex(
{
let mut context = context_lock.write(token.token());
context.wake = timeout_opt.map(|time| time.to_nanos());
context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| {
tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128
});
if let Some((tctl, pctl, _)) = context.sigcontrol()
&& tctl.currently_pending_unblocked(pctl) != 0
{
@@ -168,8 +159,9 @@ pub fn futex(
.or_insert_with(Vec::new)
.push(FutexEntry {
target_virtaddr,
context_lock: Arc::downgrade(&context_lock),
context_lock: context_lock.clone(),
addr_space: Arc::downgrade(&current_addrsp),
waitv_index: None,
});
}
@@ -206,18 +198,25 @@ pub fn futex(
if futex.target_virtaddr != target_virtaddr
|| !current_addrsp_weak.ptr_eq(&futex.addr_space)
{
if futex.addr_space.strong_count() == 0 {
futexes.swap_remove(i);
} else {
i += 1;
}
i += 1;
continue;
}
if let Some(ctx) = futex.context_lock.upgrade() {
ctx.write(token.token()).unblock();
let should_wake = {
let mut context = futex.context_lock.write(token.token());
if let Some(index) = futex.waitv_index {
if context.futex_wake_index.is_none() {
context.futex_wake_index = Some(index);
}
}
context.unblock();
true
};
if should_wake {
futexes.swap_remove(i);
woken += 1;
} else {
i += 1;
}
futexes.swap_remove(i);
woken += 1;
}
futexes.is_empty()
@@ -231,6 +230,157 @@ pub fn futex(
Ok(woken)
}
FUTEX_REQUEUE => {
// Linux 7.1 kernel/futex/requeue.c futex_requeue().
// Wake up to val waiters on primary futex, requeue up to
// val2 waiters to secondary futex (addr2).
let mut woken = 0;
let mut requeued = 0;
let target_virtaddr2 = VirtualAddress::new(addr2);
let target_physaddr2 = validate_and_translate_virt(&addr_space_guard, target_virtaddr2)
.ok_or(Error::new(EFAULT))?;
{
drop(addr_space_guard);
let mut futexes_map = FUTEXES.lock(token.token());
let (futexes_map, mut token) = futexes_map.token_split();
if let Some(futexes) = futexes_map.get_mut(&target_physaddr) {
let mut i = 0;
let current_addrsp_weak = Arc::downgrade(&current_addrsp);
let mut to_requeue = Vec::new();
while i < futexes.len() {
let futex = unsafe { futexes.get_unchecked_mut(i) };
if futex.target_virtaddr != target_virtaddr
|| !current_addrsp_weak.ptr_eq(&futex.addr_space)
{
i += 1;
continue;
}
if woken < val {
futex.context_lock.write(token.token()).unblock();
futexes.remove(i);
woken += 1;
} else if requeued < val2 {
let entry = futexes.remove(i);
to_requeue.push(entry);
requeued += 1;
} else {
i += 1;
}
}
if futexes.is_empty() {
futexes_map.remove(&target_physaddr);
}
if !to_requeue.is_empty() {
futexes_map
.entry(target_physaddr2)
.or_insert_with(Vec::new)
.extend(to_requeue);
}
}
}
Ok(woken + requeued)
}
FUTEX_WAIT_MULTIPLE => {
// Wait on multiple futexes atomically; return the index of the first
// one that is woken. Semantics: val = number of FutexWaitv entries,
// addr2 = pointer to the array, val2 = optional TimeSpec timeout.
let count = val;
if count == 0 || count > 128 {
return Err(Error::new(EINVAL));
}
let timeout_opt = UserSlice::ro(val2, size_of::<TimeSpec>())?
.none_if_null()
.map(|buf| unsafe { buf.read_exact::<TimeSpec>() })
.transpose()?;
let array_size = size_of::<FutexWaitv>()
.checked_mul(count)
.ok_or(Error::new(EINVAL))?;
let waitv_slice = UserSlice::ro(addr2, array_size)?;
let mut waitv = Vec::with_capacity(count);
waitv.resize(count, FutexWaitv::default());
let waitv_bytes = unsafe {
core::slice::from_raw_parts_mut(
waitv.as_mut_ptr() as *mut u8,
array_size,
)
};
waitv_slice.copy_to_slice(waitv_bytes)?;
let context_lock = context::current();
let mut translated = Vec::with_capacity(count);
for entry in &waitv {
if entry.flags != 0 {
return Err(Error::new(EINVAL));
}
if !entry.uaddr.is_multiple_of(4) {
return Err(Error::new(EINVAL));
}
let virt = VirtualAddress::new(entry.uaddr);
let phys = validate_and_translate_virt(&addr_space_guard, virt)
.ok_or(Error::new(EFAULT))?;
let accessible_addr = crate::memory::RmmA::phys_to_virt(phys).data();
let fetched = u64::from(unsafe {
(*(accessible_addr as *const AtomicU32)).load(Ordering::SeqCst)
});
if fetched != entry.val as u64 {
return Err(Error::new(EAGAIN));
}
translated.push((virt, phys));
}
{
let mut token = unsafe { CleanLockToken::new() };
let mut futexes = FUTEXES.lock(token.token());
let (futexes, mut token) = futexes.token_split();
{
let mut context = context_lock.write(token.token());
context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| {
tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128
});
context.futex_wake_index = None;
if let Some((tctl, pctl, _)) = context.sigcontrol()
&& tctl.currently_pending_unblocked(pctl) != 0
{
return Err(Error::new(EINTR));
}
context.block("futex_wait_multiple");
}
for (i, (virt, phys)) in translated.into_iter().enumerate() {
futexes
.entry(phys)
.or_insert_with(Vec::new)
.push(FutexEntry {
target_virtaddr: virt,
context_lock: context_lock.clone(),
addr_space: Arc::downgrade(&current_addrsp),
waitv_index: Some(i),
});
}
}
drop(addr_space_guard);
context::switch(token);
let context = context_lock.read(token.token());
if context.wake.is_none() && timeout_opt.is_some() {
Err(Error::new(ETIMEDOUT))
} else {
Ok(context.futex_wake_index.unwrap_or(0))
}
}
_ => Err(Error::new(EINVAL)),
}
}
src/syscall/mod.rs
+151 -91
View File
@@ -4,6 +4,8 @@
extern crate syscall;
use syscall::{dirent::DirentHeader, CallFlags, RwFlags, EINVAL};
pub use self::syscall::{
data, error, flag, io, number, ptrace_event, EnvRegisters, FloatRegisters, IntRegisters,
};
@@ -13,14 +15,14 @@ pub use self::{fs::*, futex::futex, process::*, time::*, usercopy::validate_regi
use self::{
data::{Map, TimeSpec},
debug::{debug_end, debug_start},
error::{Error, Result, EINVAL, ENOSYS},
flag::{CallFlags, EventFlags, MapFlags, RwFlags},
error::{Error, Result, ENOSYS},
flag::{EventFlags, MapFlags},
number::*,
usercopy::UserSlice,
};
use crate::{
context::memory::AddrSpace,
context::{self, memory::AddrSpace},
percpu::PercpuBlock,
scheme::{memory::MemoryScheme, FileHandle},
sync::CleanLockToken,
@@ -71,26 +73,34 @@ pub fn syscall(
let fd = FileHandle::from(b);
//SYS_* is declared in kernel/syscall/src/number.rs
match a {
SYS_WRITE2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
} else {
Some(
u32::try_from(f)
.ok()
.and_then(RwFlags::from_bits)
.ok_or(Error::new(EINVAL))?,
SYS_WRITE2 => {
let bytes = file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
} else {
Some(
u32::try_from(f)
.ok()
.and_then(RwFlags::from_bits)
.ok_or(Error::new(EINVAL))?,
)
};
scheme.kwriteoff(
desc.number,
UserSlice::ro(c, d)?,
e as u64,
flags.map_or(desc.flags, |f| desc.rw_flags(f)),
desc.flags,
token,
)
};
scheme.kwriteoff(
desc.number,
UserSlice::ro(c, d)?,
e as u64,
flags.map_or(desc.flags, |f| desc.rw_flags(f)),
desc.flags,
token,
)
}),
})?;
let current = context::current();
let mut guard = current.write(token.token());
guard.io_wchar = guard.io_wchar.saturating_add(bytes as u64);
guard.io_syscw = guard.io_syscw.saturating_add(1);
guard.io_write_bytes = guard.io_write_bytes.saturating_add(bytes as u64);
Ok(bytes)
}
SYS_WRITE => sys_write(fd, UserSlice::ro(c, d)?, token),
SYS_FMAP => {
let addrspace = AddrSpace::current()?;
@@ -103,43 +113,88 @@ pub fn syscall(
})
}
}
SYS_READ2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
} else {
Some(
u32::try_from(f)
.ok()
.and_then(RwFlags::from_bits)
.ok_or(Error::new(EINVAL))?,
)
};
scheme.kreadoff(
desc.number,
UserSlice::wo(c, d)?,
e as u64,
flags.map_or(desc.flags, |f| desc.rw_flags(f)),
desc.flags,
token,
)
SYS_GETDENTS => {
let header_size = u16::try_from(e).map_err(|_| Error::new(EINVAL))?;
if usize::from(header_size) != size_of::<DirentHeader>() {
// TODO: allow? If so, zero_out must be implemented for UserSlice
return Err(Error::new(EINVAL));
}
file_op_generic(fd, token, |scheme, number, token| {
scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64, token)
})
}
SYS_FUTIMENS => file_op_generic(fd, token, |scheme, number, token| {
scheme.kfutimens(number, UserSlice::ro(c, d)?, token)
}),
SYS_READ2 => {
let bytes = file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
} else {
Some(
u32::try_from(f)
.ok()
.and_then(RwFlags::from_bits)
.ok_or(Error::new(EINVAL))?,
)
};
scheme.kreadoff(
desc.number,
UserSlice::wo(c, d)?,
e as u64,
flags.map_or(desc.flags, |f| desc.rw_flags(f)),
desc.flags,
token,
)
})?;
let current = context::current();
let mut guard = current.write(token.token());
guard.io_rchar = guard.io_rchar.saturating_add(bytes as u64);
guard.io_syscr = guard.io_syscr.saturating_add(1);
guard.io_read_bytes = guard.io_read_bytes.saturating_add(bytes as u64);
Ok(bytes)
}
SYS_READ => sys_read(fd, UserSlice::wo(c, d)?, token),
SYS_FPATH => file_op_generic(fd, token, |scheme, number, token| {
scheme.kfpath(number, UserSlice::wo(c, d)?, token)
}),
// TODO: Can't replace yet with std_fs_call, as fstat overrides device ID, but that can
// be moved to UserScheme.
SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?, token).map(|()| 0),
SYS_FSTATVFS => file_op_generic(fd, token, |scheme, number, token| {
scheme
.kfstatvfs(number, UserSlice::wo(c, d)?, token)
.map(|()| 0)
}),
SYS_DUP_INTO => {
dup_into(fd, FileHandle::from(e), UserSlice::ro(c, d)?, token).map(FileHandle::into)
}
SYS_DUP => dup(fd, UserSlice::ro(c, d)?, token).map(FileHandle::into),
SYS_DUP_INTO => dup_into(
fd,
FileHandle::from(e),
UserSlice::ro(c, d)?,
token,
).map(FileHandle::into),
SYS_DUP2 => {
dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?, token).map(FileHandle::into)
}
#[cfg(target_pointer_width = "32")]
SYS_SENDFD => sendfd(
fd,
FileHandle::from(c),
d,
e as u64 | ((f as u64) << 32),
token,
),
#[cfg(target_pointer_width = "64")]
SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64, token),
SYS_LSEEK => lseek(fd, c as i64, d, token),
SYS_FCHMOD => file_op_generic(fd, token, |scheme, number, token| {
scheme.fchmod(number, c as u16, token).map(|()| 0)
}),
SYS_FCHOWN => file_op_generic(fd, token, |scheme, number, token| {
scheme.fchown(number, c as u32, d as u32, token).map(|()| 0)
}),
@@ -153,60 +208,59 @@ pub fn syscall(
SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?, token).map(|()| 0),
SYS_FUNMAP => funmap(b, c, token),
// TODO: This can't be removed yet, since the pre-libredox softbuffer crate is a blocker.
SYS_FSYNC => {
//let ctxt_name = crate::context::current().read(token.token()).name;
//warn!("Context `{ctxt_name}` is using deprecated SYS_FSYNC");
file_op_generic(fd, token, |scheme, number, token| {
scheme.fsync(number, token).map(|()| 0)
})
SYS_FSYNC => file_op_generic(fd, token, |scheme, number, token| {
scheme.fsync(number, token).map(|()| 0)
}),
SYS_SYNC => {
// Red Bear filesystem I/O is synchronous — all data is
// already on disk. sync() is a no-op per POSIX semantics.
// Cross-referenced with Linux 7.1 fs/sync.c ksys_sync().
Ok(0)
}
SYS_SYNCFS => file_op_generic(fd, token, |scheme, number, token| {
// syncfs(fd) syncs the filesystem containing fd.
// On Red Bear, I/O is synchronous so this is a no-op.
scheme.fsync(number, token).map(|()| 0)
}),
// TODO: 64-bit lengths on 32-bit platforms
SYS_FTRUNCATE => file_op_generic(fd, token, |scheme, number, token| {
scheme.ftruncate(number, c, token).map(|()| 0)
}),
SYS_CLOSE => close(fd, token).map(|()| 0),
SYS_CALL => {
let flags = CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?;
if flags.contains(CallFlags::MULTIPLE_FDS) {
if g / core::mem::size_of::<usize>() > 16 {
return Err(Error::new(EINVAL));
};
let mut fds = [0_usize; 16];
let fds_slice = UserSlice::ro(b, g)?;
// TODO: bytemuck/plain
let copied = fds_slice.copy_common_bytes_to_slice(unsafe {
core::slice::from_raw_parts_mut(
fds.as_mut_ptr().cast(),
fds.len() * core::mem::size_of::<usize>(),
)
})?;
call(
&fds[..copied / core::mem::size_of::<usize>()],
UserSlice::rw(c, d)?,
flags,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
)
} else {
call(
&[b],
UserSlice::rw(c, d)?,
flags,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
)
}
SYS_CALL => call(
fd,
UserSlice::rw(c, d)?,
CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
),
SYS_OPENAT => {
openat(fd, UserSlice::ro(c, d)?, e, f as _, 0, 0, token).map(FileHandle::into)
}
SYS_OPENAT_INTO => openat_into(
fd,
UserSlice::ro(c, d)?,
e,
f as _,
FileHandle::from(g),
f as u32,
FileHandle::from(g as usize),
token,
)
.map(FileHandle::into),
SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, token).map(|()| 0),
SYS_OPENAT_WITH_FILTER => openat(
fd,
UserSlice::ro(c, d)?,
e,
(e & syscall::O_FCNTL_MASK) as _,
f as _,
g as _,
token,
)
.map(FileHandle::into),
SYS_UNLINKAT => unlinkat(fd, UserSlice::ro(c, d)?, e, 0, 0, token).map(|()| 0),
SYS_UNLINKAT_WITH_FILTER => {
unlinkat(fd, UserSlice::ro(c, d)?, e, f as _, g as _, token).map(|()| 0)
}
SYS_YIELD => sched_yield(token).map(|()| 0),
SYS_NANOSLEEP => nanosleep(
UserSlice::ro(b, size_of::<TimeSpec>())?,
@@ -221,7 +275,13 @@ pub fn syscall(
SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d), token).map(|()| 0),
SYS_MREMAP => mremap(b, c, d, e, f, token),
_ => Err(Error::new(ENOSYS)),
SYS_MKNS => sys_mkns(UserSlice::ro(b, c)?, token).map(FileHandle::into),
SYS_SETNS => sys_setns(fd, token).map(|()| 0),
_ => {
println!("KERNEL: unimplemented syscall a={:#x} b={:#x} c={:#x} d={:#x}", a, b, c, d);
Err(Error::new(ENOSYS))
}
}
}
src/syscall/process.rs
+124 -33
View File
@@ -20,41 +20,45 @@ use crate::{
context::{self, context::FdTbl},
memory::{Page, VirtualAddress, PAGE_SIZE},
scheme::{
FileHandle, KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES,
KERNEL_SCHEMES_COUNT,
KernelScheme, SchemeExt, SchemeId, SchemeList, ALL_KERNEL_SCHEMES, KERNEL_SCHEMES_COUNT,
},
startup::Bootstrap,
syscall::{error::*, flag::MapFlags},
CurrentRmmArch,
};
use super::usercopy::UserSliceWo;
use super::usercopy::{UserSliceRo, UserSliceWo};
use crate::scheme::FileHandle;
pub fn exit_this_context(excp: Option<syscall::Exception>, token: &mut CleanLockToken) -> ! {
let mut close_files;
let addrspace_opt;
let context_lock = context::current();
let (addrspace_opt, mut close_files) = {
{
let mut context = context_lock.write(token.token());
let (context, token) = context.token_split();
let close_files = Arc::try_unwrap(mem::take(&mut context.files))
// let (context, mut token) = context.token_split();
close_files = Arc::try_unwrap(mem::take(&mut context.files))
.map_or_else(|_| FdTbl::new(), RwLock::into_inner);
let addrspace_opt = context.set_addr_space(None, token);
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
addrspace_opt = context
.set_addr_space(None, token.downgrade())
.and_then(|a| Arc::try_unwrap(a).ok());
drop(mem::replace(&mut context.syscall_head, SyscallFrame::Dummy));
drop(mem::replace(&mut context.syscall_tail, SyscallFrame::Dummy));
(addrspace_opt, close_files)
};
}
// Files must be closed while context is valid so that messages can be passed
close_files.force_close_all(token);
if let Some(addrspace) = addrspace_opt {
if let Ok(addrspace) = Arc::try_unwrap(addrspace) {
addrspace.into_drop(token);
}
addrspace.into_drop(token);
}
// TODO: Should status == Status::HardBlocked be handled differently?
let owner = {
let mut guard = context_lock.write(token.token());
guard.status = context::Status::Dead { excp };
guard.owner_proc_id
};
if let Some(owner) = owner {
@@ -73,7 +77,6 @@ pub fn exit_this_context(excp: Option<syscall::Exception>, token: &mut CleanLock
}
}
}
drop(close_files);
context::switch(token);
unreachable!();
}
@@ -92,6 +95,49 @@ pub fn mprotect(
AddrSpace::current()?.mprotect(span, flags, token)
}
/// Create a new namespace containing the named schemes (space-separated in `names`).
///
/// Returns a file handle that refers to the new namespace; the caller is expected to
/// pass it to `sys_setns` to switch into it, or to `redox-rt::sys::setns` for a
/// pure-userspace swap of the per-thread dynamic proc info `ns_fd`.
///
/// Implementation: dups the calling context's namespace fd with the caller-supplied
/// buffer (which contains `NsDup::ForkNs` followed by scheme names). The dup is
/// handled by the bootstrap's userspace namespace manager daemon (`initnsmgr`),
/// which creates the new namespace and returns a new fd.
pub fn sys_mkns(
names: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let ns_fd_opt = {
let ctx_lock = context::current();
let ctx = ctx_lock.read(token.token());
ctx.ns_fd
};
let ns_fd = ns_fd_opt.ok_or(Error::new(EBADF))?;
crate::syscall::fs::dup(FileHandle::from(ns_fd), names, token)
}
/// Switch the calling context to the namespace referenced by `fd`.
///
/// The fd must have been produced by `sys_mkns` (or by the bootstrap's namespace
/// manager). After this syscall returns, all scheme lookups in the calling
/// context resolve against the new namespace.
pub fn sys_setns(
fd: FileHandle,
token: &mut CleanLockToken,
) -> Result<()> {
let ns_fd = fd.get();
if ns_fd == 0 || ns_fd == usize::MAX {
return Err(Error::new(EBADF));
}
let ctx_lock = context::current();
let mut ctx = ctx_lock.write(token.token());
ctx.ns_fd = Some(ns_fd);
Ok(())
}
const KERNEL_METADATA_BASE: usize = crate::USER_END_OFFSET - syscall::KERNEL_METADATA_SIZE;
const KERNEL_METADATA_PAGE_COUNT: usize = syscall::KERNEL_METADATA_SIZE / PAGE_SIZE + {
if syscall::KERNEL_METADATA_SIZE.is_multiple_of(PAGE_SIZE) {
@@ -155,9 +201,15 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
inner.fd = {
let cap_fd = match scheme.as_scheme().scheme_root(token) {
Ok(fd) => fd,
Err(_) => usize::MAX,
Err(_) => usize::MAX
};
insert_fd(scheme.scheme_id(), cap_fd, token)
let inserted = insert_fd(
scheme.scheme_id(),
cap_fd,
matches!(scheme, GlobalSchemes::Proc),
token,
);
inserted
};
}
}
@@ -170,7 +222,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
};
// Second, retrieve the scheme ID.
let scheme_id = &SchemeList.id();
insert_fd(*scheme_id, cap_fd, token)
insert_fd(*scheme_id, cap_fd, false, token)
};
let mut lock_token = token.token();
@@ -239,10 +291,57 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
.expect("failed to copy memory to bootstrap");
let bootstrap_entry = u64::from_le_bytes(bootstrap_slice[0x1a..0x22].try_into().unwrap());
debug!("Bootstrap entry point: {:X}", bootstrap_entry);
assert_ne!(bootstrap_entry, 0);
// Start in a minimal environment without any stack.
// Map a minimal user stack for the bootstrap process. Without this,
// RSP defaults to 0 from InterruptStack::init() and the first `push` or
// `call` in the binary triggers a page fault at address 0, which the
// C startup turns into `ud2` (Rust panic handler) — visible as
// "Invalid opcode fault" at a low RIP before main() ever runs.
//
// The stack is mapped just above the initfs image in user virtual
// memory. The top-of-stack (RSP initial value) points to the END of
// the mapped region, since x86-64 stacks grow downward.
const BOOTSTRAP_STACK_PAGES: usize = 8; // 32 KiB — enough for Rust _start
let stack_base_vaddr = PAGE_SIZE + bootstrap.page_count * PAGE_SIZE;
let stack_top_vaddr = stack_base_vaddr + BOOTSTRAP_STACK_PAGES * PAGE_SIZE;
{
let addr_space = Arc::clone(
context::current()
.read(token.token())
.addr_space()
.expect("expected bootstrap context to have an address space"),
);
let page_count =
NonZeroUsize::new(BOOTSTRAP_STACK_PAGES).expect("BOOTSTRAP_STACK_PAGES must be non-zero");
let stack_base = Page::containing_address(VirtualAddress::new(stack_base_vaddr));
let _base_page = {
let mut lock_token = token.token();
let mut addr_space_lock = addr_space.acquire_write(lock_token.downgrade());
let _ = addr_space_lock
.mmap(
&addr_space,
Some(stack_base),
page_count,
MapFlags::MAP_FIXED_NOREPLACE | MapFlags::PROT_READ | MapFlags::PROT_WRITE,
None,
|page, flags, mapper, flusher| {
let span = PageSpan::new(page, BOOTSTRAP_STACK_PAGES);
Ok(Grant::zeroed(
span,
flags,
mapper,
flusher,
false, // private mapping for bootstrap stack
)?)
},
);
};
debug!(
"Bootstrap stack mapped at {:#x}..{:#x} ({} pages)",
stack_base_vaddr, stack_top_vaddr, BOOTSTRAP_STACK_PAGES
);
}
let ctx = context::current();
let mut lock = ctx.write(token.token());
@@ -252,8 +351,9 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
{
regs.init();
regs.set_instr_pointer(bootstrap_entry.try_into().unwrap());
regs.set_stack_pointer(stack_top_vaddr);
}
}
}
unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] {
unsafe {
@@ -264,23 +364,12 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8]
}
}
fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usize {
fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
context
.files
.write(token.token())
.resize(0, 64)
.expect("failed to resize lower fdtbl");
context
.files
.write(token.token())
.resize(syscall::flag::UPPER_FDTBL_TAG, 64)
.expect("failed to resize upper fdtbl");
context
.insert_file(
FileHandle::from(syscall::flag::UPPER_FDTBL_TAG | scheme.get()),
.add_file_min(
FileDescriptor {
description: Arc::new(RwLock::new(FileDescription {
scheme,
@@ -289,7 +378,9 @@ fn insert_fd(scheme: SchemeId, number: usize, token: &mut CleanLockToken) -> usi
flags: (O_CREAT | O_RDWR) as u32,
internal_flags: InternalFlags::empty(),
})),
cloexec,
},
syscall::flag::UPPER_FDTBL_TAG + scheme.get(),
&mut token,
)
.expect("failed to insert fd to current context")
src/syscall/time.rs
+15 -6
View File
@@ -18,7 +18,10 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo, token: &mut CleanLockToken)
_ => return Err(Error::new(EINVAL)),
};
buf.copy_exactly(&TimeSpec::from_nanos(arch_time))
buf.copy_exactly(&TimeSpec {
tv_sec: (arch_time / time::NANOS_PER_SEC) as i64,
tv_nsec: (arch_time % time::NANOS_PER_SEC) as i32,
})
}
/// Nanosleep will sleep by switching the current context
@@ -34,18 +37,18 @@ pub fn nanosleep(
}
let start = time::monotonic(token);
let end = start + req.to_nanos();
let end = start + (req.tv_sec as u128 * time::NANOS_PER_SEC) + (req.tv_nsec as u128);
let current_context = context::current();
{
let context = current_context.upgradeable_read(token.token());
let mut context = current_context.write(token.token());
if let Some((tctl, pctl, _)) = context.sigcontrol()
&& tctl.currently_pending_unblocked(pctl) != 0
{
return Err(Error::new(EINTR));
}
let mut context = context.upgrade();
context.wake = Some(end);
context.block("nanosleep");
}
@@ -61,9 +64,15 @@ pub fn nanosleep(
rem_buf.copy_exactly(&if current < end {
let diff = end - current;
TimeSpec::from_nanos(diff)
TimeSpec {
tv_sec: (diff / time::NANOS_PER_SEC) as i64,
tv_nsec: (diff % time::NANOS_PER_SEC) as i32,
}
} else {
TimeSpec::default()
TimeSpec {
tv_sec: 0,
tv_nsec: 0,
}
})?;
}
targets/aarch64-unknown-kernel.json
-1
View File
@@ -17,7 +17,6 @@
"os": "none",
"position-independent-executables": false,
"relocation-model": "pic",
"rustc-abi": "softfloat",
"target-c-int-width": 32,
"target-endian": "little",
"target-pointer-width": 64,