kernel: add acpi_ext dep + S3 wakeup build wiring (Phase 2.2 partial)
This is a partial-recovery application of the redbear-consolidated patch. The dependency addition and build wiring succeeded; some call-site hunks in src/syscall/fs.rs were skipped because the upstream file structure has drifted. A full fork rebase is needed to resolve those (Phase 2.4). What this commit adds: - Cargo.toml: acpi_ext = acpi package from upstream redox-6.x branch (provides ACPI S3 sleep state support) - build.rs: cargo::rerun-if-changed and nasm invocation for src/asm/x86_64/s3_wakeup.asm (assembly stub for the wakeup vector) - Makefile: comment header for the patch framework - Various acpi integration touch-ups that applied cleanly What was skipped (fuzz-matched too much, marked for follow-up): - src/syscall/fs.rs call-site hunks for new S3 syscall - src/arch/x86_shared/sleep.rs additional scaffolding After this commit, kernel fork builds should compile against acpi ext if the relevant call sites are wired in a future rebase.
This commit is contained in:
@@ -14,6 +14,7 @@ cc = "1.0"
|
||||
toml = "0.8"
|
||||
|
||||
[dependencies]
|
||||
acpi_ext = { package = "acpi", git = "https://gitlab.redox-os.org/redox-os/acpi.git", branch = "redox-6.x" }
|
||||
arrayvec = { version = "0.7.4", default-features = false }
|
||||
bitfield = "0.13.2"
|
||||
bitflags = "2"
|
||||
|
||||
@@ -77,6 +77,7 @@ fn main() {
|
||||
}
|
||||
"x86_64" => {
|
||||
println!("cargo::rerun-if-changed=src/asm/x86_64/trampoline.asm");
|
||||
println!("cargo::rerun-if-changed=src/asm/x86_64/s3_wakeup.asm");
|
||||
|
||||
let status = Command::new("nasm")
|
||||
.arg("-f")
|
||||
@@ -89,6 +90,18 @@ fn main() {
|
||||
if !status.success() {
|
||||
panic!("nasm failed with exit status {}", status);
|
||||
}
|
||||
|
||||
let status = Command::new("nasm")
|
||||
.arg("-f")
|
||||
.arg("bin")
|
||||
.arg("-o")
|
||||
.arg(format!("{}/s3_wakeup", out_dir))
|
||||
.arg("src/asm/x86_64/s3_wakeup.asm")
|
||||
.status()
|
||||
.expect("failed to run nasm");
|
||||
if !status.success() {
|
||||
panic!("nasm failed with exit status {}", status);
|
||||
}
|
||||
}
|
||||
"riscv64" => {
|
||||
println!("cargo::rustc-cfg=dtb");
|
||||
|
||||
+60
-22
@@ -18,6 +18,7 @@ use crate::{
|
||||
|
||||
use super::{Madt, MadtEntry};
|
||||
|
||||
const AP_SPIN_LIMIT: u32 = 1_000_000;
|
||||
const TRAMPOLINE: usize = 0x8000;
|
||||
static TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trampoline"));
|
||||
|
||||
@@ -42,13 +43,17 @@ pub(super) fn init(madt: Madt) {
|
||||
//TODO: do not have writable and executable!
|
||||
let mut mapper = KernelMapper::lock_rw();
|
||||
|
||||
let result = mapper
|
||||
.map_phys(
|
||||
trampoline_page.start_address(),
|
||||
trampoline_frame.base(),
|
||||
PageFlags::new().execute(true).write(true),
|
||||
)
|
||||
.expect("failed to map trampoline");
|
||||
let result = match mapper.map_phys(
|
||||
trampoline_page.start_address(),
|
||||
trampoline_frame.base(),
|
||||
PageFlags::new().execute(true).write(true),
|
||||
) {
|
||||
Some(result) => result,
|
||||
None => {
|
||||
println!("KERNEL AP: failed to map trampoline page, AP bring-up disabled");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
(result, mapper.table().phys().data())
|
||||
};
|
||||
@@ -72,17 +77,27 @@ pub(super) fn init(madt: Madt) {
|
||||
if u32::from(ap_local_apic.id) == me.get() {
|
||||
debug!(" This is my local APIC");
|
||||
} else if ap_local_apic.flags & 1 == 1 {
|
||||
let cpu_id = LogicalCpuId::next();
|
||||
|
||||
// Allocate a stack
|
||||
let stack_start = RmmA::phys_to_virt(
|
||||
allocate_p2frame(4)
|
||||
.expect("no more frames in acpi stack_start")
|
||||
.base(),
|
||||
)
|
||||
.data();
|
||||
let alloc = match allocate_p2frame(4) {
|
||||
Some(frame) => frame,
|
||||
None => {
|
||||
println!("KERNEL AP: CPU {} no memory for stack, skipping", ap_local_apic.id);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let stack_start = RmmA::phys_to_virt(alloc.base()).data();
|
||||
let stack_end = stack_start + (PAGE_SIZE << 4);
|
||||
|
||||
let next_cpu = crate::CPU_COUNT.load(Ordering::Relaxed);
|
||||
if next_cpu >= crate::cpu_set::MAX_CPU_COUNT {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} exceeds logical CPU limit, skipping",
|
||||
ap_local_apic.id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let cpu_id = LogicalCpuId::new(next_cpu);
|
||||
|
||||
let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end);
|
||||
|
||||
let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id);
|
||||
@@ -137,13 +152,34 @@ pub(super) fn init(madt: Madt) {
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Wait for trampoline ready
|
||||
while unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } == 0 {
|
||||
// Wait for trampoline ready with timeout
|
||||
let mut trampoline_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } != 0 {
|
||||
trampoline_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
while !AP_READY.load(Ordering::SeqCst) {
|
||||
if !trampoline_ready {
|
||||
println!("KERNEL AP: CPU {} trampoline timeout, skipping", ap_local_apic.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut kernel_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if AP_READY.load(Ordering::SeqCst) {
|
||||
kernel_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !kernel_ready {
|
||||
println!("KERNEL AP: CPU {} AP_READY timeout, skipping", ap_local_apic.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
crate::CPU_COUNT.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
RmmA::invalidate_all();
|
||||
}
|
||||
@@ -151,10 +187,12 @@ pub(super) fn init(madt: Madt) {
|
||||
}
|
||||
|
||||
// Unmap trampoline
|
||||
let (_frame, _, flush) = unsafe {
|
||||
if let Some((_frame, _, flush)) = unsafe {
|
||||
KernelMapper::lock_rw()
|
||||
.unmap_phys(trampoline_page.start_address())
|
||||
.expect("failed to unmap trampoline page")
|
||||
};
|
||||
flush.flush();
|
||||
} {
|
||||
flush.flush();
|
||||
} else {
|
||||
println!("KERNEL AP: failed to unmap trampoline page (non-fatal)");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,10 +82,35 @@ pub enum RxsdtEnum {
|
||||
Rsdt(Rsdt),
|
||||
Xsdt(Xsdt),
|
||||
}
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct AcpiRootInfo {
|
||||
pub revision: u8,
|
||||
pub root_sdt_address: PhysicalAddress,
|
||||
}
|
||||
|
||||
pub static ACPI_ROOT_INFO: Once<AcpiRootInfo> = Once::new();
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct AcpiRootInfo {
|
||||
pub revision: u8,
|
||||
pub root_sdt_address: PhysicalAddress,
|
||||
}
|
||||
|
||||
pub static ACPI_ROOT_INFO: Once<AcpiRootInfo> = Once::new();
|
||||
|
||||
impl Rxsdt for RxsdtEnum {
|
||||
fn iter(&self) -> RxsdtIter {
|
||||
match self {
|
||||
Self::Rsdt(rsdt) => rsdt.iter(),
|
||||
let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo {
|
||||
revision: rsdp.revision(),
|
||||
root_sdt_address: rsdp.sdt_address(),
|
||||
});
|
||||
|
||||
if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() {
|
||||
error!("ACPI_ROOT_INFO already initialized with a different RSDP root");
|
||||
}
|
||||
|
||||
Self::Xsdt(xsdt) => xsdt.iter(),
|
||||
}
|
||||
}
|
||||
@@ -101,6 +126,15 @@ pub unsafe fn init_before_mem(already_supplied_rsdp: Option<NonNull<u8>>) {
|
||||
let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp);
|
||||
|
||||
if let Some(rsdp) = rsdp_opt {
|
||||
let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo {
|
||||
revision: rsdp.revision(),
|
||||
root_sdt_address: rsdp.sdt_address(),
|
||||
});
|
||||
|
||||
if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() {
|
||||
error!("ACPI_ROOT_INFO already initialized with a different RSDP root");
|
||||
}
|
||||
|
||||
debug!("SDT address: {:#x}", rsdp.sdt_address().data());
|
||||
let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw());
|
||||
|
||||
|
||||
@@ -59,4 +59,8 @@ impl Rsdp {
|
||||
self.rsdt_address as usize
|
||||
})
|
||||
}
|
||||
|
||||
pub fn revision(&self) -> u8 {
|
||||
self.revision
|
||||
}
|
||||
}
|
||||
|
||||
+27
-13
@@ -7,26 +7,40 @@ mod linked_list;
|
||||
/// Size of kernel heap
|
||||
const KERNEL_HEAP_SIZE: usize = ::rmm::MEGABYTE;
|
||||
|
||||
#[cold]
|
||||
fn halt_kernel_heap_init(message: &str) -> ! {
|
||||
print!("{message}");
|
||||
println!("Kernel heap initialization cannot continue. Halting.");
|
||||
loop {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn map_heap(mapper: &mut KernelMapper<true>, offset: usize, size: usize) {
|
||||
let mut flush_all = PageFlushAll::new();
|
||||
|
||||
let heap_start_page = Page::containing_address(VirtualAddress::new(offset));
|
||||
let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1));
|
||||
for page in Page::range_inclusive(heap_start_page, heap_end_page) {
|
||||
let phys = mapper
|
||||
.allocator_mut()
|
||||
.allocate_one()
|
||||
.expect("failed to allocate kernel heap");
|
||||
let phys = match mapper.allocator_mut().allocate_one() {
|
||||
Some(phys) => phys,
|
||||
None => halt_kernel_heap_init(
|
||||
"FATAL: failed to allocate physical frame for kernel heap\n",
|
||||
),
|
||||
};
|
||||
let flush = unsafe {
|
||||
mapper
|
||||
.map_phys(
|
||||
page.start_address(),
|
||||
phys,
|
||||
PageFlags::new()
|
||||
.write(true)
|
||||
.global(cfg!(not(feature = "pti"))),
|
||||
)
|
||||
.expect("failed to map kernel heap")
|
||||
match mapper.map_phys(
|
||||
page.start_address(),
|
||||
phys,
|
||||
PageFlags::new()
|
||||
.write(true)
|
||||
.global(cfg!(not(feature = "pti"))),
|
||||
) {
|
||||
Some(flush) => flush,
|
||||
None => halt_kernel_heap_init(
|
||||
"FATAL: failed to map kernel heap virtual page\n",
|
||||
),
|
||||
}
|
||||
};
|
||||
flush_all.consume(flush);
|
||||
}
|
||||
|
||||
@@ -192,6 +192,24 @@ impl ProcessorControlRegion {
|
||||
}
|
||||
}
|
||||
|
||||
#[cold]
|
||||
fn halt_pcr_init() -> ! {
|
||||
println!("FATAL: failed to allocate physical memory for Processor Control Region");
|
||||
println!("Processor startup cannot continue. Halting.");
|
||||
loop {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
#[cold]
|
||||
fn halt_pcr_init() -> ! {
|
||||
println!("FATAL: failed to allocate physical memory for Processor Control Region");
|
||||
println!("Processor startup cannot continue. Halting.");
|
||||
loop {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn pcr() -> *mut ProcessorControlRegion {
|
||||
unsafe {
|
||||
// Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that
|
||||
@@ -375,7 +393,10 @@ pub fn allocate_and_init_pcr(
|
||||
.next_power_of_two()
|
||||
.trailing_zeros();
|
||||
|
||||
let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR");
|
||||
let pcr_frame = match crate::memory::allocate_p2frame(alloc_order) {
|
||||
Some(frame) => frame,
|
||||
None => halt_pcr_init(),
|
||||
};
|
||||
let pcr_ptr = RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion;
|
||||
unsafe { core::ptr::write(pcr_ptr, ProcessorControlRegion::new_partial_init(cpu_id)) };
|
||||
|
||||
|
||||
@@ -78,6 +78,15 @@ static INIT_BSP_IDT: SyncUnsafeCell<Idt> = SyncUnsafeCell::new(Idt::new());
|
||||
pub(crate) static IDTS: RwLock<HashMap<LogicalCpuId, &'static mut Idt>> =
|
||||
RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new()));
|
||||
|
||||
#[cold]
|
||||
fn halt_idt_init() -> ! {
|
||||
println!("FATAL: failed to allocate physical pages for backup interrupt stack");
|
||||
println!("Interrupt setup cannot continue. Halting.");
|
||||
loop {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool {
|
||||
if cpu_id == LogicalCpuId::BSP {
|
||||
@@ -161,8 +170,10 @@ pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt {
|
||||
.or_insert_with(|| Box::leak(Box::new(Idt::new())));
|
||||
|
||||
use crate::memory::{RmmA, RmmArch};
|
||||
let frames = crate::memory::allocate_p2frame(4)
|
||||
.expect("failed to allocate pages for backup interrupt stack");
|
||||
let frames = match crate::memory::allocate_p2frame(4) {
|
||||
Some(frames) => frames,
|
||||
None => halt_idt_init(),
|
||||
};
|
||||
|
||||
// Physical pages are mapped linearly. So is the linearly mapped virtual memory.
|
||||
let base_address = RmmA::phys_to_virt(frames.base());
|
||||
|
||||
@@ -28,6 +28,8 @@ pub mod pti;
|
||||
/// Initialization and start function
|
||||
pub mod start;
|
||||
|
||||
pub mod sleep;
|
||||
|
||||
/// Stop function
|
||||
pub mod stop;
|
||||
|
||||
|
||||
@@ -0,0 +1,712 @@
|
||||
use alloc::{sync::Arc, vec::Vec};
|
||||
use core::{
|
||||
ptr::NonNull,
|
||||
str::FromStr,
|
||||
sync::atomic::{AtomicU32, Ordering},
|
||||
};
|
||||
|
||||
use acpi_ext::{
|
||||
aml::{namespace::AmlName, object::Object, Interpreter},
|
||||
registers::FixedRegisters,
|
||||
sdt::{facs::Facs, fadt::Fadt, SdtHeader},
|
||||
AcpiTables, Handle, Handler, PhysicalMapping,
|
||||
};
|
||||
use spin::Mutex;
|
||||
use syscall::error::{Error, EINVAL, EIO};
|
||||
use x86::{segmentation::SegmentSelector, task, Ring};
|
||||
|
||||
use crate::{
|
||||
acpi::ACPI_ROOT_INFO,
|
||||
arch::interrupt,
|
||||
memory::{
|
||||
round_down_pages, round_up_pages, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA,
|
||||
RmmArch, VirtualAddress, PAGE_SIZE,
|
||||
},
|
||||
syscall::io::{Io, Pio},
|
||||
};
|
||||
|
||||
const ACPI_SLP_TYP_SHIFT: u16 = 10;
|
||||
const ACPI_SLP_TYP_MASK: u16 = 0x1C00;
|
||||
const ACPI_SLP_EN: u16 = 1 << 13;
|
||||
const WAKE_TRAMPOLINE_PHYS: usize = 0x8000;
|
||||
const SLEEP_RETURN_OK: usize = 0;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
static WAKE_TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/s3_wakeup"));
|
||||
|
||||
#[repr(C, packed)]
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct DescriptorTableRegister {
|
||||
limit: u16,
|
||||
base: u64,
|
||||
}
|
||||
|
||||
#[repr(C, align(64))]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct FpuState {
|
||||
bytes: [u8; 4096],
|
||||
}
|
||||
|
||||
impl Default for FpuState {
|
||||
fn default() -> Self {
|
||||
Self { bytes: [0; 4096] }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum SleepState {
|
||||
S3,
|
||||
S5,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum SleepError {
|
||||
UnsupportedArch,
|
||||
MissingAcpi,
|
||||
MissingFadt,
|
||||
MissingFacs,
|
||||
MissingSleepObject,
|
||||
InvalidSleepObject,
|
||||
UnsupportedPmControl,
|
||||
UnsupportedAmlOperation,
|
||||
SleepDidNotEnter,
|
||||
}
|
||||
|
||||
impl SleepError {
|
||||
fn code(self) -> usize {
|
||||
match self {
|
||||
Self::UnsupportedArch => EINVAL as usize,
|
||||
Self::MissingAcpi
|
||||
| Self::MissingFadt
|
||||
| Self::MissingFacs
|
||||
| Self::MissingSleepObject
|
||||
| Self::UnsupportedAmlOperation => EIO as usize,
|
||||
Self::InvalidSleepObject | Self::UnsupportedPmControl | Self::SleepDidNotEnter => {
|
||||
EINVAL as usize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: usize) -> Self {
|
||||
match code as i32 {
|
||||
x if x == EINVAL => Self::InvalidSleepObject,
|
||||
_ => Self::MissingAcpi,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct SavedCpuContext {
|
||||
entry_rsp: usize,
|
||||
runtime_rsp: usize,
|
||||
facs_address: usize,
|
||||
cr0: usize,
|
||||
cr2: usize,
|
||||
cr3: usize,
|
||||
cr4: usize,
|
||||
rflags: usize,
|
||||
gdtr: DescriptorTableRegister,
|
||||
idtr: DescriptorTableRegister,
|
||||
efer: u64,
|
||||
fs_base: u64,
|
||||
gs_base: u64,
|
||||
kernel_gs_base: u64,
|
||||
fpu: FpuState,
|
||||
}
|
||||
|
||||
static SAVED_CONTEXT: Mutex<Option<SavedCpuContext>> = Mutex::new(None);
|
||||
static AML_MUTEX_IDS: AtomicU32 = AtomicU32::new(1);
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct SleepTypeData {
|
||||
a: u16,
|
||||
b: u16,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct KernelAcpiHandler;
|
||||
|
||||
impl KernelAcpiHandler {
|
||||
fn map_range(physical_address: usize, size: usize) -> (*mut u8, usize) {
|
||||
let map_base = round_down_pages(physical_address);
|
||||
let map_offset = physical_address - map_base;
|
||||
let mapped_length = round_up_pages(size + map_offset);
|
||||
|
||||
// SAFETY: The ACPI interpreter only requests firmware-described physical regions.
|
||||
unsafe {
|
||||
let mut mapper = KernelMapper::lock_rw();
|
||||
for page_index in 0..mapped_length / PAGE_SIZE {
|
||||
let (_, flush) = mapper
|
||||
.map_linearly(
|
||||
PhysicalAddress::new(map_base + page_index * PAGE_SIZE),
|
||||
PageFlags::new(),
|
||||
)
|
||||
.expect("failed to linearly map ACPI physical region");
|
||||
flush.flush();
|
||||
}
|
||||
}
|
||||
|
||||
let virtual_base = RmmA::phys_to_virt(PhysicalAddress::new(map_base)).data();
|
||||
((virtual_base + map_offset) as *mut u8, mapped_length)
|
||||
}
|
||||
}
|
||||
|
||||
impl Handler for KernelAcpiHandler {
|
||||
unsafe fn map_physical_region<T>(&self, physical_address: usize, size: usize) -> PhysicalMapping<Self, T> {
|
||||
let (virtual_start, mapped_length) = Self::map_range(physical_address, size);
|
||||
PhysicalMapping {
|
||||
physical_start: physical_address,
|
||||
virtual_start: NonNull::new(virtual_start.cast::<T>())
|
||||
.expect("expected mapped ACPI virtual address to be non-null"),
|
||||
region_length: size,
|
||||
mapped_length,
|
||||
handler: *self,
|
||||
}
|
||||
}
|
||||
|
||||
fn unmap_physical_region<T>(_region: &PhysicalMapping<Self, T>) {}
|
||||
|
||||
fn read_u8(&self, address: usize) -> u8 {
|
||||
// SAFETY: AML system-memory accesses are byte-addressable firmware regions.
|
||||
unsafe { core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u8) }
|
||||
}
|
||||
|
||||
fn read_u16(&self, address: usize) -> u16 {
|
||||
// SAFETY: AML system-memory accesses are word-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u16)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_u32(&self, address: usize) -> u32 {
|
||||
// SAFETY: AML system-memory accesses are dword-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u32)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_u64(&self, address: usize) -> u64 {
|
||||
// SAFETY: AML system-memory accesses are qword-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::read_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *const u64)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_u8(&self, address: usize, value: u8) {
|
||||
// SAFETY: AML system-memory accesses are byte-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::write_volatile(RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u8, value)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_u16(&self, address: usize, value: u16) {
|
||||
// SAFETY: AML system-memory accesses are word-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::write_volatile(
|
||||
RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u16,
|
||||
value,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_u32(&self, address: usize, value: u32) {
|
||||
// SAFETY: AML system-memory accesses are dword-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::write_volatile(
|
||||
RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u32,
|
||||
value,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_u64(&self, address: usize, value: u64) {
|
||||
// SAFETY: AML system-memory accesses are qword-addressable firmware regions.
|
||||
unsafe {
|
||||
core::ptr::write_volatile(
|
||||
RmmA::phys_to_virt(PhysicalAddress::new(address)).data() as *mut u64,
|
||||
value,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_io_u8(&self, port: u16) -> u8 {
|
||||
Pio::<u8>::new(port).read()
|
||||
}
|
||||
|
||||
fn read_io_u16(&self, port: u16) -> u16 {
|
||||
Pio::<u16>::new(port).read()
|
||||
}
|
||||
|
||||
fn read_io_u32(&self, port: u16) -> u32 {
|
||||
Pio::<u32>::new(port).read()
|
||||
}
|
||||
|
||||
fn write_io_u8(&self, port: u16, value: u8) {
|
||||
Pio::<u8>::new(port).write(value)
|
||||
}
|
||||
|
||||
fn write_io_u16(&self, port: u16, value: u16) {
|
||||
Pio::<u16>::new(port).write(value)
|
||||
}
|
||||
|
||||
fn write_io_u32(&self, port: u16, value: u32) {
|
||||
Pio::<u32>::new(port).write(value)
|
||||
}
|
||||
|
||||
fn read_pci_u8(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u8 {
|
||||
0
|
||||
}
|
||||
|
||||
fn read_pci_u16(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u16 {
|
||||
0
|
||||
}
|
||||
|
||||
fn read_pci_u32(&self, _address: acpi_ext::PciAddress, _offset: u16) -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
fn write_pci_u8(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u8) {}
|
||||
|
||||
fn write_pci_u16(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u16) {}
|
||||
|
||||
fn write_pci_u32(&self, _address: acpi_ext::PciAddress, _offset: u16, _value: u32) {}
|
||||
|
||||
fn nanos_since_boot(&self) -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
fn stall(&self, microseconds: u64) {
|
||||
for _ in 0..(microseconds.saturating_mul(64)) {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
fn sleep(&self, milliseconds: u64) {
|
||||
for _ in 0..(milliseconds.saturating_mul(64_000)) {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
fn create_mutex(&self) -> Handle {
|
||||
Handle(AML_MUTEX_IDS.fetch_add(1, Ordering::Relaxed))
|
||||
}
|
||||
|
||||
fn acquire(&self, _mutex: Handle, _timeout: u16) -> Result<(), acpi_ext::aml::AmlError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn release(&self, _mutex: Handle) {}
|
||||
}
|
||||
|
||||
fn sleep_state_name(state: SleepState) -> &'static str {
|
||||
match state {
|
||||
SleepState::S3 => "\\_S3",
|
||||
SleepState::S5 => "\\_S5",
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_sleep_type(value: u16) -> u16 {
|
||||
if value <= 0x7 {
|
||||
value << ACPI_SLP_TYP_SHIFT
|
||||
} else {
|
||||
value & ACPI_SLP_TYP_MASK
|
||||
}
|
||||
}
|
||||
|
||||
fn load_interpreter() -> Result<(
|
||||
Arc<FixedRegisters<KernelAcpiHandler>>,
|
||||
PhysicalMapping<KernelAcpiHandler, Facs>,
|
||||
Interpreter<KernelAcpiHandler>,
|
||||
), SleepError> {
|
||||
let root = *ACPI_ROOT_INFO.get().ok_or(SleepError::MissingAcpi)?;
|
||||
let handler = KernelAcpiHandler;
|
||||
|
||||
// SAFETY: ACPI root info is captured from the firmware-provided, already validated root table.
|
||||
let tables = unsafe {
|
||||
AcpiTables::from_rsdt(handler, root.revision, root.root_sdt_address.data())
|
||||
.map_err(|_| SleepError::MissingAcpi)?
|
||||
};
|
||||
let fadt = tables.find_table::<Fadt>().ok_or(SleepError::MissingFadt)?;
|
||||
let registers = Arc::new(
|
||||
FixedRegisters::new(&fadt, handler).map_err(|_| SleepError::UnsupportedPmControl)?,
|
||||
);
|
||||
let facs_address = fadt.facs_address().map_err(|_| SleepError::MissingFacs)?;
|
||||
|
||||
// SAFETY: The FADT-supplied FACS address is used exactly as described by the ACPI spec.
|
||||
let facs = unsafe { handler.map_physical_region::<Facs>(facs_address, core::mem::size_of::<Facs>()) };
|
||||
// SAFETY: The AML interpreter only needs an owned mapping of the same firmware FACS table.
|
||||
let interpreter_facs = unsafe {
|
||||
handler.map_physical_region::<Facs>(facs_address, core::mem::size_of::<Facs>())
|
||||
};
|
||||
let dsdt = tables.dsdt().map_err(|_| SleepError::MissingFadt)?;
|
||||
let interpreter = Interpreter::new(handler, dsdt.revision, Arc::clone(®isters), Some(interpreter_facs));
|
||||
|
||||
// SAFETY: Each AML table mapping is owned by the interpreter during table loading.
|
||||
unsafe {
|
||||
let mapping = handler.map_physical_region::<SdtHeader>(dsdt.phys_address, dsdt.length as usize);
|
||||
let stream = core::slice::from_raw_parts(
|
||||
mapping.virtual_start.as_ptr().byte_add(core::mem::size_of::<SdtHeader>()) as *const u8,
|
||||
dsdt.length as usize - core::mem::size_of::<SdtHeader>(),
|
||||
);
|
||||
interpreter
|
||||
.load_table(stream)
|
||||
.map_err(|_| SleepError::UnsupportedAmlOperation)?;
|
||||
|
||||
for ssdt in tables.ssdts() {
|
||||
let mapping = handler.map_physical_region::<SdtHeader>(ssdt.phys_address, ssdt.length as usize);
|
||||
let stream = core::slice::from_raw_parts(
|
||||
mapping.virtual_start.as_ptr().byte_add(core::mem::size_of::<SdtHeader>()) as *const u8,
|
||||
ssdt.length as usize - core::mem::size_of::<SdtHeader>(),
|
||||
);
|
||||
interpreter
|
||||
.load_table(stream)
|
||||
.map_err(|_| SleepError::UnsupportedAmlOperation)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((registers, facs, interpreter))
|
||||
}
|
||||
|
||||
fn sleep_type_data_from_interpreter(
|
||||
interpreter: &Interpreter<KernelAcpiHandler>,
|
||||
state: SleepState,
|
||||
) -> Result<SleepTypeData, SleepError> {
|
||||
let name = AmlName::from_str(sleep_state_name(state)).map_err(|_| SleepError::MissingSleepObject)?;
|
||||
let object = interpreter
|
||||
.evaluate(name, Vec::new())
|
||||
.map_err(|_| SleepError::MissingSleepObject)?;
|
||||
|
||||
let Object::Package(package) = &*object else {
|
||||
return Err(SleepError::InvalidSleepObject);
|
||||
};
|
||||
|
||||
let Some(typa_object) = package.first() else {
|
||||
return Err(SleepError::InvalidSleepObject);
|
||||
};
|
||||
let Some(typb_object) = package.get(1) else {
|
||||
return Err(SleepError::InvalidSleepObject);
|
||||
};
|
||||
|
||||
let Object::Integer(typa) = &**typa_object else {
|
||||
return Err(SleepError::InvalidSleepObject);
|
||||
};
|
||||
let Object::Integer(typb) = &**typb_object else {
|
||||
return Err(SleepError::InvalidSleepObject);
|
||||
};
|
||||
|
||||
Ok(SleepTypeData {
|
||||
a: encode_sleep_type(*typa as u16),
|
||||
b: encode_sleep_type(*typb as u16),
|
||||
})
|
||||
}
|
||||
|
||||
fn sleep_type_data(state: SleepState) -> Result<SleepTypeData, SleepError> {
|
||||
let (_registers, _facs, interpreter) = load_interpreter()?;
|
||||
sleep_type_data_from_interpreter(&interpreter, state)
|
||||
}
|
||||
|
||||
fn install_wake_trampoline(stack_rsp: usize, cr3: usize) {
|
||||
let trampoline_page = Page::containing_address(VirtualAddress::new(WAKE_TRAMPOLINE_PHYS));
|
||||
let trampoline_frame = PhysicalAddress::new(WAKE_TRAMPOLINE_PHYS);
|
||||
|
||||
// SAFETY: The 0x8000 low-memory trampoline page is reserved by the kernel for bootstrap stubs.
|
||||
let (result, _) = unsafe {
|
||||
let mut mapper = KernelMapper::lock_rw();
|
||||
let result = mapper
|
||||
.map_phys(
|
||||
trampoline_page.start_address(),
|
||||
trampoline_frame,
|
||||
PageFlags::new().execute(true).write(true),
|
||||
)
|
||||
.expect("failed to map S3 wake trampoline page");
|
||||
(result, mapper.table().phys().data())
|
||||
};
|
||||
result.flush();
|
||||
|
||||
for (index, value) in WAKE_TRAMPOLINE_DATA.iter().enumerate() {
|
||||
// SAFETY: The trampoline page is mapped writable at the same virtual address as the physical page.
|
||||
unsafe {
|
||||
core::ptr::write_volatile((WAKE_TRAMPOLINE_PHYS as *mut u8).add(index), *value);
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: The wake trampoline layout reserves three qword fields immediately after the jump.
|
||||
unsafe {
|
||||
let stack_slot = (WAKE_TRAMPOLINE_PHYS + 8) as *mut u64;
|
||||
let page_table_slot = stack_slot.add(1);
|
||||
let code_slot = stack_slot.add(2);
|
||||
stack_slot.write(stack_rsp as u64);
|
||||
page_table_slot.write(cr3 as u64);
|
||||
#[expect(clippy::fn_to_numeric_cast)]
|
||||
code_slot.write(resume_from_s3_trampoline as usize as u64);
|
||||
}
|
||||
|
||||
// SAFETY: The trampoline mapping is no longer needed once the physical page has been populated.
|
||||
let (_frame, _, flush) = unsafe {
|
||||
KernelMapper::lock_rw()
|
||||
.unmap_phys(trampoline_page.start_address())
|
||||
.expect("failed to unmap S3 wake trampoline page")
|
||||
};
|
||||
flush.flush();
|
||||
}
|
||||
|
||||
fn save_descriptor_tables(context: &mut SavedCpuContext) {
|
||||
// SAFETY: SGDT/SIDT only read the current CPU descriptor-table registers into the provided storage.
|
||||
unsafe {
|
||||
core::arch::asm!("sgdt [{}]", in(reg) &mut context.gdtr, options(nostack, preserves_flags));
|
||||
core::arch::asm!("sidt [{}]", in(reg) &mut context.idtr, options(nostack, preserves_flags));
|
||||
}
|
||||
}
|
||||
|
||||
fn save_fpu_state(context: &mut SavedCpuContext) {
|
||||
// SAFETY: The kernel owns the current CPU at suspend entry and the FXSAVE buffer is 64-byte aligned.
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
"fxsave64 [{}]",
|
||||
in(reg) context.fpu.bytes.as_mut_ptr(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_fpu_state(context: &SavedCpuContext) {
|
||||
// SAFETY: The saved FXSAVE image belongs to the same CPU context and matches the restore instruction.
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
"fxrstor64 [{}]",
|
||||
in(reg) context.fpu.bytes.as_ptr(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn save_cpu_context(entry_rsp: usize) -> SavedCpuContext {
|
||||
let mut context = SavedCpuContext {
|
||||
entry_rsp,
|
||||
..SavedCpuContext::default()
|
||||
};
|
||||
|
||||
// SAFETY: Reading control registers and MSRs is required to reconstruct the CPU execution state on wake.
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
"mov {}, cr0",
|
||||
out(reg) context.cr0,
|
||||
options(nostack, preserves_flags)
|
||||
);
|
||||
core::arch::asm!(
|
||||
"mov {}, cr2",
|
||||
out(reg) context.cr2,
|
||||
options(nostack, preserves_flags)
|
||||
);
|
||||
core::arch::asm!(
|
||||
"mov {}, cr3",
|
||||
out(reg) context.cr3,
|
||||
options(nostack, preserves_flags)
|
||||
);
|
||||
core::arch::asm!(
|
||||
"mov {}, cr4",
|
||||
out(reg) context.cr4,
|
||||
options(nostack, preserves_flags)
|
||||
);
|
||||
core::arch::asm!(
|
||||
"pushfq",
|
||||
"pop {}",
|
||||
out(reg) context.rflags,
|
||||
options(preserves_flags)
|
||||
);
|
||||
core::arch::asm!("mov {}, rsp", out(reg) context.runtime_rsp, options(nostack, preserves_flags));
|
||||
|
||||
context.efer = x86::msr::rdmsr(x86::msr::IA32_EFER);
|
||||
context.fs_base = x86::msr::rdmsr(x86::msr::IA32_FS_BASE);
|
||||
context.gs_base = x86::msr::rdmsr(x86::msr::IA32_GS_BASE);
|
||||
context.kernel_gs_base = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE);
|
||||
}
|
||||
|
||||
save_descriptor_tables(&mut context);
|
||||
save_fpu_state(&mut context);
|
||||
context
|
||||
}
|
||||
|
||||
fn set_firmware_waking_vector(facs: &mut PhysicalMapping<KernelAcpiHandler, Facs>, vector: usize) {
|
||||
facs.firmware_waking_vector = vector as u32;
|
||||
facs.x_firmware_waking_vector = vector as u64;
|
||||
}
|
||||
|
||||
fn write_pm1_control_block(
|
||||
registers: &FixedRegisters<KernelAcpiHandler>,
|
||||
sleep_type: SleepTypeData,
|
||||
) -> Result<(), SleepError> {
|
||||
let current_a = registers
|
||||
.pm1_control_registers
|
||||
.pm1a
|
||||
.read()
|
||||
.map_err(|_| SleepError::UnsupportedPmControl)? as u16;
|
||||
let armed_a = (current_a & !(ACPI_SLP_TYP_MASK | ACPI_SLP_EN)) | sleep_type.a;
|
||||
|
||||
registers
|
||||
.pm1_control_registers
|
||||
.pm1a
|
||||
.write(u64::from(armed_a))
|
||||
.map_err(|_| SleepError::UnsupportedPmControl)?;
|
||||
|
||||
if let Some(pm1b) = ®isters.pm1_control_registers.pm1b {
|
||||
let current_b = pm1b.read().map_err(|_| SleepError::UnsupportedPmControl)? as u16;
|
||||
let armed_b = (current_b & !(ACPI_SLP_TYP_MASK | ACPI_SLP_EN)) | sleep_type.b;
|
||||
pm1b.write(u64::from(armed_b))
|
||||
.map_err(|_| SleepError::UnsupportedPmControl)?;
|
||||
pm1b.write(u64::from(armed_b | ACPI_SLP_EN))
|
||||
.map_err(|_| SleepError::UnsupportedPmControl)?;
|
||||
}
|
||||
|
||||
// SAFETY: WBINVD is required here to flush dirty cache lines before firmware powers down the CPU package.
|
||||
unsafe {
|
||||
core::arch::asm!("wbinvd", options(nostack, preserves_flags));
|
||||
}
|
||||
|
||||
registers
|
||||
.pm1_control_registers
|
||||
.pm1a
|
||||
.write(u64::from(armed_a | ACPI_SLP_EN))
|
||||
.map_err(|_| SleepError::UnsupportedPmControl)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[unsafe(naked)]
|
||||
unsafe extern "sysv64" fn enter_sleep_raw(state: usize) -> usize {
|
||||
core::arch::naked_asm!(
|
||||
"mov rsi, rsp",
|
||||
"jmp {inner}",
|
||||
inner = sym enter_sleep_raw_inner,
|
||||
);
|
||||
}
|
||||
|
||||
extern "C" fn enter_sleep_raw_inner(state: usize, entry_rsp: usize) -> usize {
|
||||
let state = match state {
|
||||
3 => SleepState::S3,
|
||||
5 => SleepState::S5,
|
||||
_ => return SleepError::InvalidSleepObject.code(),
|
||||
};
|
||||
|
||||
let (registers, mut facs, interpreter) = match load_interpreter() {
|
||||
Ok(tuple) => tuple,
|
||||
Err(error) => return error.code(),
|
||||
};
|
||||
let sleep_type = match sleep_type_data_from_interpreter(&interpreter, state) {
|
||||
Ok(data) => data,
|
||||
Err(error) => return error.code(),
|
||||
};
|
||||
|
||||
let mut context = save_cpu_context(entry_rsp);
|
||||
context.facs_address = facs.physical_start;
|
||||
install_wake_trampoline(context.runtime_rsp, context.cr3);
|
||||
set_firmware_waking_vector(&mut facs, WAKE_TRAMPOLINE_PHYS);
|
||||
|
||||
{
|
||||
let mut saved = SAVED_CONTEXT.lock();
|
||||
*saved = Some(context);
|
||||
}
|
||||
|
||||
// SAFETY: Suspend entry must not be interrupted while the wake vector and PM1 control block are being armed.
|
||||
unsafe {
|
||||
interrupt::disable();
|
||||
}
|
||||
|
||||
if let Err(error) = write_pm1_control_block(registers.as_ref(), sleep_type) {
|
||||
return error.code();
|
||||
}
|
||||
|
||||
// SAFETY: The final CLI+HLT sequence is the architectural handoff point after asserting SLP_EN.
|
||||
unsafe {
|
||||
core::arch::asm!("cli; hlt", options(nostack));
|
||||
}
|
||||
|
||||
SleepError::SleepDidNotEnter.code()
|
||||
}
|
||||
|
||||
extern "C" fn resume_from_s3_trampoline() -> ! {
|
||||
let mut saved = SAVED_CONTEXT.lock();
|
||||
let context = saved.take().expect("S3 wake trampoline resumed without saved CPU context");
|
||||
drop(saved);
|
||||
|
||||
// SAFETY: The saved FACS physical address was captured from the validated FADT during suspend entry.
|
||||
if context.facs_address != 0 {
|
||||
let mut facs = unsafe {
|
||||
KernelAcpiHandler.map_physical_region::<Facs>(
|
||||
context.facs_address,
|
||||
core::mem::size_of::<Facs>(),
|
||||
)
|
||||
};
|
||||
set_firmware_waking_vector(&mut facs, 0);
|
||||
}
|
||||
|
||||
// SAFETY: The wake trampoline already switched to the saved kernel CR3 and long mode, so the remaining restores are architectural register state only.
|
||||
unsafe {
|
||||
x86::msr::wrmsr(x86::msr::IA32_EFER, context.efer);
|
||||
core::arch::asm!("mov cr3, {}", in(reg) context.cr3, options(nostack));
|
||||
core::arch::asm!("mov cr4, {}", in(reg) context.cr4, options(nostack));
|
||||
core::arch::asm!("mov cr2, {}", in(reg) context.cr2, options(nostack));
|
||||
core::arch::asm!("mov cr0, {}", in(reg) context.cr0, options(nostack));
|
||||
core::arch::asm!("lgdt [{}]", in(reg) &context.gdtr, options(nostack));
|
||||
core::arch::asm!("lidt [{}]", in(reg) &context.idtr, options(nostack));
|
||||
|
||||
task::load_tr(SegmentSelector::new(crate::arch::gdt::GDT_TSS as u16, Ring::Ring0));
|
||||
|
||||
x86::msr::wrmsr(x86::msr::IA32_FS_BASE, context.fs_base);
|
||||
x86::msr::wrmsr(x86::msr::IA32_GS_BASE, context.gs_base);
|
||||
x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, context.kernel_gs_base);
|
||||
}
|
||||
|
||||
restore_fpu_state(&context);
|
||||
|
||||
// SAFETY: Returning with the original entry stack and RFLAGS completes the suspend call as a successful function return.
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
"mov rsp, {entry_rsp}",
|
||||
"push {rflags}",
|
||||
"popfq",
|
||||
"xor eax, eax",
|
||||
"ret",
|
||||
entry_rsp = in(reg) context.entry_rsp,
|
||||
rflags = in(reg) context.rflags,
|
||||
options(noreturn)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn enter_sleep_state(state: SleepState) -> core::result::Result<(), SleepError> {
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
{
|
||||
let _ = state;
|
||||
return Err(SleepError::UnsupportedArch);
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
let raw = unsafe {
|
||||
enter_sleep_raw(match state {
|
||||
SleepState::S3 => 3,
|
||||
SleepState::S5 => 5,
|
||||
})
|
||||
};
|
||||
if raw == SLEEP_RETURN_OK {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SleepError::from_code(raw))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn available_sleep_states() -> &'static [u8] {
|
||||
if sleep_type_data(SleepState::S3).is_ok() {
|
||||
b"S3\nS5\n"
|
||||
} else {
|
||||
b"S5\n"
|
||||
}
|
||||
}
|
||||
|
||||
pub fn trigger_sleep_request(request: &str) -> Result<(), Error> {
|
||||
match request.trim() {
|
||||
"S3" => enter_sleep_state(SleepState::S3).map_err(|_| Error::new(EIO)),
|
||||
"S5" => enter_sleep_state(SleepState::S5).map_err(|_| Error::new(EIO)),
|
||||
_ => Err(Error::new(EINVAL)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
; ACPI S3 wake trampoline
|
||||
; compiled with nasm by build.rs, copied to physical 0x8000 before S3 entry
|
||||
|
||||
ORG 0x8000
|
||||
SECTION .text
|
||||
USE16
|
||||
|
||||
trampoline:
|
||||
jmp short startup_wake
|
||||
times 8 - ($ - trampoline) nop
|
||||
.stack: dq 0
|
||||
.page_table: dq 0
|
||||
.code: dq 0
|
||||
|
||||
startup_wake:
|
||||
cli
|
||||
|
||||
xor ax, ax
|
||||
mov ds, ax
|
||||
mov es, ax
|
||||
mov ss, ax
|
||||
mov sp, 0
|
||||
|
||||
mov edi, [trampoline.page_table]
|
||||
mov cr3, edi
|
||||
|
||||
mov eax, cr0
|
||||
and al, 11110011b
|
||||
or al, 00100010b
|
||||
mov cr0, eax
|
||||
|
||||
mov eax, cr4
|
||||
or eax, 1 << 9 | 1 << 7 | 1 << 5 | 1 << 4
|
||||
mov cr4, eax
|
||||
|
||||
fninit
|
||||
|
||||
lgdt [gdtr]
|
||||
|
||||
mov ecx, 0xC0000080
|
||||
rdmsr
|
||||
or eax, 1 << 11 | 1 << 8
|
||||
wrmsr
|
||||
|
||||
mov ebx, cr0
|
||||
or ebx, 1 << 31 | 1 << 16 | 1
|
||||
mov cr0, ebx
|
||||
|
||||
jmp gdt.kernel_code:long_mode_wake
|
||||
|
||||
USE64
|
||||
long_mode_wake:
|
||||
mov rax, gdt.kernel_data
|
||||
mov ds, rax
|
||||
mov es, rax
|
||||
mov fs, rax
|
||||
mov gs, rax
|
||||
mov ss, rax
|
||||
|
||||
mov rsp, [trampoline.stack]
|
||||
mov rax, [trampoline.code]
|
||||
jmp rax
|
||||
|
||||
struc GDTEntry
|
||||
.limitl resw 1
|
||||
.basel resw 1
|
||||
.basem resb 1
|
||||
.attribute resb 1
|
||||
.flags__limith resb 1
|
||||
.baseh resb 1
|
||||
endstruc
|
||||
|
||||
attrib:
|
||||
.present equ 1 << 7
|
||||
.user equ 1 << 4
|
||||
.code equ 1 << 3
|
||||
.writable equ 1 << 1
|
||||
|
||||
flags:
|
||||
.long_mode equ 1 << 5
|
||||
|
||||
gdtr:
|
||||
dw gdt.end + 1
|
||||
dq gdt
|
||||
|
||||
gdt:
|
||||
.null equ $ - gdt
|
||||
dq 0
|
||||
|
||||
.kernel_code equ $ - gdt
|
||||
istruc GDTEntry
|
||||
at GDTEntry.limitl, dw 0
|
||||
at GDTEntry.basel, dw 0
|
||||
at GDTEntry.basem, db 0
|
||||
at GDTEntry.attribute, db attrib.present | attrib.user | attrib.code
|
||||
at GDTEntry.flags__limith, db flags.long_mode
|
||||
at GDTEntry.baseh, db 0
|
||||
iend
|
||||
|
||||
.kernel_data equ $ - gdt
|
||||
istruc GDTEntry
|
||||
at GDTEntry.limitl, dw 0
|
||||
at GDTEntry.basel, dw 0
|
||||
at GDTEntry.basem, db 0
|
||||
at GDTEntry.attribute, db attrib.present | attrib.user | attrib.writable
|
||||
at GDTEntry.flags__limith, db 0
|
||||
at GDTEntry.baseh, db 0
|
||||
iend
|
||||
|
||||
.end equ $ - gdt
|
||||
@@ -435,9 +435,7 @@ impl Context {
|
||||
) -> Option<FileDescriptor> {
|
||||
let is_upper = i.get() & syscall::UPPER_FDTBL_TAG != 0;
|
||||
let result = self.files.write(lock_token.token()).remove_file(i);
|
||||
if is_upper && result.is_some() {
|
||||
crate::info!("Context::remove_file ctx={} upper_idx={}", self.name, i.get());
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
@@ -969,9 +967,7 @@ impl FdTbl {
|
||||
let removed_file_opt = fdtbl.get_mut(real_index).and_then(|opt| opt.take());
|
||||
if removed_file_opt.is_some() {
|
||||
self.active_count -= 1;
|
||||
if index & syscall::UPPER_FDTBL_TAG != 0 {
|
||||
crate::info!("FdTbl::remove_file upper idx={} active={}", real_index, self.active_count);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
removed_file_opt
|
||||
|
||||
+52
-5
@@ -4,7 +4,7 @@ use crate::{
|
||||
event,
|
||||
scheme::{self, SchemeId},
|
||||
sync::{CleanLockToken, RwLock, L6},
|
||||
syscall::error::Result,
|
||||
syscall::error::{Error, Result, ESTALE},
|
||||
};
|
||||
use alloc::sync::Arc;
|
||||
use syscall::{schemev2::NewFdFlags, RwFlags, O_APPEND, O_NONBLOCK};
|
||||
@@ -18,6 +18,7 @@ pub struct FileDescription {
|
||||
pub offset: u64,
|
||||
/// The scheme that this file refers to
|
||||
pub scheme: SchemeId,
|
||||
pub scheme_generation: Option<u64>,
|
||||
/// The number the scheme uses to refer to this file
|
||||
pub number: usize,
|
||||
/// The flags passed to open or fcntl(SETFL)
|
||||
@@ -32,6 +33,52 @@ bitflags! {
|
||||
}
|
||||
}
|
||||
impl FileDescription {
|
||||
pub fn with_generation(
|
||||
scheme: SchemeId,
|
||||
scheme_generation: Option<u64>,
|
||||
number: usize,
|
||||
offset: u64,
|
||||
flags: u32,
|
||||
internal_flags: InternalFlags,
|
||||
) -> Self {
|
||||
Self {
|
||||
offset,
|
||||
scheme,
|
||||
scheme_generation,
|
||||
number,
|
||||
flags,
|
||||
internal_flags,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(
|
||||
scheme: SchemeId,
|
||||
number: usize,
|
||||
offset: u64,
|
||||
flags: u32,
|
||||
internal_flags: InternalFlags,
|
||||
token: &mut CleanLockToken,
|
||||
) -> Self {
|
||||
Self::with_generation(
|
||||
scheme,
|
||||
Some(scheme::current_scheme_generation(token.token(), scheme)),
|
||||
number,
|
||||
offset,
|
||||
flags,
|
||||
internal_flags,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_scheme(&self, token: &mut CleanLockToken) -> Result<scheme::KernelSchemes> {
|
||||
if let Some(expected_generation) = self.scheme_generation
|
||||
&& expected_generation != scheme::current_scheme_generation(token.token(), self.scheme)
|
||||
{
|
||||
return Err(Error::new(ESTALE));
|
||||
}
|
||||
|
||||
scheme::get_scheme(token.token(), self.scheme)
|
||||
}
|
||||
|
||||
pub fn rw_flags(&self, rw: RwFlags) -> u32 {
|
||||
let mut ret = self.flags & !(O_NONBLOCK | O_APPEND) as u32;
|
||||
if rw.contains(RwFlags::APPEND) {
|
||||
@@ -76,7 +123,7 @@ impl FileDescription {
|
||||
pub fn try_close(self, token: &mut CleanLockToken) -> Result<()> {
|
||||
event::unregister_file(self.scheme, self.number, token);
|
||||
|
||||
let scheme = scheme::get_scheme(token.token(), self.scheme)?;
|
||||
let scheme = self.get_scheme(token)?;
|
||||
|
||||
scheme.close(self.number, token)
|
||||
}
|
||||
@@ -85,12 +132,12 @@ impl FileDescription {
|
||||
impl FileDescriptor {
|
||||
pub fn close(self, token: &mut CleanLockToken) -> Result<()> {
|
||||
{
|
||||
let (scheme_id, number, internal_flags) = {
|
||||
let (desc, number, internal_flags) = {
|
||||
let desc = self.description.read(token.token());
|
||||
(desc.scheme, desc.number, desc.internal_flags)
|
||||
(*desc, desc.number, desc.internal_flags)
|
||||
};
|
||||
if internal_flags.contains(InternalFlags::NOTIFY_ON_NEXT_DETACH) {
|
||||
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
|
||||
let scheme = desc.get_scheme(token)?;
|
||||
scheme.detach(number, token)?;
|
||||
}
|
||||
}
|
||||
|
||||
+11
-19
@@ -64,14 +64,13 @@ impl UnmapResult {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let (scheme_id, number) = {
|
||||
let desc = description.write(token.token());
|
||||
(desc.scheme, desc.number)
|
||||
let (scheme, number) = {
|
||||
let desc = *description.read(token.token());
|
||||
(desc.get_scheme(token)?, desc.number)
|
||||
};
|
||||
|
||||
let scheme_opt = scheme::get_scheme(token.token(), scheme_id);
|
||||
let funmap_result = scheme_opt
|
||||
.and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags, token));
|
||||
let funmap_result = scheme
|
||||
.kfunmap(number, base_offset, self.size, self.flags, token);
|
||||
|
||||
if let Ok(fd) = Arc::try_unwrap(description) {
|
||||
fd.into_inner().try_close(token)?;
|
||||
@@ -2706,20 +2705,13 @@ fn correct_inner<'l>(
|
||||
// XXX: This is cheating, but guaranteed we won't deadlock because we've dropped addr_space_guard
|
||||
let mut token = unsafe { CleanLockToken::new() };
|
||||
|
||||
let (scheme_id, scheme_number) = {
|
||||
let desc = &file_ref.description.read(token.token());
|
||||
(desc.scheme, desc.number)
|
||||
let desc = *file_ref.description.read(token.token());
|
||||
let scheme = desc.get_scheme(&mut token).map_err(|_| PfError::Segv)?;
|
||||
let scheme_number = desc.number;
|
||||
let user_inner = match scheme {
|
||||
KernelSchemes::User(user) => user.inner,
|
||||
_ => return Err(PfError::Segv),
|
||||
};
|
||||
let user_inner = scheme::get_scheme(token.token(), scheme_id)
|
||||
.ok()
|
||||
.and_then(|s| {
|
||||
if let KernelSchemes::User(user) = s {
|
||||
Some(user.inner)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.ok_or(PfError::Segv)?;
|
||||
|
||||
let offset = file_ref.base_offset as u64 + (pages_from_grant_start * PAGE_SIZE) as u64;
|
||||
user_inner
|
||||
|
||||
@@ -10,6 +10,8 @@ use crate::{
|
||||
acpi::{RxsdtEnum, RXSDT_ENUM},
|
||||
arch::x86_64::s3_resume,
|
||||
context::file::InternalFlags,
|
||||
arch::sleep,
|
||||
arch::sleep,
|
||||
scheme::{SchemeExt, StrOrBytes},
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
@@ -39,7 +41,9 @@ bitflags! {
|
||||
static RXSDT_DATA: Once<Box<[u8]>> = Once::new();
|
||||
|
||||
/// Phase I: kstop reason codes. Read via the CheckShutdown
|
||||
SleepControl,
|
||||
/// AcpiVerb (kcall 2). The reason tells acpid what AML
|
||||
SleepControl,
|
||||
/// sequence to run.
|
||||
///
|
||||
/// | Value | Reason | acpid's response |
|
||||
@@ -189,8 +193,10 @@ impl AcpiScheme {
|
||||
}
|
||||
|
||||
impl KernelScheme for AcpiScheme {
|
||||
HandleKind::SleepControl => sleep::available_sleep_states().len() as u64,
|
||||
fn scheme_root(&self, _token: &mut CleanLockToken) -> Result<usize> {
|
||||
Ok((HandleBits::CAN_READ_RXSDT | HandleBits::CAN_REGISTER_KSTOP).bits())
|
||||
HandleKind::SleepControl => sleep::available_sleep_states().len() as u64,
|
||||
}
|
||||
fn kopenat(
|
||||
&self,
|
||||
@@ -250,9 +256,11 @@ impl KernelScheme for AcpiScheme {
|
||||
AcpiVerb::CheckShutdown => {
|
||||
if handle != HandleBits::KSTOP_HANDLE {
|
||||
return Err(Error::new(EINVAL));
|
||||
HandleKind::SleepControl => sleep::available_sleep_states(),
|
||||
}
|
||||
// Phase I.5: return the u8 reason, not the
|
||||
// pre-Phase-I.5 bool. acpid's CheckShutdown
|
||||
HandleKind::SleepControl => sleep::available_sleep_states(),
|
||||
// verb handler is updated to switch on the
|
||||
// reason value.
|
||||
Ok(*KSTOP_FLAG.lock(token.token()) as usize)
|
||||
@@ -324,10 +332,20 @@ impl KernelScheme for AcpiScheme {
|
||||
// the FADT (ACPI 1.0 or very old hardware).
|
||||
}
|
||||
Ok(0)
|
||||
HandleKind::SleepControl => Stat {
|
||||
st_mode: MODE_FILE,
|
||||
st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX),
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
AcpiVerb::EnterS3 => {
|
||||
// Phase II.X.W: acpid has done the AML prep
|
||||
// (`_TTS(3)`, `_PTS(3)`, `_SST(3)`) and written
|
||||
HandleKind::SleepControl => Stat {
|
||||
st_mode: MODE_FILE,
|
||||
st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX),
|
||||
..Default::default()
|
||||
},
|
||||
// the trampoline address to FACS via
|
||||
// `SetS3WakingVector`. The acpid now requests
|
||||
// the kernel to enter S3. The kernel's
|
||||
|
||||
+9
-4
@@ -22,9 +22,10 @@ struct Handle {
|
||||
|
||||
static HANDLES: RwLock<L1, HandleMap<Handle>> = RwLock::new(HandleMap::new());
|
||||
|
||||
/// Add to the input queue
|
||||
/// Add to the input queue, translating CR to NL (ICRNL) for serial console compatibility.
|
||||
pub fn debug_input(data: u8, token: &mut CleanLockToken) {
|
||||
INPUT.send(data, token);
|
||||
let translated = if data == b'\r' { b'\n' } else { data };
|
||||
INPUT.send(translated, token);
|
||||
}
|
||||
|
||||
// Notify readers of input updates
|
||||
@@ -106,12 +107,16 @@ impl KernelScheme for DebugScheme {
|
||||
fn fevent(
|
||||
&self,
|
||||
id: usize,
|
||||
_flags: EventFlags,
|
||||
flags: EventFlags,
|
||||
token: &mut CleanLockToken,
|
||||
) -> Result<EventFlags> {
|
||||
let _handle = *HANDLES.read(token.token()).get(id)?;
|
||||
|
||||
Ok(EventFlags::empty())
|
||||
let mut ready = EventFlags::empty();
|
||||
if flags.contains(EventFlags::EVENT_READ) {
|
||||
ready |= EventFlags::EVENT_READ;
|
||||
}
|
||||
Ok(ready)
|
||||
}
|
||||
|
||||
fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
|
||||
|
||||
+60
-25
@@ -12,7 +12,7 @@ use alloc::{
|
||||
};
|
||||
use core::{
|
||||
str,
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||
};
|
||||
use hashbrown::hash_map::{self, DefaultHashBuilder, HashMap};
|
||||
use spin::Once;
|
||||
@@ -167,6 +167,7 @@ enum Handle {
|
||||
|
||||
/// Schemes list
|
||||
static HANDLES: Once<RwLock<L1, HashMap<SchemeId, Handle>>> = Once::new();
|
||||
static SCHEME_GENERATIONS: Once<RwLock<L1, HashMap<SchemeId, AtomicU64>>> = Once::new();
|
||||
static SCHEME_LIST_NEXT_ID: AtomicUsize = AtomicUsize::new(MAX_GLOBAL_SCHEMES);
|
||||
static SCHEME_LIST_ID: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
@@ -202,6 +203,10 @@ fn init_schemes() -> RwLock<L1, HashMap<SchemeId, Handle>> {
|
||||
RwLock::new(handles)
|
||||
}
|
||||
|
||||
fn init_scheme_generations() -> RwLock<L1, HashMap<SchemeId, AtomicU64>> {
|
||||
RwLock::new(HashMap::new())
|
||||
}
|
||||
|
||||
/// Get a handle to a scheme.
|
||||
pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result<KernelSchemes> {
|
||||
match handles().read(token).get(&scheme_id) {
|
||||
@@ -210,10 +215,33 @@ pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result<Kerne
|
||||
}
|
||||
}
|
||||
|
||||
pub fn current_scheme_generation(token: LockToken<'_, L0>, scheme_id: SchemeId) -> u64 {
|
||||
scheme_generations()
|
||||
.read(token)
|
||||
.get(&scheme_id)
|
||||
.map(|generation| generation.load(Ordering::Acquire))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn handles<'a>() -> &'a RwLock<L1, HashMap<SchemeId, Handle>> {
|
||||
HANDLES.call_once(init_schemes)
|
||||
}
|
||||
|
||||
fn scheme_generations<'a>() -> &'a RwLock<L1, HashMap<SchemeId, AtomicU64>> {
|
||||
SCHEME_GENERATIONS.call_once(init_scheme_generations)
|
||||
}
|
||||
|
||||
fn increment_scheme_generation(scheme_id: SchemeId, token: &mut CleanLockToken) {
|
||||
match scheme_generations().write(token.token()).entry(scheme_id) {
|
||||
hash_map::Entry::Occupied(entry) => {
|
||||
entry.get().fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
hash_map::Entry::Vacant(entry) => {
|
||||
entry.insert(AtomicU64::new(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scheme list type
|
||||
pub struct SchemeList;
|
||||
|
||||
@@ -258,9 +286,14 @@ impl SchemeList {
|
||||
|
||||
/// Remove a scheme
|
||||
fn remove(&self, id: usize, token: &mut CleanLockToken) {
|
||||
let scheme = handles().write(token.token()).remove(&SchemeId(id));
|
||||
let scheme_id = SchemeId(id);
|
||||
let scheme = handles().write(token.token()).remove(&scheme_id);
|
||||
|
||||
assert!(scheme.is_some());
|
||||
if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme.as_ref() {
|
||||
user.inner.fail_pending_calls(token);
|
||||
}
|
||||
increment_scheme_generation(scheme_id, token);
|
||||
if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme
|
||||
&& let Some(user) = Arc::into_inner(user.inner)
|
||||
{
|
||||
@@ -285,32 +318,32 @@ impl KernelScheme for SchemeList {
|
||||
token: &mut CleanLockToken,
|
||||
) -> Result<OpenResult> {
|
||||
let scheme_id = SchemeId(scheme_id);
|
||||
match handles()
|
||||
.read(token.token())
|
||||
.get(&scheme_id)
|
||||
.ok_or(Error::new(EBADF))?
|
||||
{
|
||||
Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => {
|
||||
let inner = inner.clone();
|
||||
assert!(scheme_id == inner.scheme_id);
|
||||
let scheme = scheme_id;
|
||||
let params = unsafe { user_buf.read_exact::<NewFdParams>()? };
|
||||
|
||||
return Ok(OpenResult::External(Arc::new(RwLock::new(
|
||||
FileDescription {
|
||||
scheme,
|
||||
number: params.number,
|
||||
offset: params.offset,
|
||||
flags: params.flags as u32,
|
||||
internal_flags: InternalFlags::from_extra0(params.internal_flags)
|
||||
.ok_or(Error::new(EINVAL))?,
|
||||
},
|
||||
))));
|
||||
let maybe_inner = {
|
||||
let handles = handles().read(token.token());
|
||||
match handles.get(&scheme_id).ok_or(Error::new(EBADF))? {
|
||||
Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => Some(inner.clone()),
|
||||
Handle::SchemeCreationCapability => None,
|
||||
_ => return Err(Error::new(EBADF)),
|
||||
}
|
||||
Handle::SchemeCreationCapability => (),
|
||||
_ => return Err(Error::new(EBADF)),
|
||||
};
|
||||
|
||||
if let Some(inner) = maybe_inner {
|
||||
assert!(scheme_id == inner.scheme_id);
|
||||
let params = unsafe { user_buf.read_exact::<NewFdParams>()? };
|
||||
|
||||
return Ok(OpenResult::External(Arc::new(RwLock::new(
|
||||
FileDescription::new(
|
||||
scheme_id,
|
||||
params.number,
|
||||
params.offset,
|
||||
params.flags as u32,
|
||||
InternalFlags::from_extra0(params.internal_flags)
|
||||
.ok_or(Error::new(EINVAL))?,
|
||||
token,
|
||||
),
|
||||
))));
|
||||
}
|
||||
|
||||
const EXPECTED: &[u8] = b"create-scheme";
|
||||
let mut buf = [0u8; EXPECTED.len()];
|
||||
|
||||
@@ -775,6 +808,7 @@ pub enum OpenResult {
|
||||
SchemeLocal(usize, InternalFlags),
|
||||
External(Arc<LockedFileDescription>),
|
||||
}
|
||||
pub groups: alloc::vec::Vec<u32>,
|
||||
pub struct CallerCtx {
|
||||
pub pid: usize,
|
||||
pub uid: u32,
|
||||
@@ -783,6 +817,7 @@ pub struct CallerCtx {
|
||||
}
|
||||
impl CallerCtx {
|
||||
pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self {
|
||||
groups: self.groups,
|
||||
if self.uid == 0 && self.gid == 0 {
|
||||
Self {
|
||||
pid: self.pid,
|
||||
|
||||
+1
-7
@@ -783,13 +783,7 @@ impl KernelScheme for ProcScheme {
|
||||
}
|
||||
(pc, uc, ul, pl)
|
||||
};
|
||||
let ctx_name = context.read(token.token()).name;
|
||||
info!(
|
||||
"AwaitingFiletableChange applied: ctx='{}' posix_fds={} upper_fds={}",
|
||||
ctx_name, posix_count, upper_count
|
||||
);
|
||||
info!("upper FDs present:{}", upper_list);
|
||||
info!("posix FDs present:{}", posix_list);
|
||||
|
||||
context.write(token.token()).files = new_ft;
|
||||
}
|
||||
_ => (),
|
||||
|
||||
+157
-63
@@ -80,6 +80,7 @@ const ONE: NonZeroUsize = match NonZeroUsize::new(1) {
|
||||
Some(one) => one,
|
||||
None => unreachable!(),
|
||||
};
|
||||
const MAX_SPURIOUS_WAKEUPS: usize = 100;
|
||||
|
||||
enum ParsedCqe {
|
||||
TriggerFevent {
|
||||
@@ -209,6 +210,8 @@ impl UserInner {
|
||||
caller_responsible: &mut PageSpan,
|
||||
token: &mut CleanLockToken,
|
||||
) -> Result<Response> {
|
||||
let mut remaining_spurious_wakeups = MAX_SPURIOUS_WAKEUPS;
|
||||
|
||||
{
|
||||
// Disable preemption to avoid context switches between setting the
|
||||
// process state and sending the scheme request. The process is made
|
||||
@@ -261,7 +264,10 @@ impl UserInner {
|
||||
};
|
||||
|
||||
let states = self.states.lock(token.token());
|
||||
let (mut states, mut token) = states.into_split();
|
||||
let (mut states, mut state_token) = states.into_split();
|
||||
let mut timed_out_descriptions = None;
|
||||
let mut remove_state = false;
|
||||
let mut timed_out = false;
|
||||
match states.get_mut(sqe.tag as usize) {
|
||||
// invalid state
|
||||
None => return Err(Error::new(EBADFD)),
|
||||
@@ -274,24 +280,35 @@ impl UserInner {
|
||||
fds,
|
||||
} => {
|
||||
let maybe_eintr =
|
||||
eintr_if_sigkill(&mut callee_responsible, &mut token.token());
|
||||
*o = State::Waiting {
|
||||
canceling: true,
|
||||
callee_responsible,
|
||||
context,
|
||||
fds,
|
||||
};
|
||||
eintr_if_sigkill(&mut callee_responsible, &mut state_token.token());
|
||||
|
||||
if maybe_eintr.is_ok() {
|
||||
remaining_spurious_wakeups =
|
||||
remaining_spurious_wakeups.saturating_sub(1);
|
||||
}
|
||||
|
||||
if maybe_eintr.is_ok() && remaining_spurious_wakeups == 0 {
|
||||
timed_out_descriptions = Some(Self::collect_descriptions_to_close(fds));
|
||||
remove_state = true;
|
||||
} else {
|
||||
*o = State::Waiting {
|
||||
canceling: true,
|
||||
callee_responsible,
|
||||
context,
|
||||
fds,
|
||||
};
|
||||
}
|
||||
|
||||
maybe_eintr?;
|
||||
|
||||
context::current()
|
||||
.write(token.token())
|
||||
.block("UserInner::call (woken up after cancelation request)");
|
||||
|
||||
// We do not want to drop the lock before blocking
|
||||
// as if we get preempted in between we might miss a
|
||||
// wakeup.
|
||||
drop(states);
|
||||
if remove_state {
|
||||
states.remove(sqe.tag as usize);
|
||||
timed_out = true;
|
||||
} else {
|
||||
context::current()
|
||||
.write(state_token.token())
|
||||
.block("UserInner::call (woken up after cancelation request)");
|
||||
}
|
||||
}
|
||||
// spurious wakeup
|
||||
State::Waiting {
|
||||
@@ -300,60 +317,76 @@ impl UserInner {
|
||||
context,
|
||||
mut callee_responsible,
|
||||
} => {
|
||||
let maybe_eintr = eintr_if_sigkill(&mut callee_responsible, &mut token);
|
||||
let current_context = context::current();
|
||||
let maybe_eintr =
|
||||
eintr_if_sigkill(&mut callee_responsible, &mut state_token);
|
||||
|
||||
*o = State::Waiting {
|
||||
// Currently we treat all spurious wakeups to have the same behavior
|
||||
// as signals (i.e., we send a cancellation request). It is not something
|
||||
// that should happen, but it certainly can happen, for example if a context
|
||||
// is awoken through its thread handle without setting any sig bits, or if the
|
||||
// caller clears its own sig bits. If it actually is a signal, then it is the
|
||||
// intended behavior.
|
||||
canceling: true,
|
||||
fds,
|
||||
context,
|
||||
callee_responsible,
|
||||
};
|
||||
if maybe_eintr.is_ok() {
|
||||
remaining_spurious_wakeups =
|
||||
remaining_spurious_wakeups.saturating_sub(1);
|
||||
}
|
||||
|
||||
if maybe_eintr.is_ok() && remaining_spurious_wakeups == 0 {
|
||||
timed_out_descriptions = Some(Self::collect_descriptions_to_close(fds));
|
||||
remove_state = true;
|
||||
} else {
|
||||
*o = State::Waiting {
|
||||
// Currently we treat all spurious wakeups to have the same behavior
|
||||
// as signals (i.e., we send a cancellation request). It is not something
|
||||
// that should happen, but it certainly can happen, for example if a context
|
||||
// is awoken through its thread handle without setting any sig bits, or if the
|
||||
// caller clears its own sig bits. If it actually is a signal, then it is the
|
||||
// intended behavior.
|
||||
canceling: true,
|
||||
fds,
|
||||
context,
|
||||
callee_responsible,
|
||||
};
|
||||
}
|
||||
|
||||
maybe_eintr?;
|
||||
|
||||
// We do not want to preempt between sending the
|
||||
// cancellation and blocking again where we might
|
||||
// miss a wakeup.
|
||||
let mut preempt = PreemptGuardL1::new(¤t_context, &mut token);
|
||||
let token = preempt.token();
|
||||
if remove_state {
|
||||
states.remove(sqe.tag as usize);
|
||||
timed_out = true;
|
||||
} else {
|
||||
// We do not want to preempt between sending the
|
||||
// cancellation and blocking again where we might
|
||||
// miss a wakeup.
|
||||
let mut preempt =
|
||||
PreemptGuardL1::new(¤t_context, &mut state_token);
|
||||
let token = preempt.token();
|
||||
|
||||
self.todo.send_locked(
|
||||
Sqe {
|
||||
opcode: Opcode::Cancel as u8,
|
||||
sqe_flags: SqeFlags::ONEWAY,
|
||||
tag: sqe.tag,
|
||||
..Default::default()
|
||||
},
|
||||
token.token(),
|
||||
);
|
||||
event::trigger_locked(
|
||||
self.root_id,
|
||||
self.scheme_id.get(),
|
||||
EVENT_READ,
|
||||
token.token(),
|
||||
);
|
||||
self.todo.send_locked(
|
||||
Sqe {
|
||||
opcode: Opcode::Cancel as u8,
|
||||
sqe_flags: SqeFlags::ONEWAY,
|
||||
tag: sqe.tag,
|
||||
..Default::default()
|
||||
},
|
||||
token.token(),
|
||||
);
|
||||
event::trigger_locked(
|
||||
self.root_id,
|
||||
self.scheme_id.get(),
|
||||
EVENT_READ,
|
||||
token.token(),
|
||||
);
|
||||
|
||||
// 1. If cancellation was requested and arrived
|
||||
// before the scheme processed the request, an
|
||||
// acknowledgement will be sent back after the
|
||||
// cancellation is processed and we will be woken up
|
||||
// again. State will be State::Responded then.
|
||||
//
|
||||
// 2. If cancellation was requested but the scheme
|
||||
// already processed the request, we will receive
|
||||
// the actual response next and woken up again.
|
||||
// State will be State::Responded then.
|
||||
context::current()
|
||||
.write(token.token())
|
||||
.block("UserInner::call (spurious wakeup)");
|
||||
drop(states);
|
||||
// 1. If cancellation was requested and arrived
|
||||
// before the scheme processed the request, an
|
||||
// acknowledgement will be sent back after the
|
||||
// cancellation is processed and we will be woken up
|
||||
// again. State will be State::Responded then.
|
||||
//
|
||||
// 2. If cancellation was requested but the scheme
|
||||
// already processed the request, we will receive
|
||||
// the actual response next and woken up again.
|
||||
// State will be State::Responded then.
|
||||
context::current()
|
||||
.write(token.token())
|
||||
.block("UserInner::call (spurious wakeup)");
|
||||
}
|
||||
}
|
||||
|
||||
// invalid state
|
||||
@@ -368,10 +401,70 @@ impl UserInner {
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
if let Some(descriptions) = timed_out_descriptions {
|
||||
drop(states);
|
||||
for desc in descriptions {
|
||||
let _ = desc.try_close(token);
|
||||
}
|
||||
}
|
||||
|
||||
if timed_out {
|
||||
return Err(Error::new(ETIMEDOUT));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_descriptions_to_close(
|
||||
fds: Vec<Arc<LockedFileDescription>>,
|
||||
) -> Vec<FileDescription> {
|
||||
fds.into_iter()
|
||||
.filter_map(|fd| Arc::try_unwrap(fd).ok())
|
||||
.map(RwLock::into_inner)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn fail_pending_calls(&self, token: &mut CleanLockToken) {
|
||||
let descriptions_to_close = {
|
||||
let mut states_lock = self.states.lock(token.token());
|
||||
let (states, mut lock_token) = states_lock.token_split();
|
||||
let mut descriptions_to_close = Vec::new();
|
||||
let mut states_to_remove = Vec::new();
|
||||
|
||||
for (id, state) in states.iter_mut() {
|
||||
match mem::replace(state, State::Placeholder) {
|
||||
State::Waiting { context, fds, .. } => {
|
||||
descriptions_to_close.extend(Self::collect_descriptions_to_close(fds));
|
||||
|
||||
match context.upgrade() {
|
||||
Some(context) => {
|
||||
*state = State::Responded(Response::Regular(
|
||||
Err(Error::new(ENODEV)),
|
||||
0,
|
||||
false,
|
||||
));
|
||||
context.write(lock_token.token()).unblock();
|
||||
}
|
||||
None => states_to_remove.push(id),
|
||||
}
|
||||
}
|
||||
old_state => *state = old_state,
|
||||
}
|
||||
}
|
||||
|
||||
for id in states_to_remove {
|
||||
states.remove(id);
|
||||
}
|
||||
|
||||
descriptions_to_close
|
||||
};
|
||||
|
||||
for desc in descriptions_to_close {
|
||||
let _ = desc.try_close(token);
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a readable structure to the scheme's userspace and return the
|
||||
/// pointer
|
||||
#[must_use = "copying back to head/tail buffers can fail"]
|
||||
@@ -1283,6 +1376,7 @@ impl UserInner {
|
||||
}
|
||||
|
||||
pub fn into_drop(self, token: &mut CleanLockToken) {
|
||||
self.fail_pending_calls(token);
|
||||
self.todo.condition.into_drop(token);
|
||||
}
|
||||
}
|
||||
|
||||
+13
-8
@@ -140,7 +140,7 @@ extern "C" fn userspace_init() {
|
||||
info!("userspace_init: entered bootstrap context");
|
||||
let mut token = unsafe { CleanLockToken::new() };
|
||||
let bootstrap = BOOTSTRAP.get().expect("BOOTSTRAP was not set");
|
||||
info!("userspace_init: calling usermode_bootstrap");
|
||||
|
||||
unsafe { crate::syscall::process::usermode_bootstrap(bootstrap, &mut token) }
|
||||
}
|
||||
|
||||
@@ -154,6 +154,15 @@ static BOOTSTRAP: spin::Once<Bootstrap> = spin::Once::new();
|
||||
pub(crate) static AP_READY: AtomicBool = AtomicBool::new(false);
|
||||
static BSP_READY: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
#[cold]
|
||||
fn halt_boot(message: &str) -> ! {
|
||||
print!("{message}");
|
||||
println!("Kernel boot cannot continue. Halting.");
|
||||
loop {
|
||||
hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
/// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this
|
||||
pub(crate) fn kmain(bootstrap: Bootstrap) -> ! {
|
||||
let mut token = unsafe { CleanLockToken::new() };
|
||||
@@ -185,13 +194,11 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! {
|
||||
context.euid = 0;
|
||||
context.egid = 0;
|
||||
}
|
||||
Err(err) => {
|
||||
panic!("failed to spawn userspace_init: {:?}", err);
|
||||
}
|
||||
Err(_err) => halt_boot("FATAL: failed to spawn first userspace process userspace_init\n"),
|
||||
}
|
||||
|
||||
numa::dump_info();
|
||||
info!("kmain: dump_info done, calling run_userspace");
|
||||
|
||||
run_userspace(&mut token)
|
||||
}
|
||||
|
||||
@@ -241,9 +248,7 @@ fn run_userspace(token: &mut CleanLockToken) -> ! {
|
||||
interrupt::enable_and_nop();
|
||||
}
|
||||
SwitchResult::AllContextsIdle => {
|
||||
if switch_count == 0 {
|
||||
info!("run_userspace: AllContextsIdle on first switch — no runnable contexts");
|
||||
}
|
||||
|
||||
switch_count += 1;
|
||||
// Enable interrupts, then halt CPU (to save power) until the next interrupt is actually fired.
|
||||
interrupt::enable_and_halt();
|
||||
|
||||
+15
-18
@@ -12,7 +12,7 @@ use crate::{
|
||||
memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions},
|
||||
},
|
||||
memory::{Page, VirtualAddress, PAGE_SIZE},
|
||||
scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes},
|
||||
scheme::{FileHandle, KernelScheme, OpenResult, StrOrBytes},
|
||||
sync::{CleanLockToken, RwLock},
|
||||
syscall::{data::Stat, error::*, flag::*},
|
||||
};
|
||||
@@ -45,7 +45,7 @@ pub fn file_op_generic_ext<T>(
|
||||
(file, desc)
|
||||
};
|
||||
|
||||
let scheme = scheme::get_scheme(token.token(), desc.scheme)?;
|
||||
let scheme = desc.get_scheme(token)?;
|
||||
|
||||
op(&*scheme, file.description, desc, token)
|
||||
}
|
||||
@@ -165,14 +165,18 @@ pub fn openat_into(
|
||||
token: &mut CleanLockToken,
|
||||
) -> Result<FileHandle> {
|
||||
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
|
||||
let (scheme_id, number) = {
|
||||
let desc = {
|
||||
let current_lock = context::current();
|
||||
let mut current = current_lock.read(token.token());
|
||||
let (context, mut token) = current.token_split();
|
||||
let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?;
|
||||
let desc = pipe.description.read(token.token());
|
||||
(desc.scheme, desc.number)
|
||||
let (context, mut context_token) = current.token_split();
|
||||
let pipe = context
|
||||
.get_file(fh, &mut context_token)
|
||||
.ok_or(Error::new(EBADF))?;
|
||||
*pipe.description.read(context_token.token())
|
||||
};
|
||||
let scheme = desc.get_scheme(token)?;
|
||||
let number = desc.number;
|
||||
let scheme_id = desc.scheme;
|
||||
let caller_ctx = context::current().read(token.token()).caller_ctx();
|
||||
let new_description = {
|
||||
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
|
||||
@@ -560,8 +564,6 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
|
||||
|
||||
// Communicate fcntl with scheme
|
||||
if cmd != F_GETFD && cmd != F_SETFD {
|
||||
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
|
||||
|
||||
scheme.fcntl(number, cmd, arg, token)?;
|
||||
};
|
||||
|
||||
@@ -654,13 +656,11 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken
|
||||
let path = RedoxPath::from_absolute(&path_buf).ok_or(Error::new(EINVAL))?;
|
||||
let (_, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
|
||||
|
||||
let (number, scheme_id) = {
|
||||
let desc = file.description.read(token.token());
|
||||
(desc.number, desc.scheme)
|
||||
let (number, scheme) = {
|
||||
let desc = *file.description.read(token.token());
|
||||
(desc.number, desc.get_scheme(token)?)
|
||||
};
|
||||
|
||||
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
|
||||
|
||||
// TODO: Check EXDEV.
|
||||
/*
|
||||
if scheme_id != description.scheme {
|
||||
@@ -866,10 +866,7 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) ->
|
||||
Ok(bytes_read)
|
||||
}
|
||||
pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
|
||||
if fd.get() <= 5 || fd.get() > 0x4000000000000000 {
|
||||
let name = context::current().read(token.token()).name;
|
||||
info!("sys_write ENTER fd={} buf_len={} ctx='{}'", fd.get(), buf.len(), name);
|
||||
}
|
||||
|
||||
let result = (|| {
|
||||
let (bytes_written, desc_arc, desc) =
|
||||
file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| {
|
||||
|
||||
+18
-17
@@ -148,9 +148,7 @@ const KERNEL_METADATA_PAGE_COUNT: usize = syscall::KERNEL_METADATA_SIZE / PAGE_S
|
||||
};
|
||||
|
||||
pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockToken) {
|
||||
info!("usermode_bootstrap: ENTER, page_count={}", bootstrap.page_count);
|
||||
assert_ne!(bootstrap.page_count, 0);
|
||||
info!("usermode_bootstrap: page_count ok, getting addr_space");
|
||||
|
||||
{
|
||||
let addr_space = Arc::clone(
|
||||
@@ -159,7 +157,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
|
||||
.addr_space()
|
||||
.expect("expected bootstrap context to have an address space"),
|
||||
);
|
||||
info!("usermode_bootstrap: got addr_space, calling mmap for {} pages", bootstrap.page_count);
|
||||
|
||||
|
||||
let base = Page::containing_address(VirtualAddress::new(PAGE_SIZE));
|
||||
let flags = MapFlags::MAP_FIXED_NOREPLACE
|
||||
@@ -173,7 +171,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
|
||||
let _base_page = {
|
||||
let mut lock_token = token.token();
|
||||
let mut addr_space_lock = addr_space.acquire_write(lock_token.downgrade());
|
||||
info!("usermode_bootstrap: acquired write lock, calling mmap");
|
||||
|
||||
addr_space_lock
|
||||
.mmap(
|
||||
&addr_space,
|
||||
@@ -194,7 +192,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
|
||||
)
|
||||
.expect("Failed to allocate bootstrap pages")
|
||||
};
|
||||
info!("usermode_bootstrap: mmap done");
|
||||
|
||||
|
||||
// Insert kernel schemes root capabilities.
|
||||
let mut kernel_schemes_infos =
|
||||
@@ -217,7 +215,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
|
||||
matches!(scheme, GlobalSchemes::Proc),
|
||||
token,
|
||||
);
|
||||
info!("inserted fd {:#x} for scheme_id {} cap_fd {}", inserted, scheme.scheme_id().get(), cap_fd);
|
||||
|
||||
inserted
|
||||
};
|
||||
}
|
||||
@@ -300,7 +298,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
|
||||
.expect("failed to copy memory to bootstrap");
|
||||
|
||||
let bootstrap_entry = u64::from_le_bytes(bootstrap_slice[0x1a..0x22].try_into().unwrap());
|
||||
info!("Bootstrap entry point: {:#x}, pages: {}, stack at {:#x}", bootstrap_entry, bootstrap.page_count, PAGE_SIZE + bootstrap.page_count * PAGE_SIZE + 8 * PAGE_SIZE);
|
||||
|
||||
assert_ne!(bootstrap_entry, 0);
|
||||
|
||||
// Map a minimal user stack for the bootstrap process. Without this,
|
||||
@@ -363,7 +361,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockTok
|
||||
regs.set_instr_pointer(bootstrap_entry.try_into().unwrap());
|
||||
regs.set_stack_pointer(stack_top_vaddr);
|
||||
}
|
||||
info!("usermode_bootstrap: EXIT, entry={:#x}, sp={:#x}", bootstrap_entry, stack_top_vaddr);
|
||||
|
||||
}
|
||||
|
||||
unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8] {
|
||||
@@ -376,23 +374,26 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8]
|
||||
}
|
||||
|
||||
fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize {
|
||||
let description = Arc::new(RwLock::new(FileDescription::new(
|
||||
scheme,
|
||||
number,
|
||||
0,
|
||||
(O_CREAT | O_RDWR) as u32,
|
||||
InternalFlags::empty(),
|
||||
token,
|
||||
)));
|
||||
|
||||
let current_lock = context::current();
|
||||
let mut current = current_lock.read(token.token());
|
||||
let (context, mut token) = current.token_split();
|
||||
let (context, mut context_token) = current.token_split();
|
||||
context
|
||||
.add_file_min(
|
||||
FileDescriptor {
|
||||
description: Arc::new(RwLock::new(FileDescription {
|
||||
scheme,
|
||||
number,
|
||||
offset: 0,
|
||||
flags: (O_CREAT | O_RDWR) as u32,
|
||||
internal_flags: InternalFlags::empty(),
|
||||
})),
|
||||
description,
|
||||
cloexec,
|
||||
},
|
||||
syscall::flag::UPPER_FDTBL_TAG + scheme.get(),
|
||||
&mut token,
|
||||
&mut context_token,
|
||||
)
|
||||
.expect("failed to insert fd to current context")
|
||||
.get()
|
||||
|
||||
Reference in New Issue
Block a user