Merge master into 0.2.0 (688 commits, theirs pref for conflicts)

This commit is contained in:
2026-05-21 21:37:43 +03:00
3898 changed files with 4062828 additions and 147781 deletions
@@ -3,8 +3,11 @@ use core::{
sync::atomic::{AtomicU8, Ordering},
};
<<<<<<< HEAD
use x86::time::rdtsc;
=======
>>>>>>> master
use crate::{
arch::{
device::local_apic::the_local_apic,
@@ -12,14 +15,20 @@ use crate::{
},
cpu_set::LogicalCpuId,
memory::{
<<<<<<< HEAD
allocate_p2frame, map_device_memory, Frame, KernelMapper, Page, PageFlags,
PhysicalAddress, RmmA, RmmArch, VirtualAddress, PAGE_SIZE,
=======
allocate_p2frame, Frame, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch,
VirtualAddress, PAGE_SIZE,
>>>>>>> master
},
startup::AP_READY,
};
use super::{Madt, MadtEntry};
<<<<<<< HEAD
use alloc::collections::BTreeSet;
use alloc::vec::Vec;
@@ -137,6 +146,11 @@ fn apply_lapic_address_override(
debug!("Applied LAPIC address override: {:#x}", address);
}
=======
const TRAMPOLINE: usize = 0x8000;
static TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trampoline"));
>>>>>>> master
pub(super) fn init(madt: Madt) {
let local_apic = unsafe { the_local_apic() };
let me = local_apic.id();
@@ -148,10 +162,13 @@ pub(super) fn init(madt: Madt) {
}
if cfg!(not(feature = "multi_core")) {
<<<<<<< HEAD
unsafe {
record_apic_mapping(me.get(), LogicalCpuId::new(0));
}
crate::numa::init_default();
=======
>>>>>>> master
return;
}
@@ -162,6 +179,7 @@ pub(super) fn init(madt: Madt) {
//TODO: do not have writable and executable!
let mut mapper = KernelMapper::lock_rw();
<<<<<<< HEAD
let result = match mapper.map_phys(
trampoline_page.start_address(),
trampoline_frame.base(),
@@ -173,6 +191,15 @@ pub(super) fn init(madt: Madt) {
return;
}
};
=======
let result = mapper
.map_phys(
trampoline_page.start_address(),
trampoline_frame.base(),
PageFlags::new().execute(true).write(true),
)
.expect("failed to map trampoline");
>>>>>>> master
(result, mapper.table().phys().data())
};
@@ -185,6 +212,7 @@ pub(super) fn init(madt: Madt) {
}
}
<<<<<<< HEAD
// Detect whether MADT contains any LocalX2Apic entries.
// Some firmware (notably QEMU and some older BIOS) provides only 8-bit
// LocalApic entries even when the CPU supports x2APIC. In that case we must
@@ -427,6 +455,29 @@ pub(super) fn init(madt: Madt) {
);
continue;
}
=======
unsafe {
let preliminary_cpu_count = madt.iter().filter(|e| matches!(e, MadtEntry::LocalApic(entry) if u32::from(entry.id) == me.get() || entry.flags & 1 == 1)).count();
crate::profiling::allocate(preliminary_cpu_count as u32);
}
for madt_entry in madt.iter() {
debug!(" {:x?}", madt_entry);
if let MadtEntry::LocalApic(ap_local_apic) = madt_entry {
if u32::from(ap_local_apic.id) == me.get() {
debug!(" This is my local APIC");
} else if ap_local_apic.flags & 1 == 1 {
let cpu_id = LogicalCpuId::next();
// Allocate a stack
let stack_start = RmmA::phys_to_virt(
allocate_p2frame(4)
.expect("no more frames in acpi stack_start")
.base(),
)
.data();
let stack_end = stack_start + (PAGE_SIZE << 4);
>>>>>>> master
let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end);
@@ -452,6 +503,7 @@ pub(super) fn init(madt: Madt) {
#[expect(clippy::fn_to_numeric_cast)]
ap_code.write(kstart_ap as u64);
<<<<<<< HEAD
// Ensure all trampoline writes are visible to the AP before
// it starts executing. asm!("") is only a compiler barrier;
// fence(SeqCst) is a full hardware memory barrier.
@@ -467,6 +519,16 @@ pub(super) fn init(madt: Madt) {
{
// ICR: Delivery Mode=INIT(101), Level=Assert, Trigger=Edge
let mut icr = 0x4500u64;
=======
// TODO: Is this necessary (this fence)?
core::arch::asm!("");
};
AP_READY.store(false, Ordering::SeqCst);
// Send INIT IPI
{
let mut icr = 0x4500;
>>>>>>> master
if local_apic.x2 {
icr |= u64::from(ap_local_apic.id) << 32;
} else {
@@ -475,6 +537,7 @@ pub(super) fn init(madt: Madt) {
local_apic.set_icr(icr);
}
<<<<<<< HEAD
// Intel SDM Vol 3A §8.4.4: wait 10ms after INIT deassert
// before sending first SIPI. Modern CPUs may need less,
// but 10ms is the safe specification-compliant value.
@@ -486,11 +549,19 @@ pub(super) fn init(madt: Madt) {
// ICR: Delivery Mode=StartUp(110), Vector=ap_segment
// Note: bit 14 (Level) must be 0 for SIPI per Intel SDM.
let mut icr = 0x0600 | ap_segment as u64;
=======
// Send START IPI
{
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
let mut icr = 0x4600 | ap_segment as u64;
>>>>>>> master
if local_apic.x2 {
icr |= u64::from(ap_local_apic.id) << 32;
} else {
icr |= u64::from(ap_local_apic.id) << 56;
}
<<<<<<< HEAD
local_apic.set_icr(icr);
}
@@ -757,4 +828,30 @@ pub(super) fn init(madt: Madt) {
} else {
println!("KERNEL AP: failed to unmap trampoline page (non-fatal)");
}
=======
local_apic.set_icr(icr);
}
// Wait for trampoline ready
while unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } == 0 {
hint::spin_loop();
}
while !AP_READY.load(Ordering::SeqCst) {
hint::spin_loop();
}
RmmA::invalidate_all();
}
}
}
// Unmap trampoline
let (_frame, _, flush) = unsafe {
KernelMapper::lock_rw()
.unmap_phys(trampoline_page.start_address())
.expect("failed to unmap trampoline page")
};
flush.flush();
>>>>>>> master
}
@@ -34,12 +34,15 @@ impl Madt {
let madt = Madt::new(find_one_sdt!("APIC"));
if let Some(madt) = madt {
<<<<<<< HEAD
// Validate MADT checksum per ACPI 6.5 §5.2.2
if !madt.sdt.validate_checksum() {
error!("MADT checksum validation failed, skipping APIC initialization");
return;
}
=======
>>>>>>> master
// safe because no APs have been started yet.
unsafe { MADT.get().write(Some(madt)) };
@@ -152,6 +155,7 @@ pub struct MadtGicd {
_reserved2: [u8; 3],
}
<<<<<<< HEAD
/// MADT Local x2APIC (entry type 0x9)
#[derive(Clone, Copy, Debug)]
#[repr(C, packed)]
@@ -194,6 +198,8 @@ const _: () = assert!(size_of::<MadtLocalApicNmi>() == 4);
const _: () = assert!(size_of::<MadtLapicAddressOverride>() == 10);
const _: () = assert!(size_of::<MadtLocalX2ApicNmi>() == 10);
=======
>>>>>>> master
/// MADT Entries
#[derive(Debug)]
#[allow(dead_code)]
@@ -204,18 +210,24 @@ pub enum MadtEntry {
InvalidIoApic(usize),
IntSrcOverride(&'static MadtIntSrcOverride),
InvalidIntSrcOverride(usize),
<<<<<<< HEAD
LocalApicNmi(&'static MadtLocalApicNmi),
InvalidLocalApicNmi(usize),
LapicAddressOverride(&'static MadtLapicAddressOverride),
InvalidLapicAddressOverride(usize),
=======
>>>>>>> master
Gicc(&'static MadtGicc),
InvalidGicc(usize),
Gicd(&'static MadtGicd),
InvalidGicd(usize),
<<<<<<< HEAD
LocalX2Apic(&'static MadtLocalX2Apic),
InvalidLocalX2Apic(usize),
LocalX2ApicNmi(&'static MadtLocalX2ApicNmi),
InvalidLocalX2ApicNmi(usize),
=======
>>>>>>> master
Unknown(u8),
}
@@ -232,10 +244,13 @@ impl Iterator for MadtIter {
let entry_len =
unsafe { *(self.sdt.data_address() as *const u8).add(self.i + 1) } as usize;
<<<<<<< HEAD
if entry_len < 2 {
return None;
}
=======
>>>>>>> master
if self.i + entry_len <= self.sdt.data_len() {
let item = match entry_type {
0x0 => {
@@ -266,6 +281,7 @@ impl Iterator for MadtIter {
MadtEntry::InvalidIntSrcOverride(entry_len)
}
}
<<<<<<< HEAD
0x4 => {
if entry_len == size_of::<MadtLocalApicNmi>() + 2 {
MadtEntry::LocalApicNmi(unsafe {
@@ -306,6 +322,8 @@ impl Iterator for MadtIter {
MadtEntry::InvalidLocalX2ApicNmi(entry_len)
}
}
=======
>>>>>>> master
0xB => {
if entry_len >= size_of::<MadtGicc>() + 2 {
MadtEntry::Gicc(unsafe {
@@ -20,8 +20,11 @@ mod rxsdt;
pub mod sdt;
#[cfg(target_arch = "aarch64")]
mod spcr;
<<<<<<< HEAD
pub mod slit;
pub mod srat;
=======
>>>>>>> master
mod xsdt;
unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) {
@@ -84,6 +87,7 @@ impl Rxsdt for RxsdtEnum {
pub static RXSDT_ENUM: Once<RxsdtEnum> = Once::new();
<<<<<<< HEAD
#[derive(Clone, Copy, Debug)]
pub struct AcpiRootInfo {
pub revision: u8,
@@ -92,6 +96,8 @@ pub struct AcpiRootInfo {
pub static ACPI_ROOT_INFO: Once<AcpiRootInfo> = Once::new();
=======
>>>>>>> master
/// Parse the ACPI tables to gather CPU, interrupt, and timer information
pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
unsafe {
@@ -104,6 +110,7 @@ pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp);
if let Some(rsdp) = rsdp_opt {
<<<<<<< HEAD
let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo {
revision: rsdp.revision(),
root_sdt_address: rsdp.sdt_address(),
@@ -113,6 +120,8 @@ pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
error!("ACPI_ROOT_INFO already initialized with a different RSDP root");
}
=======
>>>>>>> master
debug!("SDT address: {:#x}", rsdp.sdt_address().data());
let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw());
@@ -165,6 +174,7 @@ pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
// TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface
// to initialize enumerated processors to userspace?
<<<<<<< HEAD
// Parse SRAT BEFORE MADT so NUMA node mapping is available
// when APs are started and PercpuBlocks are created.
srat::init();
@@ -173,6 +183,9 @@ pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
// Parse SLIT after MADT for the NUMA distance matrix.
slit::init();
=======
Madt::init();
>>>>>>> master
//TODO: support this on any arch
// SPCR must be initialized after MADT for interrupt controllers
#[cfg(target_arch = "aarch64")]
@@ -17,6 +17,7 @@ pub struct Rsdp {
impl Rsdp {
pub unsafe fn get_rsdp(already_supplied_rsdp: Option<*const u8>) -> Option<Rsdp> {
<<<<<<< HEAD
already_supplied_rsdp.and_then(|rsdp_ptr| {
let rsdp = unsafe { *(rsdp_ptr as *const Rsdp) };
@@ -44,6 +45,11 @@ impl Rsdp {
}
Some(rsdp)
=======
already_supplied_rsdp.map(|rsdp_ptr| {
// TODO: Validate
unsafe { *(rsdp_ptr as *const Rsdp) }
>>>>>>> master
})
}
@@ -55,8 +61,11 @@ impl Rsdp {
self.rsdt_address as usize
})
}
<<<<<<< HEAD
pub fn revision(&self) -> u8 {
self.revision
}
=======
>>>>>>> master
}
@@ -24,6 +24,7 @@ impl Sdt {
let header_size = size_of::<Sdt>();
total_size.saturating_sub(header_size)
}
<<<<<<< HEAD
/// Validate the SDT checksum.
///
@@ -40,4 +41,6 @@ impl Sdt {
.fold(0u8, |acc, &b| acc.wrapping_add(b));
sum == 0
}
=======
>>>>>>> master
}
@@ -7,6 +7,7 @@ mod linked_list;
/// Size of kernel heap
const KERNEL_HEAP_SIZE: usize = ::rmm::MEGABYTE;
<<<<<<< HEAD
#[cold]
fn halt_kernel_heap_init(message: &str) -> ! {
print!("{message}");
@@ -16,12 +17,15 @@ fn halt_kernel_heap_init(message: &str) -> ! {
}
}
=======
>>>>>>> master
unsafe fn map_heap(mapper: &mut KernelMapper<true>, offset: usize, size: usize) {
let mut flush_all = PageFlushAll::new();
let heap_start_page = Page::containing_address(VirtualAddress::new(offset));
let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1));
for page in Page::range_inclusive(heap_start_page, heap_end_page) {
<<<<<<< HEAD
let phys = match mapper.allocator_mut().allocate_one() {
Some(phys) => phys,
None => halt_kernel_heap_init(
@@ -41,6 +45,22 @@ unsafe fn map_heap(mapper: &mut KernelMapper<true>, offset: usize, size: usize)
"FATAL: failed to map kernel heap virtual page\n",
),
}
=======
let phys = mapper
.allocator_mut()
.allocate_one()
.expect("failed to allocate kernel heap");
let flush = unsafe {
mapper
.map_phys(
page.start_address(),
phys,
PageFlags::new()
.write(true)
.global(cfg!(not(feature = "pti"))),
)
.expect("failed to map kernel heap")
>>>>>>> master
};
flush_all.consume(flush);
}
@@ -91,7 +91,11 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
dtb::serial::init_early(dtb);
}
<<<<<<< HEAD
info!("RedBear OS starting...");
=======
info!("Redox OS starting...");
>>>>>>> master
args.print();
// Initialize RMM
@@ -97,7 +97,11 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
init_early(dtb);
}
<<<<<<< HEAD
info!("RedBear OS starting...");
=======
info!("Redox OS starting...");
>>>>>>> master
args.print();
if let Some(dtb) = &dtb {
@@ -14,10 +14,13 @@ pub struct IoApicRegs {
pointer: *const u32,
}
impl IoApicRegs {
<<<<<<< HEAD
fn redirection_index_valid(&mut self, idx: u8) -> bool {
idx <= self.max_redirection_table_entries()
}
=======
>>>>>>> master
fn ioregsel(&self) -> *const u32 {
self.pointer
}
@@ -48,6 +51,7 @@ impl IoApicRegs {
pub fn read_ioapicver(&mut self) -> u32 {
self.read_reg(0x01)
}
<<<<<<< HEAD
pub fn read_ioredtbl(&mut self, idx: u8) -> Option<u64> {
if !self.redirection_index_valid(idx) {
warn!("IOAPIC read_ioredtbl index {} out of range", idx);
@@ -63,13 +67,27 @@ impl IoApicRegs {
warn!("IOAPIC write_ioredtbl index {} out of range", idx);
return false;
}
=======
pub fn read_ioredtbl(&mut self, idx: u8) -> u64 {
assert!(idx < 24);
let lo = self.read_reg(0x10 + idx * 2);
let hi = self.read_reg(0x10 + idx * 2 + 1);
u64::from(lo) | (u64::from(hi) << 32)
}
pub fn write_ioredtbl(&mut self, idx: u8, value: u64) {
assert!(idx < 24);
>>>>>>> master
let lo = value as u32;
let hi = (value >> 32) as u32;
self.write_reg(0x10 + idx * 2, lo);
self.write_reg(0x10 + idx * 2 + 1, hi);
<<<<<<< HEAD
true
=======
>>>>>>> master
}
pub fn max_redirection_table_entries(&mut self) -> u8 {
@@ -103,16 +121,22 @@ impl IoApic {
}
/// Map an interrupt vector to a physical local APIC ID of a processor (thus physical mode).
#[allow(dead_code)]
<<<<<<< HEAD
pub fn map(&self, idx: u8, info: MapInfo) -> bool {
let Some(raw) = info.as_raw() else {
return false;
};
self.regs.lock().write_ioredtbl(idx, raw)
=======
pub fn map(&self, idx: u8, info: MapInfo) {
self.regs.lock().write_ioredtbl(idx, info.as_raw())
>>>>>>> master
}
pub fn set_mask(&self, gsi: u32, mask: bool) {
let idx = (gsi - self.gsi_start) as u8;
let mut guard = self.regs.lock();
<<<<<<< HEAD
let Some(mut reg) = guard.read_ioredtbl(idx) else {
return;
};
@@ -134,6 +158,12 @@ impl IoApic {
entry &= !(0xFF_u64 << 56);
entry |= u64::from(dest.get()) << 56;
guard.write_ioredtbl(idx, entry)
=======
let mut reg = guard.read_ioredtbl(idx);
reg &= !(1 << 16);
reg |= u64::from(mask) << 16;
guard.write_ioredtbl(idx, reg);
>>>>>>> master
}
}
@@ -180,6 +210,7 @@ pub struct MapInfo {
}
impl MapInfo {
<<<<<<< HEAD
pub fn as_raw(&self) -> Option<u64> {
if !(0x20..=0xFE).contains(&self.vector) {
warn!(
@@ -193,13 +224,26 @@ impl MapInfo {
Some(
(u64::from(self.dest.get()) << 56)
=======
pub fn as_raw(&self) -> u64 {
assert!(self.vector >= 0x20);
assert!(self.vector <= 0xFE);
// TODO: Check for reserved fields.
(u64::from(self.dest.get()) << 56)
>>>>>>> master
| (u64::from(self.mask) << 16)
| ((self.trigger_mode as u64) << 15)
| ((self.polarity as u64) << 13)
| ((self.dest_mode as u64) << 11)
| ((self.delivery_mode as u64) << 8)
<<<<<<< HEAD
| u64::from(self.vector),
)
=======
| u64::from(self.vector)
>>>>>>> master
}
}
@@ -213,7 +257,11 @@ impl fmt::Debug for IoApic {
let count = guard.max_redirection_table_entries();
f.debug_list()
<<<<<<< HEAD
.entries((0..=count).filter_map(|i| guard.read_ioredtbl(i)))
=======
.entries((0..count).map(|i| guard.read_ioredtbl(i)))
>>>>>>> master
.finish()
}
}
@@ -275,6 +323,7 @@ pub unsafe fn handle_ioapic(madt_ioapic: &'static MadtIoApic) {
let ioapic_registers = virt.data() as *const u32;
let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base);
<<<<<<< HEAD
let detected_id = ioapic.regs.lock().id();
if detected_id != madt_ioapic.id {
warn!(
@@ -283,6 +332,13 @@ pub unsafe fn handle_ioapic(madt_ioapic: &'static MadtIoApic) {
detected_id
);
}
=======
assert_eq!(
ioapic.regs.lock().id(),
madt_ioapic.id,
"mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC"
);
>>>>>>> master
(*IOAPICS.get()).get_or_insert_with(Vec::new).push(ioapic);
}
@@ -351,11 +407,19 @@ pub unsafe fn init() {
}
}
}
<<<<<<< HEAD
for ioapic in ioapics() {
for idx in 0..=ioapic.count {
ioapic.set_mask(ioapic.gsi_start + u32::from(idx), true);
}
}
=======
println!(
"I/O APICs: {:?}, overrides: {:?}",
ioapics(),
src_overrides()
);
>>>>>>> master
// map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it
// wouldn't have been disabled due to this I/O APIC)
@@ -370,6 +434,10 @@ pub unsafe fn init() {
.iter()
.any(|over| over.bus_irq == legacy_irq)
{
<<<<<<< HEAD
=======
// there's an IRQ conflict, making this legacy IRQ inaccessible.
>>>>>>> master
continue;
}
(
@@ -389,6 +457,10 @@ pub unsafe fn init() {
let redir_tbl_index = (gsi - apic.gsi_start) as u8;
let map_info = MapInfo {
<<<<<<< HEAD
=======
// only send to the BSP
>>>>>>> master
dest: bsp_apic_id,
dest_mode: DestinationMode::Physical,
delivery_mode: DeliveryMode::Fixed,
@@ -405,6 +477,7 @@ pub unsafe fn init() {
},
vector: 32 + legacy_irq,
};
<<<<<<< HEAD
if !apic.map(redir_tbl_index, map_info) {
warn!(
"Unable to map legacy IRQ {} (GSI {}) through IOAPIC index {}",
@@ -431,6 +504,9 @@ pub unsafe fn init() {
);
}
}
=======
apic.map(redir_tbl_index, map_info);
>>>>>>> master
}
println!(
"I/O APICs: {:?}, overrides: {:?}",
@@ -470,7 +546,11 @@ fn resolve(irq: u8) -> u32 {
fn find_ioapic(gsi: u32) -> Option<&'static IoApic> {
ioapics()
.iter()
<<<<<<< HEAD
.find(|apic| gsi >= apic.gsi_start && gsi <= apic.gsi_start + u32::from(apic.count))
=======
.find(|apic| gsi >= apic.gsi_start && gsi < apic.gsi_start + u32::from(apic.count))
>>>>>>> master
}
pub unsafe fn mask(irq: u8) {
@@ -489,6 +569,7 @@ pub unsafe fn unmask(irq: u8) {
};
apic.set_mask(gsi, false);
}
<<<<<<< HEAD
/// Change the destination CPU for an IRQ by reprogramming the IOAPIC redirection entry.
/// Resolves the legacy IRQ to its GSI, finds the owning IOAPIC, and updates the destination
@@ -500,3 +581,5 @@ pub unsafe fn set_affinity(irq: u8, dest: ApicId) -> bool {
None => false,
}
}
=======
>>>>>>> master
@@ -59,10 +59,17 @@ impl LocalApic {
.is_some_and(|feature_info| feature_info.has_x2apic());
if !self.x2 {
<<<<<<< HEAD
info!("Detected xAPIC at {:#x}", physaddr.data());
self.address = map_device_memory(physaddr, 4096).data();
} else {
info!("Detected x2APIC");
=======
debug!("Detected xAPIC at {:#x}", physaddr.data());
self.address = map_device_memory(physaddr, 4096).data();
} else {
debug!("Detected x2APIC");
>>>>>>> master
}
self.init_ap();
@@ -103,7 +110,11 @@ impl LocalApic {
ApicId::new(if self.x2 {
unsafe { rdmsr(IA32_X2APIC_APICID) as u32 }
} else {
<<<<<<< HEAD
unsafe { self.read(0x20) >> 24 }
=======
unsafe { self.read(0x20) }
>>>>>>> master
})
}
@@ -126,6 +137,7 @@ impl LocalApic {
pub fn set_icr(&mut self, value: u64) {
if self.x2 {
unsafe {
<<<<<<< HEAD
const PENDING: u32 = 1 << 12;
while (rdmsr(IA32_X2APIC_ICR) as u32) & PENDING == PENDING {
core::hint::spin_loop();
@@ -134,6 +146,9 @@ impl LocalApic {
while (rdmsr(IA32_X2APIC_ICR) as u32) & PENDING == PENDING {
core::hint::spin_loop();
}
=======
wrmsr(IA32_X2APIC_ICR, value);
>>>>>>> master
}
} else {
unsafe {
@@ -263,6 +278,7 @@ impl LocalApic {
}
}
}
<<<<<<< HEAD
pub unsafe fn set_lvt_nmi(&mut self, pin: u8, flags: u16) {
let polarity = match flags & 0b11 {
@@ -296,6 +312,8 @@ impl LocalApic {
}
}
=======
>>>>>>> master
unsafe fn setup_error_int(&mut self) {
unsafe {
let vector = 49u32;
@@ -4,11 +4,17 @@ pub mod cpu;
pub mod hpet;
pub mod ioapic;
pub mod local_apic;
<<<<<<< HEAD
pub mod msi;
pub mod pic;
pub mod pit;
pub mod serial;
pub mod vector;
=======
pub mod pic;
pub mod pit;
pub mod serial;
>>>>>>> master
#[cfg(feature = "system76_ec_debug")]
pub mod system76_ec;
@@ -25,7 +31,12 @@ pub unsafe fn init() {
}
}
pub unsafe fn init_after_acpi() {
<<<<<<< HEAD
unsafe { ioapic::init() };
=======
// this will disable the IOAPIC if needed.
//ioapic::init(mapper);
>>>>>>> master
}
unsafe fn init_hpet() -> bool {
@@ -192,6 +192,7 @@ impl ProcessorControlRegion {
}
}
<<<<<<< HEAD
#[cold]
fn halt_pcr_init() -> ! {
println!("FATAL: failed to allocate physical memory for Processor Control Region");
@@ -201,6 +202,8 @@ fn halt_pcr_init() -> ! {
}
}
=======
>>>>>>> master
pub unsafe fn pcr() -> *mut ProcessorControlRegion {
unsafe {
// Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that
@@ -384,10 +387,14 @@ pub fn allocate_and_init_pcr(
.next_power_of_two()
.trailing_zeros();
<<<<<<< HEAD
let pcr_frame = match crate::memory::allocate_p2frame(alloc_order) {
Some(frame) => frame,
None => halt_pcr_init(),
};
=======
let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR");
>>>>>>> master
let pcr_ptr = RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion;
unsafe { core::ptr::write(pcr_ptr, ProcessorControlRegion::new_partial_init(cpu_id)) };
@@ -78,6 +78,7 @@ static INIT_BSP_IDT: SyncUnsafeCell<Idt> = SyncUnsafeCell::new(Idt::new());
pub(crate) static IDTS: RwLock<HashMap<LogicalCpuId, &'static mut Idt>> =
RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new()));
<<<<<<< HEAD
#[cold]
fn halt_idt_init() -> ! {
println!("FATAL: failed to allocate physical pages for backup interrupt stack");
@@ -87,6 +88,8 @@ fn halt_idt_init() -> ! {
}
}
=======
>>>>>>> master
#[inline]
pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool {
if cpu_id == LogicalCpuId::BSP {
@@ -110,8 +113,11 @@ pub fn set_reserved(cpu_id: LogicalCpuId, index: u8, reserved: bool) {
}
pub fn available_irqs_iter(cpu_id: LogicalCpuId) -> impl Iterator<Item = u8> + 'static {
<<<<<<< HEAD
let count = (32..=254).filter(|&index| !is_reserved(cpu_id, index)).count();
info!("available_irqs_iter: cpu_id={} count={}", cpu_id.get(), count);
=======
>>>>>>> master
(32..=254).filter(move |&index| !is_reserved(cpu_id, index))
}
@@ -172,10 +178,15 @@ pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt {
.or_insert_with(|| Box::leak(Box::new(Idt::new())));
use crate::memory::{RmmA, RmmArch};
<<<<<<< HEAD
let frames = match crate::memory::allocate_p2frame(4) {
Some(frames) => frames,
None => halt_idt_init(),
};
=======
let frames = crate::memory::allocate_p2frame(4)
.expect("failed to allocate pages for backup interrupt stack");
>>>>>>> master
// Physical pages are mapped linearly. So is the linearly mapped virtual memory.
let base_address = RmmA::phys_to_virt(frames.base());
@@ -1,5 +1,8 @@
<<<<<<< HEAD
use core::sync::atomic::{AtomicBool, Ordering};
=======
>>>>>>> master
use syscall::Exception;
use x86::irq::PageFaultError;
@@ -12,6 +15,7 @@ use crate::{
syscall::flag::*,
};
<<<<<<< HEAD
static NMI_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
unsafe fn nmi_raw_serial_write(bytes: &[u8]) {
@@ -28,6 +32,8 @@ unsafe fn nmi_raw_serial_write(bytes: &[u8]) {
}
}
=======
>>>>>>> master
interrupt_stack!(divide_by_zero, |stack| {
println!("Divide by zero");
stack.trace();
@@ -73,6 +79,7 @@ interrupt_stack!(non_maskable, @paranoid, |stack| {
#[cfg(not(all(target_arch = "x86_64", feature = "profiling")))]
{
<<<<<<< HEAD
if NMI_IN_PROGRESS.swap(true, Ordering::SeqCst) {
return;
}
@@ -102,6 +109,11 @@ interrupt_stack!(non_maskable, @paranoid, |stack| {
}
NMI_IN_PROGRESS.store(false, Ordering::SeqCst);
=======
// TODO: This will likely deadlock
println!("Non-maskable interrupt");
stack.dump();
>>>>>>> master
}
});
@@ -28,8 +28,11 @@ pub mod pti;
/// Initialization and start function
pub mod start;
<<<<<<< HEAD
pub mod sleep;
=======
>>>>>>> master
/// Stop function
pub mod stop;
@@ -82,6 +82,7 @@ extern "C" fn kstart() {
/// The entry to Rust, all things must be initialized
unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
unsafe {
<<<<<<< HEAD
// EARLY CANARY: write 'R' to COM1 before any kernel init.
// This proves the serial hardware works and the kernel reached Rust entry.
// If this character appears but "RedBear OS starting..." does not,
@@ -91,6 +92,8 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'R', options(nostack, preserves_flags));
}
=======
>>>>>>> master
let bootstrap = {
let args = args_ptr.read();
@@ -100,6 +103,7 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Set up graphical debug
graphical_debug::init(args.env());
<<<<<<< HEAD
// SECOND CANARY: write 'S' to COM1 after serial init.
// If 'R' appears but 'S' does not, the hang is in serial::init() or graphical_debug::init().
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
@@ -125,24 +129,41 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{ core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'3', options(nostack, preserves_flags)); }
=======
info!("Redox OS starting...");
args.print();
// Set up GDT
gdt::init_bsp(stack_end);
// Set up IDT
idt::init_bsp();
>>>>>>> master
// Initialize RMM
#[cfg(target_arch = "x86")]
crate::startup::memory::init(&args, Some(0x100000), Some(0x40000000));
#[cfg(target_arch = "x86_64")]
crate::startup::memory::init(&args, Some(0x100000), None);
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{ core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'4', options(nostack, preserves_flags)); }
=======
>>>>>>> master
// Initialize paging
paging::init();
#[cfg(target_arch = "x86_64")]
crate::arch::alternative::early_init(true);
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{ core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'5', options(nostack, preserves_flags)); }
=======
>>>>>>> master
// Set up syscall instruction
interrupt::syscall::init();
@@ -152,9 +173,12 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Activate memory logging
crate::log::init();
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{ core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'6', options(nostack, preserves_flags)); }
=======
>>>>>>> master
// Initialize miscellaneous processor features
#[cfg(target_arch = "x86_64")]
crate::arch::misc::init(LogicalCpuId::BSP);
@@ -162,9 +186,12 @@ unsafe extern "C" fn start(args_ptr: *const KernelArgs, stack_end: usize) -> ! {
// Initialize devices
device::init();
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{ core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'7', options(nostack, preserves_flags)); }
=======
>>>>>>> master
// Read ACPI tables, starts APs
if cfg!(feature = "acpi") {
crate::acpi::init(args.acpi_rsdp());
@@ -4,10 +4,23 @@ use crate::{
percpu::PercpuBlock,
syscall::FloatRegisters,
};
<<<<<<< HEAD
use core::{mem::offset_of, ptr};
use spin::Once;
use syscall::{EnvRegisters, Result};
=======
use core::{mem::offset_of, ptr, sync::atomic::AtomicBool};
use spin::Once;
use syscall::{EnvRegisters, Result};
/// This must be used by the kernel to ensure that context switches are done atomically
/// Compare and exchange this to true when beginning a context switch on any CPU
/// The `Context::switch_to` function will set it back to false, allowing other CPU's to switch
/// This must be done, as no locks can be held on the stack during switch
pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false);
>>>>>>> master
// 512 bytes for registers, extra bytes for fpcr and fpsr
pub const KFX_ALIGN: usize = 16;
@@ -2,11 +2,20 @@ use crate::{
arch::interrupt::InterruptStack, context::context::Kstack, memory::RmmA, percpu::PercpuBlock,
syscall::FloatRegisters,
};
<<<<<<< HEAD
use core::mem::offset_of;
=======
use core::{mem::offset_of, sync::atomic::AtomicBool};
>>>>>>> master
use rmm::{Arch, VirtualAddress};
use spin::Once;
use syscall::{error::*, EnvRegisters};
<<<<<<< HEAD
=======
pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false);
>>>>>>> master
pub const KFX_ALIGN: usize = 16;
#[derive(Clone, Debug, Default)]
@@ -1,4 +1,8 @@
<<<<<<< HEAD
use core::mem::offset_of;
=======
use core::{mem::offset_of, sync::atomic::AtomicBool};
>>>>>>> master
use rmm::{Arch, VirtualAddress};
use spin::Once;
use syscall::{error::*, EnvRegisters};
@@ -14,6 +18,15 @@ use crate::{
syscall::FloatRegisters,
};
<<<<<<< HEAD
=======
/// This must be used by the kernel to ensure that context switches are done atomically
/// Compare and exchange this to true when beginning a context switch on any CPU
/// The `Context::switch_to` function will set it back to false, allowing other CPU's to switch
/// This must be done, as no locks can be held on the stack during switch
pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false);
>>>>>>> master
const ST_RESERVED: u128 = 0xFFFF_FFFF_FFFF_0000_0000_0000_0000_0000;
pub const KFX_ALIGN: usize = 16;
@@ -1,5 +1,9 @@
use core::{
ptr::{addr_of, addr_of_mut},
<<<<<<< HEAD
=======
sync::atomic::AtomicBool,
>>>>>>> master
};
use crate::syscall::FloatRegisters;
@@ -11,6 +15,15 @@ use spin::Once;
use syscall::{error::*, EnvRegisters};
use x86::msr;
<<<<<<< HEAD
=======
/// This must be used by the kernel to ensure that context switches are done atomically
/// Compare and exchange this to true when beginning a context switch on any CPU
/// The `Context::switch_to` function will set it back to false, allowing other CPU's to switch
/// This must be done, as no locks can be held on the stack during switch
pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false);
>>>>>>> master
const ST_RESERVED: u128 = 0xFFFF_FFFF_FFFF_0000_0000_0000_0000_0000;
#[cfg(cpu_feature_never = "xsave")]
@@ -148,8 +148,11 @@ pub struct Context {
pub euid: u32,
pub egid: u32,
pub pid: usize,
<<<<<<< HEAD
/// Supplementary group IDs for access control decisions.
pub groups: Vec<u32>,
=======
>>>>>>> master
// See [`PreemptGuard`]
//
@@ -206,7 +209,10 @@ impl Context {
euid: 0,
egid: 0,
pid: 0,
<<<<<<< HEAD
groups: Vec::new(),
=======
>>>>>>> master
#[cfg(feature = "syscall_debug")]
syscall_debug_info: crate::syscall::debug::SyscallDebugInfo::default(),
@@ -482,7 +488,10 @@ impl Context {
uid: self.euid,
gid: self.egid,
pid: self.pid,
<<<<<<< HEAD
groups: self.groups.clone(),
=======
>>>>>>> master
}
}
}
@@ -4,7 +4,11 @@ use crate::{
event,
scheme::{self, SchemeId},
sync::{CleanLockToken, RwLock, L6},
<<<<<<< HEAD
syscall::error::{Error, Result, ESTALE},
=======
syscall::error::Result,
>>>>>>> master
};
use alloc::sync::Arc;
use syscall::{schemev2::NewFdFlags, RwFlags, O_APPEND, O_NONBLOCK};
@@ -18,7 +22,10 @@ pub struct FileDescription {
pub offset: u64,
/// The scheme that this file refers to
pub scheme: SchemeId,
<<<<<<< HEAD
pub scheme_generation: Option<u64>,
=======
>>>>>>> master
/// The number the scheme uses to refer to this file
pub number: usize,
/// The flags passed to open or fcntl(SETFL)
@@ -33,6 +40,7 @@ bitflags! {
}
}
impl FileDescription {
<<<<<<< HEAD
pub fn with_generation(
scheme: SchemeId,
scheme_generation: Option<u64>,
@@ -79,6 +87,8 @@ impl FileDescription {
scheme::get_scheme(token.token(), self.scheme)
}
=======
>>>>>>> master
pub fn rw_flags(&self, rw: RwFlags) -> u32 {
let mut ret = self.flags & !(O_NONBLOCK | O_APPEND) as u32;
if rw.contains(RwFlags::APPEND) {
@@ -123,7 +133,11 @@ impl FileDescription {
pub fn try_close(self, token: &mut CleanLockToken) -> Result<()> {
event::unregister_file(self.scheme, self.number, token);
<<<<<<< HEAD
let scheme = self.get_scheme(token)?;
=======
let scheme = scheme::get_scheme(token.token(), self.scheme)?;
>>>>>>> master
scheme.close(self.number, token)
}
@@ -132,12 +146,21 @@ impl FileDescription {
impl FileDescriptor {
pub fn close(self, token: &mut CleanLockToken) -> Result<()> {
{
<<<<<<< HEAD
let (desc, number, internal_flags) = {
let desc = self.description.read(token.token());
(*desc, desc.number, desc.internal_flags)
};
if internal_flags.contains(InternalFlags::NOTIFY_ON_NEXT_DETACH) {
let scheme = desc.get_scheme(token)?;
=======
let (scheme_id, number, internal_flags) = {
let desc = self.description.read(token.token());
(desc.scheme, desc.number, desc.internal_flags)
};
if internal_flags.contains(InternalFlags::NOTIFY_ON_NEXT_DETACH) {
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
>>>>>>> master
scheme.detach(number, token)?;
}
}
@@ -64,6 +64,7 @@ impl UnmapResult {
return Ok(());
};
<<<<<<< HEAD
let (scheme, number) = {
let desc = *description.read(token.token());
(desc.get_scheme(token)?, desc.number)
@@ -71,6 +72,16 @@ impl UnmapResult {
let funmap_result = scheme
.kfunmap(number, base_offset, self.size, self.flags, token);
=======
let (scheme_id, number) = {
let desc = description.write(token.token());
(desc.scheme, desc.number)
};
let scheme_opt = scheme::get_scheme(token.token(), scheme_id);
let funmap_result = scheme_opt
.and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags, token));
>>>>>>> master
if let Ok(fd) = Arc::try_unwrap(description) {
fd.into_inner().try_close(token)?;
@@ -2686,6 +2697,7 @@ fn correct_inner<'l>(
// XXX: This is cheating, but guaranteed we won't deadlock because we've dropped addr_space_guard
let mut token = unsafe { CleanLockToken::new() };
<<<<<<< HEAD
let desc = *file_ref.description.read(token.token());
let scheme = desc.get_scheme(&mut token).map_err(|_| PfError::Segv)?;
let scheme_number = desc.number;
@@ -2693,6 +2705,22 @@ fn correct_inner<'l>(
KernelSchemes::User(user) => user.inner,
_ => return Err(PfError::Segv),
};
=======
let (scheme_id, scheme_number) = {
let desc = &file_ref.description.read(token.token());
(desc.scheme, desc.number)
};
let user_inner = scheme::get_scheme(token.token(), scheme_id)
.ok()
.and_then(|s| {
if let KernelSchemes::User(user) = s {
Some(user.inner)
} else {
None
}
})
.ok_or(PfError::Segv)?;
>>>>>>> master
let offset = file_ref.base_offset as u64 + (pages_from_grant_start * PAGE_SIZE) as u64;
user_inner
@@ -14,8 +14,13 @@ use crate::{
memory::{RmmA, RmmArch, TableKind},
percpu::PercpuBlock,
sync::{
<<<<<<< HEAD
ArcRwLockWriteGuard, CleanLockToken, LockToken, McsMutex, McsMutexGuard, Mutex,
MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard, L0, L1, L2, L4,
=======
ArcRwLockWriteGuard, CleanLockToken, LockToken, Mutex, MutexGuard, RwLock, RwLockReadGuard,
RwLockWriteGuard, L0, L1, L2, L4,
>>>>>>> master
},
syscall::error::Result,
};
@@ -74,12 +79,19 @@ pub use self::arch::empty_cr3;
// the context file descriptors.
static CONTEXTS: RwLock<L2, BTreeSet<ContextRef>> = RwLock::new(BTreeSet::new());
<<<<<<< HEAD
// Actual context store for the scheduler — uses MCS fair spinlock to
// eliminate cache-line bouncing under multi-CPU contention.
static RUN_CONTEXTS: McsMutex<L1, RunContextData> = McsMutex::new(RunContextData::new());
// Context that has been pushed out from RUN_CONTEXTS after being idle.
// Uses regular Mutex (lower contention; wakeup_contexts uses try_lock).
=======
// Actual context store for the scheduler
static RUN_CONTEXTS: Mutex<L1, RunContextData> = Mutex::new(RunContextData::new());
// Context that has been pushed out from RUN_CONTEXTS after being idle
>>>>>>> master
static IDLE_CONTEXTS: Mutex<L2, VecDeque<WeakContextRef>> = Mutex::new(VecDeque::new());
pub struct RunContextData {
@@ -115,7 +127,11 @@ pub fn idle_contexts_try(
IDLE_CONTEXTS.try_lock(token)
}
<<<<<<< HEAD
pub fn run_contexts(token: LockToken<'_, L0>) -> McsMutexGuard<'_, L1, RunContextData> {
=======
pub fn run_contexts(token: LockToken<'_, L0>) -> MutexGuard<'_, L1, RunContextData> {
>>>>>>> master
RUN_CONTEXTS.lock(token)
}
@@ -15,7 +15,11 @@ use crate::{
use alloc::{sync::Arc, vec::Vec};
use core::{
cell::{Cell, RefCell},
<<<<<<< HEAD
mem,
=======
hint, mem,
>>>>>>> master
sync::atomic::Ordering,
};
use syscall::PtraceFlags;
@@ -26,11 +30,14 @@ enum UpdateResult {
Blocked,
}
<<<<<<< HEAD
/// Default number of PIT ticks before triggering a context switch.
/// At ~2.25 ms per tick, 3 ticks ≈ 6.75 ms timeslice.
/// Configurable per-CPU via `ContextSwitchPercpu::preempt_interval`.
const DEFAULT_PREEMPT_INTERVAL: usize = 3;
=======
>>>>>>> master
// A simple geometric series where value[i] ~= value[i - 1] * 1.25
const SCHED_PRIO_TO_WEIGHT: [usize; 40] = [
88761, 71755, 56483, 46273, 36291, 29154, 23254, 18705, 14949, 11916, 9548, 7620, 6100, 4904,
@@ -95,15 +102,24 @@ struct SwitchResultInner {
///
/// The function also calls the signal handler after switching contexts.
pub fn tick(token: &mut CleanLockToken) {
<<<<<<< HEAD
let percpu = PercpuBlock::current();
let ticks_cell = &percpu.switch_internals.pit_ticks;
=======
let ticks_cell = &PercpuBlock::current().switch_internals.pit_ticks;
>>>>>>> master
let new_ticks = ticks_cell.get() + 1;
ticks_cell.set(new_ticks);
<<<<<<< HEAD
// Trigger a context switch when the per-CPU preempt interval is reached.
let interval = percpu.switch_internals.preempt_interval.get();
if new_ticks >= interval {
=======
// Trigger a context switch after every 3 ticks (approx. 6.75 ms).
if new_ticks >= 3 {
>>>>>>> master
switch(token);
crate::context::signal::signal_handler(token);
}
@@ -127,10 +143,14 @@ pub unsafe extern "C" fn switch_finish_hook() {
crate::arch::stop::emergency_reset();
}
}
<<<<<<< HEAD
PercpuBlock::current()
.switch_internals
.in_context_switch
.set(false);
=======
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
>>>>>>> master
crate::percpu::switch_arch_hook();
}
}
@@ -160,6 +180,7 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
//set PIT Interrupt counter to 0, giving each process same amount of PIT ticks
percpu.switch_internals.pit_ticks.set(0);
<<<<<<< HEAD
// Acquire the per-CPU context switch flag. Each CPU can only be in one context
// switch at a time. The per-context write locks provide cross-CPU safety; this
// flag catches re-entrant switches on the same CPU (a kernel bug).
@@ -169,6 +190,18 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
percpu.cpu_id
);
percpu.switch_internals.in_context_switch.set(true);
=======
// Acquire the global lock to ensure exclusive access during context switch and avoid
// issues that would be caused by the unsafe operations below
// TODO: Better memory orderings?
while arch::CONTEXT_SWITCH_LOCK
.compare_exchange_weak(false, true, Ordering::SeqCst, Ordering::Relaxed)
.is_err()
{
hint::spin_loop();
percpu.maybe_handle_tlb_shootdown();
}
>>>>>>> master
// Lock the previous context.
let prev_context_lock = crate::context::current();
@@ -176,8 +209,13 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
let mut prev_context_guard = unsafe { prev_context_lock.write_arc() };
if !prev_context_guard.is_preemptable() {
<<<<<<< HEAD
// Unset per-CPU context switch flag
percpu.switch_internals.in_context_switch.set(false);
=======
// Unset global lock
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
>>>>>>> master
// Pretend to have finished switching, so CPU is not idled
return SwitchResult::Switched;
@@ -301,8 +339,13 @@ pub fn switch(token: &mut CleanLockToken) -> SwitchResult {
SwitchResult::Switched
}
_ => {
<<<<<<< HEAD
// No target was found, unset per-CPU context switch flag and return
percpu.switch_internals.in_context_switch.set(false);
=======
// No target was found, unset global lock and return
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
>>>>>>> master
percpu.stats.set_state(cpu_stats::CpuState::Idle);
@@ -361,7 +404,10 @@ fn wakeup_contexts(token: &mut CleanLockToken, switch_time: u128) -> Vec<(usize,
}
/// This is the scheduler function which currently utilises Deficit Weighted Round Robin Scheduler
<<<<<<< HEAD
/// with NUMA-aware context selection preference.
=======
>>>>>>> master
fn select_next_context(
token: &mut CleanLockToken,
percpu: &PercpuBlock,
@@ -387,10 +433,13 @@ fn select_next_context(
let total_contexts: usize = contexts_list.iter().map(|q| q.len()).sum();
let mut skipped_contexts = 0;
<<<<<<< HEAD
// NUMA-aware selection: remember cross-node fallback candidate.
let my_numa_node = percpu.numa_node.get();
let mut cross_node_fallback: Option<(usize, ArcContextLockWriteGuard)> = None;
=======
>>>>>>> master
'priority: loop {
i = (i + 1) % 40;
total_iters += 1;
@@ -455,6 +504,7 @@ fn select_next_context(
// Is this context runnable on this CPU?
let sw = unsafe { update_runnable(&mut next_context_guard, cpu_id, switch_time) };
if let UpdateResult::CanSwitch = sw {
<<<<<<< HEAD
// NUMA-aware selection: check if this context's last CPU was on the same node.
let same_node = if my_numa_node != u8::MAX {
next_context_guard.cpu_id
@@ -493,6 +543,11 @@ fn select_next_context(
continue;
}
}
=======
next_context_guard_opt = Some(next_context_guard);
balance[i] -= SCHED_PRIO_TO_WEIGHT[20];
break 'priority;
>>>>>>> master
} else {
if matches!(sw, UpdateResult::Blocked) {
idle_contexts(token.token()).push_back(next_context_ref);
@@ -507,6 +562,7 @@ fn select_next_context(
}
}
}
<<<<<<< HEAD
// If we found a cross-node fallback but no same-node context, use it
if next_context_guard_opt.is_none() {
@@ -516,6 +572,8 @@ fn select_next_context(
}
}
=======
>>>>>>> master
percpu.balance.set(balance);
percpu.last_queue.set(i);
@@ -523,10 +581,14 @@ fn select_next_context(
// Send the old process to the back of the line (if it is still runnable)
let prev_ctx = WeakContextRef(Arc::downgrade(&prev_context_lock));
if prev_context_guard.status.is_runnable() {
<<<<<<< HEAD
let raw_prio = prev_context_guard.prio;
let prio = percpu.effective_prio(raw_prio);
// Clear PI donation — previous context is being re-queued
percpu.pi_donated_prio.store(u32::MAX, Ordering::Relaxed);
=======
let prio = prev_context_guard.prio;
>>>>>>> master
contexts_list[prio].push_back(prev_ctx);
} else {
idle_contexts(token.token()).push_back(prev_ctx);
@@ -538,8 +600,12 @@ fn select_next_context(
return Ok(Some(next_context_guard));
} else {
if !was_idle && !Arc::ptr_eq(&prev_context_lock, &idle_context) {
<<<<<<< HEAD
// Switching to idle context — cache lowest priority
percpu.current_prio.set(39);
=======
// We switch into the idle context
>>>>>>> master
Ok(Some(unsafe { idle_context.write_arc() }))
} else {
// We found no other process to run.
@@ -556,6 +622,7 @@ pub struct ContextSwitchPercpu {
switch_result: Cell<Option<SwitchResultInner>>,
switch_time: Cell<u128>,
pit_ticks: Cell<usize>,
<<<<<<< HEAD
/// Per-CPU context switch flag. Set to true during a context switch on this CPU.
/// Replaced the global CONTEXT_SWITCH_LOCK to eliminate cross-CPU serialization.
in_context_switch: Cell<bool>,
@@ -563,6 +630,8 @@ pub struct ContextSwitchPercpu {
/// Default: 3 (≈6.75 ms). Lower values improve interactive responsiveness;
/// higher values improve throughput for batch/compute workloads.
preempt_interval: Cell<usize>,
=======
>>>>>>> master
current_ctxt: RefCell<Option<Arc<ContextLock>>>,
@@ -577,8 +646,11 @@ impl ContextSwitchPercpu {
switch_result: Cell::new(None),
switch_time: Cell::new(0),
pit_ticks: Cell::new(0),
<<<<<<< HEAD
in_context_switch: Cell::new(false),
preempt_interval: Cell::new(DEFAULT_PREEMPT_INTERVAL),
=======
>>>>>>> master
current_ctxt: RefCell::new(None),
idle_ctxt: RefCell::new(None),
being_sigkilled: Cell::new(false),
+11
View File
@@ -42,18 +42,29 @@ impl core::fmt::Display for LogicalCpuId {
}
#[cfg(target_pointer_width = "64")]
<<<<<<< HEAD
pub const MAX_CPU_COUNT: u32 = 256;
=======
pub const MAX_CPU_COUNT: u32 = 128;
>>>>>>> master
#[cfg(target_pointer_width = "32")]
pub const MAX_CPU_COUNT: u32 = 32;
const SET_WORDS: usize = (MAX_CPU_COUNT / usize::BITS) as usize;
<<<<<<< HEAD
// TODO: Support more than 256 CPUs.
// The maximum number of CPUs on Linux is configurable, and the type for LogicalCpuSet and
// LogicalCpuId may be optimized accordingly. In that case, box the mask if it's larger than some
// base size (probably 256 bytes). AMD EPYC has 128C/256T, Threadripper PRO 96C/192T —
// 256 covers current hardware.
=======
// TODO: Support more than 128 CPUs.
// The maximum number of CPUs on Linux is configurable, and the type for LogicalCpuSet and
// LogicalCpuId may be optimized accordingly. In that case, box the mask if it's larger than some
// base size (probably 256 bytes).
>>>>>>> master
#[derive(Debug)]
pub struct LogicalCpuSet([AtomicUsize; SET_WORDS]);
+10
View File
@@ -1,5 +1,9 @@
use alloc::sync::Arc;
<<<<<<< HEAD
use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
=======
use core::sync::atomic::{AtomicUsize, Ordering};
>>>>>>> master
use hashbrown::{hash_map::DefaultHashBuilder, HashMap};
use smallvec::SmallVec;
use syscall::data::GlobalSchemes;
@@ -23,7 +27,10 @@ int_like!(EventQueueId, AtomicEventQueueId, usize, AtomicUsize);
pub struct EventQueue {
id: EventQueueId,
queue: WaitQueue<Event>,
<<<<<<< HEAD
pub eventfd: Option<(AtomicU64, bool)>, // (counter, semaphore_mode)
=======
>>>>>>> master
}
impl EventQueue {
@@ -31,6 +38,7 @@ impl EventQueue {
EventQueue {
id,
queue: WaitQueue::new(),
<<<<<<< HEAD
eventfd: None,
}
}
@@ -40,6 +48,8 @@ impl EventQueue {
id,
queue: WaitQueue::new(),
eventfd: Some((AtomicU64::new(initval), semaphore)),
=======
>>>>>>> master
}
}
+3
View File
@@ -70,9 +70,12 @@ mod log;
/// Memory management
mod memory;
<<<<<<< HEAD
/// NUMA topology
mod numa;
=======
>>>>>>> master
/// Panic
mod panic;
+38
View File
@@ -4,6 +4,7 @@ use alloc::{
};
use core::{
cell::{Cell, RefCell},
<<<<<<< HEAD
hint,
sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, Ordering},
};
@@ -12,6 +13,11 @@ use core::{
/// back to a full TLB flush (CR3 reload).
const TLB_RANGE_THRESHOLD: u32 = 32;
=======
sync::atomic::{AtomicBool, AtomicPtr, Ordering},
};
>>>>>>> master
use rmm::Arch;
use syscall::PtraceFlags;
@@ -21,7 +27,11 @@ use crate::{
cpu_set::{LogicalCpuId, MAX_CPU_COUNT},
cpu_stats::{CpuStats, CpuStatsData},
ptrace::Session,
<<<<<<< HEAD
sync::{mcs::McsNode, mcs::McsRawLock, CleanLockToken},
=======
sync::CleanLockToken,
>>>>>>> master
syscall::debug::SyscallDebugInfo,
};
@@ -39,6 +49,7 @@ pub struct PercpuBlock {
pub balance: Cell<[usize; 40]>,
pub last_queue: Cell<usize>,
<<<<<<< HEAD
/// Per-CPU MCS node for the scheduler run-queue lock (RUN_CONTEXTS).
pub mcs_sched_node: McsNode,
@@ -71,6 +82,8 @@ pub struct PercpuBlock {
/// entering the spin loop, cleared upon acquisition.
pub waiting_on_lock: AtomicPtr<McsRawLock>,
=======
>>>>>>> master
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
pub profiling: Option<&'static crate::profiling::RingBuffer>,
@@ -94,6 +107,7 @@ pub unsafe fn init_tlb_shootdown(id: LogicalCpuId, block: *mut PercpuBlock) {
ALL_PERCPU_BLOCKS[id.get() as usize].store(block, Ordering::Release)
}
<<<<<<< HEAD
/// Get a reference to another CPU's PercpuBlock by logical CPU ID.
pub fn get_for_cpu(id: LogicalCpuId) -> Option<&'static PercpuBlock> {
unsafe {
@@ -103,6 +117,8 @@ pub fn get_for_cpu(id: LogicalCpuId) -> Option<&'static PercpuBlock> {
}
}
=======
>>>>>>> master
pub fn get_all_stats() -> Vec<(LogicalCpuId, CpuStatsData)> {
let mut res = ALL_PERCPU_BLOCKS
.iter()
@@ -147,6 +163,7 @@ pub fn shootdown_tlb_ipi(target: Option<LogicalCpuId>) {
core::hint::spin_loop();
}
}
<<<<<<< HEAD
// Full flush — clear range info (Release ordering ensures the flag
// swap and these stores are visible to the handler before the IPI).
percpublock.tlb_flush_start.store(0, Ordering::Release);
@@ -270,12 +287,26 @@ impl PercpuBlock {
}
}
=======
crate::ipi::ipi_single(crate::ipi::IpiKind::Tlb, percpublock);
} else {
for id in 0..crate::cpu_count() {
// TODO: Optimize: use global counter and percpu ack counters, send IPI using
// destination shorthand "all CPUs".
shootdown_tlb_ipi(Some(LogicalCpuId::new(id)));
}
}
}
impl PercpuBlock {
>>>>>>> master
pub fn maybe_handle_tlb_shootdown(&self) {
#[expect(clippy::bool_comparison)]
if self.wants_tlb_shootdown.swap(false, Ordering::Relaxed) == false {
return;
}
<<<<<<< HEAD
let start = self.tlb_flush_start.load(Ordering::Acquire);
let count = self.tlb_flush_count.load(Ordering::Acquire);
@@ -289,6 +320,10 @@ impl PercpuBlock {
// Full TLB flush (CR3 reload) for large ranges or global shootdowns.
crate::memory::RmmA::invalidate_all();
}
=======
// TODO: Finer-grained flush
crate::memory::RmmA::invalidate_all();
>>>>>>> master
if let Some(addrsp) = &*self.current_addrsp.borrow() {
addrsp.tlb_ack.fetch_add(1, Ordering::Release);
@@ -358,6 +393,7 @@ impl PercpuBlock {
wants_tlb_shootdown: AtomicBool::new(false),
balance: Cell::new([0; 40]),
last_queue: Cell::new(39),
<<<<<<< HEAD
mcs_sched_node: McsNode::new(),
mcs_contention_count: Cell::new(0),
tlb_flush_start: AtomicU64::new(0),
@@ -366,6 +402,8 @@ impl PercpuBlock {
current_prio: Cell::new(39),
numa_node: Cell::new(u8::MAX),
waiting_on_lock: AtomicPtr::new(core::ptr::null_mut()),
=======
>>>>>>> master
ptrace_flags: Cell::new(PtraceFlags::empty()),
ptrace_session: RefCell::new(None),
inside_syscall: Cell::new(false),
@@ -10,7 +10,10 @@ use syscall::{
use crate::{
acpi::{RxsdtEnum, RXSDT_ENUM},
<<<<<<< HEAD
arch::sleep,
=======
>>>>>>> master
context::file::InternalFlags,
event,
sync::{CleanLockToken, RwLock, WaitCondition, L1},
@@ -41,7 +44,10 @@ enum HandleKind {
TopLevel,
Rxsdt,
ShutdownPipe,
<<<<<<< HEAD
SleepControl,
=======
>>>>>>> master
SchemeRoot,
}
@@ -148,11 +154,19 @@ impl KernelScheme for AcpiScheme {
if flags & O_EXCL == O_EXCL || flags & O_SYMLINK == O_SYMLINK {
return Err(Error::new(EINVAL));
}
<<<<<<< HEAD
let (handle_kind, int_flags) = match path {
"" => {
if flags & O_ACCMODE != O_RDONLY && flags & O_STAT != O_STAT {
return Err(Error::new(EROFS));
}
=======
if flags & O_ACCMODE != O_RDONLY && flags & O_STAT != O_STAT {
return Err(Error::new(EROFS));
}
let (handle_kind, int_flags) = match path {
"" => {
>>>>>>> master
if flags & O_DIRECTORY != O_DIRECTORY && flags & O_STAT != O_STAT {
return Err(Error::new(EISDIR));
}
@@ -160,23 +174,30 @@ impl KernelScheme for AcpiScheme {
(HandleKind::TopLevel, InternalFlags::POSITIONED)
}
"rxsdt" => {
<<<<<<< HEAD
if flags & O_ACCMODE != O_RDONLY && flags & O_STAT != O_STAT {
return Err(Error::new(EROFS));
}
=======
>>>>>>> master
if flags & O_DIRECTORY == O_DIRECTORY && flags & O_STAT != O_STAT {
return Err(Error::new(ENOTDIR));
}
(HandleKind::Rxsdt, InternalFlags::POSITIONED)
}
"kstop" => {
<<<<<<< HEAD
if flags & O_ACCMODE != O_RDONLY && flags & O_STAT != O_STAT {
return Err(Error::new(EROFS));
}
=======
>>>>>>> master
if flags & O_DIRECTORY == O_DIRECTORY && flags & O_STAT != O_STAT {
return Err(Error::new(ENOTDIR));
}
(HandleKind::ShutdownPipe, InternalFlags::empty())
}
<<<<<<< HEAD
"sleep" => {
if flags & O_ACCMODE == O_RDONLY || flags & O_STAT == O_STAT {
// allowed
@@ -190,6 +211,8 @@ impl KernelScheme for AcpiScheme {
}
(HandleKind::SleepControl, InternalFlags::POSITIONED)
}
=======
>>>>>>> master
_ => return Err(Error::new(ENOENT)),
};
@@ -212,7 +235,10 @@ impl KernelScheme for AcpiScheme {
Ok(match handle.kind {
HandleKind::Rxsdt => DATA.get().ok_or(Error::new(EBADFD))?.len() as u64,
HandleKind::ShutdownPipe => 1,
<<<<<<< HEAD
HandleKind::SleepControl => sleep::available_sleep_states().len() as u64,
=======
>>>>>>> master
HandleKind::TopLevel => 0,
HandleKind::SchemeRoot => return Err(Error::new(EBADF))?,
})
@@ -275,7 +301,10 @@ impl KernelScheme for AcpiScheme {
return dst_buf.copy_exactly(&[0x42]).map(|()| 1);
}
<<<<<<< HEAD
HandleKind::SleepControl => sleep::available_sleep_states(),
=======
>>>>>>> master
HandleKind::Rxsdt => DATA.get().ok_or(Error::new(EBADFD))?,
HandleKind::TopLevel => return Err(Error::new(EISDIR)),
HandleKind::SchemeRoot => return Err(Error::new(EBADF)),
@@ -318,6 +347,7 @@ impl KernelScheme for AcpiScheme {
kind: DirentKind::Socket,
name: "kstop",
inode: 0,
<<<<<<< HEAD
next_opaque_id: 2,
})?;
}
@@ -326,11 +356,14 @@ impl KernelScheme for AcpiScheme {
kind: DirentKind::Regular,
name: "sleep",
inode: 0,
=======
>>>>>>> master
next_opaque_id: u64::MAX,
})?;
}
Ok(buf.finalize())
}
<<<<<<< HEAD
fn kwrite(
&self,
id: usize,
@@ -357,6 +390,8 @@ impl KernelScheme for AcpiScheme {
_ => Err(Error::new(EBADF)),
}
}
=======
>>>>>>> master
fn kfpath(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<usize> {
//TODO: construct useful path?
buf.copy_common_bytes_from_slice("/scheme/kernel.acpi/".as_bytes())
@@ -385,11 +420,14 @@ impl KernelScheme for AcpiScheme {
st_size: 1,
..Default::default()
},
<<<<<<< HEAD
HandleKind::SleepControl => Stat {
st_mode: MODE_FILE,
st_size: sleep::available_sleep_states().len().try_into().unwrap_or(u64::MAX),
..Default::default()
},
=======
>>>>>>> master
HandleKind::SchemeRoot => return Err(Error::new(EBADF)),
})?;
@@ -22,10 +22,16 @@ struct Handle {
static HANDLES: RwLock<L1, HandleMap<Handle>> = RwLock::new(HandleMap::new());
<<<<<<< HEAD
/// Add to the input queue, translating CR to NL (ICRNL) for serial console compatibility.
pub fn debug_input(data: u8, token: &mut CleanLockToken) {
let translated = if data == b'\r' { b'\n' } else { data };
INPUT.send(translated, token);
=======
/// Add to the input queue
pub fn debug_input(data: u8, token: &mut CleanLockToken) {
INPUT.send(data, token);
>>>>>>> master
}
// Notify readers of input updates
@@ -107,16 +113,24 @@ impl KernelScheme for DebugScheme {
fn fevent(
&self,
id: usize,
<<<<<<< HEAD
flags: EventFlags,
=======
_flags: EventFlags,
>>>>>>> master
token: &mut CleanLockToken,
) -> Result<EventFlags> {
let _handle = *HANDLES.read(token.token()).get(id)?;
<<<<<<< HEAD
let mut ready = EventFlags::empty();
if flags.contains(EventFlags::EVENT_READ) {
ready |= EventFlags::EVENT_READ;
}
Ok(ready)
=======
Ok(EventFlags::empty())
>>>>>>> master
}
fn fsync(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
@@ -1,5 +1,8 @@
use alloc::sync::Arc;
<<<<<<< HEAD
use core::sync::atomic::Ordering;
=======
>>>>>>> master
use syscall::{EventFlags, O_NONBLOCK};
use crate::{
@@ -26,12 +29,17 @@ impl KernelScheme for EventScheme {
fn kopenat(
&self,
id: usize,
<<<<<<< HEAD
user_buf: StrOrBytes,
=======
_user_buf: StrOrBytes,
>>>>>>> master
_flags: usize,
_fcntl_flags: u32,
_ctx: CallerCtx,
token: &mut CleanLockToken,
) -> Result<OpenResult> {
<<<<<<< HEAD
let path = match &user_buf {
StrOrBytes::Str(s) => s,
StrOrBytes::Bytes(b) => core::str::from_utf8(b).unwrap_or(""),
@@ -45,6 +53,8 @@ impl KernelScheme for EventScheme {
queues_mut(token.token()).insert(id, Arc::new(EventQueue::new_eventfd(id, initval, sem)));
return Ok(OpenResult::SchemeLocal(id.get(), InternalFlags::empty()));
}
=======
>>>>>>> master
if id != SCHEME_ROOT_ID {
return Err(Error::new(EACCES));
}
@@ -81,6 +91,7 @@ impl KernelScheme for EventScheme {
handle.clone()
};
<<<<<<< HEAD
if let Some((ref counter, semaphore)) = queue.eventfd {
let is_nonblock = flags & O_NONBLOCK as u32 != 0;
if semaphore {
@@ -106,6 +117,8 @@ impl KernelScheme for EventScheme {
}
}
=======
>>>>>>> master
queue.read(buf, flags & O_NONBLOCK as u32 == 0, token)
}
@@ -124,6 +137,7 @@ impl KernelScheme for EventScheme {
let handle = handles.get(&id).ok_or(Error::new(EBADF))?;
handle.clone()
};
<<<<<<< HEAD
if let Some((ref counter, _semaphore)) = queue.eventfd {
if buf.len() >= 8 {
@@ -137,6 +151,8 @@ impl KernelScheme for EventScheme {
return Err(Error::new(EINVAL));
}
=======
>>>>>>> master
let mut events_written = 0;
for chunk in buf.in_exact_chunks(size_of::<Event>()) {
@@ -19,9 +19,12 @@ use crate::context::file::InternalFlags;
use super::{CallerCtx, HandleMap, OpenResult, SchemeExt, StrOrBytes};
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
<<<<<<< HEAD
use crate::arch::device::{ioapic, local_apic::ApicId};
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
=======
>>>>>>> master
use crate::arch::interrupt::{available_irqs_iter, irq::acknowledge, is_reserved, set_reserved};
#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
use crate::dtb::irqchip::{acknowledge, available_irqs_iter, is_reserved, set_reserved, IRQ_CHIP};
@@ -59,11 +62,16 @@ const INO_AVAIL: u64 = 0x8000_0000_0000_0000;
const INO_BSP: u64 = 0x8001_0000_0000_0000;
const INO_PHANDLE: u64 = 0x8003_0000_0000_0000;
<<<<<<< HEAD
/// Add to the input queue, with iommu validation gate for MSI vectors
pub fn irq_trigger(irq: u8, token: &mut CleanLockToken) {
if irq >= 16 && !iommu_validate_msi_irq(irq) {
return;
}
=======
/// Add to the input queue
pub fn irq_trigger(irq: u8, token: &mut CleanLockToken) {
>>>>>>> master
COUNTS.lock()[irq as usize] += 1;
let fds: SmallVec<[usize; 8]> = {
HANDLES
@@ -83,17 +91,28 @@ pub fn irq_trigger(irq: u8, token: &mut CleanLockToken) {
#[allow(dead_code)]
enum Handle {
SchemeRoot,
<<<<<<< HEAD
Irq { ack: AtomicUsize, irq: u8, cpu_id: LogicalCpuId },
=======
Irq { ack: AtomicUsize, irq: u8 },
>>>>>>> master
Avail(LogicalCpuId),
TopLevel,
Phandle(u8, Vec<u8>),
Bsp,
<<<<<<< HEAD
IrqAffinity { irq: u8, mask: AtomicUsize },
=======
>>>>>>> master
}
impl Handle {
fn as_irq_handle(&self) -> Option<(&AtomicUsize, u8)> {
match self {
<<<<<<< HEAD
&Self::Irq { ref ack, irq, cpu_id: _ } => Some((ack, irq)),
=======
&Self::Irq { ref ack, irq } => Some((ack, irq)),
>>>>>>> master
_ => None,
}
}
@@ -147,7 +166,10 @@ impl IrqScheme {
Handle::Irq {
ack: AtomicUsize::new(0),
irq: irq_number,
<<<<<<< HEAD
cpu_id: LogicalCpuId::BSP,
=======
>>>>>>> master
},
InternalFlags::empty(),
)
@@ -166,7 +188,10 @@ impl IrqScheme {
Handle::Irq {
ack: AtomicUsize::new(0),
irq: irq_number,
<<<<<<< HEAD
cpu_id,
=======
>>>>>>> master
},
InternalFlags::empty(),
)
@@ -208,7 +233,10 @@ impl IrqScheme {
Handle::Irq {
ack: AtomicUsize::new(0),
irq: irq_number as u8,
<<<<<<< HEAD
cpu_id: LogicalCpuId::new(0),
=======
>>>>>>> master
},
InternalFlags::empty(),
)
@@ -224,6 +252,7 @@ const fn vector_to_irq(vector: u8) -> u8 {
vector - 32
}
<<<<<<< HEAD
const fn msi_vector_is_valid(vector: u8) -> bool {
vector >= 32 && vector < 0xEF
}
@@ -232,6 +261,8 @@ fn iommu_validate_msi_irq(_irq: u8) -> bool {
true
}
=======
>>>>>>> master
impl crate::scheme::KernelScheme for IrqScheme {
fn scheme_root(&self, token: &mut CleanLockToken) -> Result<usize> {
let id = HANDLES.write(token.token()).insert(Handle::SchemeRoot);
@@ -298,6 +329,7 @@ impl crate::scheme::KernelScheme for IrqScheme {
InternalFlags::POSITIONED,
)
} else if let Some(path_str) = path_str.strip_prefix('/') {
<<<<<<< HEAD
let (irq_str, affinity) = path_str
.trim_end_matches('/')
.rsplit_once('/')
@@ -313,6 +345,9 @@ impl crate::scheme::KernelScheme for IrqScheme {
} else {
Self::open_ext_irq(flags, LogicalCpuId::new(cpu_id.into()), path_str)?
}
=======
Self::open_ext_irq(flags, LogicalCpuId::new(cpu_id.into()), path_str)?
>>>>>>> master
} else {
return Err(Error::new(ENOENT));
}
@@ -339,6 +374,7 @@ impl crate::scheme::KernelScheme for IrqScheme {
}
#[cfg(not(dtb))]
panic!("")
<<<<<<< HEAD
} else if let Some(rest) = path_str.strip_suffix("/affinity") {
let irq_number = u8::from_str(rest).or(Err(Error::new(ENOENT)))?;
if irq_number >= TOTAL_IRQ_COUNT {
@@ -346,13 +382,18 @@ impl crate::scheme::KernelScheme for IrqScheme {
}
(Handle::IrqAffinity { irq: irq_number, mask: AtomicUsize::new(0) },
InternalFlags::empty())
=======
>>>>>>> master
} else if let Ok(plain_irq_number) = u8::from_str(path_str) {
if plain_irq_number < BASE_IRQ_COUNT {
(
Handle::Irq {
ack: AtomicUsize::new(0),
irq: plain_irq_number,
<<<<<<< HEAD
cpu_id: LogicalCpuId::BSP,
=======
>>>>>>> master
},
InternalFlags::empty(),
)
@@ -408,7 +449,10 @@ impl crate::scheme::KernelScheme for IrqScheme {
}
}
Handle::Avail(cpu_id) => {
<<<<<<< HEAD
let mut listed = 0;
=======
>>>>>>> master
for vector in available_irqs_iter(cpu_id).skip(opaque) {
let irq = vector_to_irq(vector);
if cpu_id == LogicalCpuId::BSP && irq < BASE_IRQ_COUNT {
@@ -422,9 +466,13 @@ impl crate::scheme::KernelScheme for IrqScheme {
name: &intermediate,
next_opaque_id: u64::from(vector) + 1,
})?;
<<<<<<< HEAD
listed += 1;
}
info!("irq getdents Avail: cpu_id={} opaque={} listed={}", cpu_id.get(), opaque, listed);
=======
}
>>>>>>> master
}
_ => return Err(Error::new(ENOTDIR)),
}
@@ -459,6 +507,7 @@ impl crate::scheme::KernelScheme for IrqScheme {
let handle = handles_guard.get(id)?;
if let &Handle::Irq {
<<<<<<< HEAD
irq: handle_irq,
cpu_id: handle_cpu_id,
..
@@ -467,6 +516,13 @@ impl crate::scheme::KernelScheme for IrqScheme {
{
info!("irq close: unreserving vector {} on cpu_id={}", irq_to_vector(handle_irq), handle_cpu_id.get());
set_reserved(handle_cpu_id, irq_to_vector(handle_irq), false);
=======
irq: handle_irq, ..
} = handle
&& handle_irq > BASE_IRQ_COUNT
{
set_reserved(LogicalCpuId::BSP, irq_to_vector(handle_irq), false);
>>>>>>> master
}
Ok(())
}
@@ -482,6 +538,7 @@ impl crate::scheme::KernelScheme for IrqScheme {
let handle = handles_guard.get(file)?;
match handle {
<<<<<<< HEAD
&Handle::IrqAffinity { irq: _handle_irq, ref mask } => {
if buffer.len() < size_of::<u32>() {
return Err(Error::new(EINVAL));
@@ -508,6 +565,11 @@ impl crate::scheme::KernelScheme for IrqScheme {
irq: handle_irq,
ack: ref handle_ack,
cpu_id: _,
=======
&Handle::Irq {
irq: handle_irq,
ack: ref handle_ack,
>>>>>>> master
} => {
if buffer.len() < size_of::<usize>() {
return Err(Error::new(EINVAL));
@@ -544,6 +606,7 @@ impl crate::scheme::KernelScheme for IrqScheme {
st_nlink: 1,
..Default::default()
},
<<<<<<< HEAD
Handle::IrqAffinity { irq, .. } => Stat {
st_mode: MODE_CHR | 0o200,
st_size: size_of::<u32>() as u64,
@@ -553,6 +616,8 @@ impl crate::scheme::KernelScheme for IrqScheme {
st_nlink: 1,
..Default::default()
},
=======
>>>>>>> master
Handle::Bsp => Stat {
st_mode: MODE_CHR | 0o400,
st_size: size_of::<usize>() as u64,
@@ -594,9 +659,14 @@ impl crate::scheme::KernelScheme for IrqScheme {
let scheme_path = match handle {
Handle::Irq { irq, .. } => format!("irq:{}", irq),
<<<<<<< HEAD
Handle::IrqAffinity { irq, .. } => format!("irq:{}/affinity", irq),
Handle::Bsp => "irq:bsp".to_owned(),
Handle::Avail(cpu_id) => format!("irq:cpu-{:02x}", cpu_id.get()),
=======
Handle::Bsp => "irq:bsp".to_owned(),
Handle::Avail(cpu_id) => format!("irq:cpu-{:2x}", cpu_id.get()),
>>>>>>> master
Handle::Phandle(phandle, _) => format!("irq:phandle-{}", phandle),
Handle::TopLevel => "irq:".to_owned(),
_ => return Err(Error::new(EBADF)),
@@ -622,7 +692,10 @@ impl crate::scheme::KernelScheme for IrqScheme {
Handle::Irq {
irq: handle_irq,
ack: ref handle_ack,
<<<<<<< HEAD
cpu_id: _,
=======
>>>>>>> master
} => {
if buffer.len() < size_of::<usize>() {
return Err(Error::new(EINVAL));
@@ -642,7 +715,11 @@ impl crate::scheme::KernelScheme for IrqScheme {
buffer.write_u32(LogicalCpuId::BSP.get())?;
Ok(size_of::<usize>())
}
<<<<<<< HEAD
Handle::Avail(_) | Handle::TopLevel | Handle::Phandle(_, _) | Handle::SchemeRoot | Handle::IrqAffinity { .. } => {
=======
Handle::Avail(_) | Handle::TopLevel | Handle::Phandle(_, _) | Handle::SchemeRoot => {
>>>>>>> master
Err(Error::new(EISDIR))
}
}
@@ -14,7 +14,11 @@ use alloc::{
};
use core::{
str,
<<<<<<< HEAD
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
=======
sync::atomic::{AtomicUsize, Ordering},
>>>>>>> master
};
use hashbrown::hash_map::{self, DefaultHashBuilder, HashMap};
use spin::Once;
@@ -169,7 +173,10 @@ enum Handle {
/// Schemes list
static HANDLES: Once<RwLock<L1, HashMap<SchemeId, Handle>>> = Once::new();
<<<<<<< HEAD
static SCHEME_GENERATIONS: Once<RwLock<L1, HashMap<SchemeId, AtomicU64>>> = Once::new();
=======
>>>>>>> master
static SCHEME_LIST_NEXT_ID: AtomicUsize = AtomicUsize::new(MAX_GLOBAL_SCHEMES);
static SCHEME_LIST_ID: AtomicUsize = AtomicUsize::new(0);
@@ -205,10 +212,13 @@ fn init_schemes() -> RwLock<L1, HashMap<SchemeId, Handle>> {
RwLock::new(handles)
}
<<<<<<< HEAD
fn init_scheme_generations() -> RwLock<L1, HashMap<SchemeId, AtomicU64>> {
RwLock::new(HashMap::new())
}
=======
>>>>>>> master
/// Get a handle to a scheme.
pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result<KernelSchemes> {
match handles().read(token).get(&scheme_id) {
@@ -217,6 +227,7 @@ pub fn get_scheme(token: LockToken<'_, L0>, scheme_id: SchemeId) -> Result<Kerne
}
}
<<<<<<< HEAD
pub fn current_scheme_generation(token: LockToken<'_, L0>, scheme_id: SchemeId) -> u64 {
scheme_generations()
.read(token)
@@ -225,10 +236,13 @@ pub fn current_scheme_generation(token: LockToken<'_, L0>, scheme_id: SchemeId)
.unwrap_or(0)
}
=======
>>>>>>> master
fn handles<'a>() -> &'a RwLock<L1, HashMap<SchemeId, Handle>> {
HANDLES.call_once(init_schemes)
}
<<<<<<< HEAD
fn scheme_generations<'a>() -> &'a RwLock<L1, HashMap<SchemeId, AtomicU64>> {
SCHEME_GENERATIONS.call_once(init_scheme_generations)
}
@@ -244,6 +258,8 @@ fn increment_scheme_generation(scheme_id: SchemeId, token: &mut CleanLockToken)
}
}
=======
>>>>>>> master
/// Scheme list type
pub struct SchemeList;
@@ -288,6 +304,7 @@ impl SchemeList {
/// Remove a scheme
fn remove(&self, id: usize, token: &mut CleanLockToken) {
<<<<<<< HEAD
let scheme_id = SchemeId(id);
let scheme = handles().write(token.token()).remove(&scheme_id);
@@ -296,6 +313,11 @@ impl SchemeList {
user.inner.fail_pending_calls(token);
}
increment_scheme_generation(scheme_id, token);
=======
let scheme = handles().write(token.token()).remove(&SchemeId(id));
assert!(scheme.is_some());
>>>>>>> master
if let Some(Handle::Scheme(KernelSchemes::User(user))) = scheme
&& let Some(user) = Arc::into_inner(user.inner)
{
@@ -320,6 +342,7 @@ impl KernelScheme for SchemeList {
token: &mut CleanLockToken,
) -> Result<OpenResult> {
let scheme_id = SchemeId(scheme_id);
<<<<<<< HEAD
let maybe_inner = {
let handles = handles().read(token.token());
match handles.get(&scheme_id).ok_or(Error::new(EBADF))? {
@@ -346,6 +369,34 @@ impl KernelScheme for SchemeList {
))));
}
=======
match handles()
.read(token.token())
.get(&scheme_id)
.ok_or(Error::new(EBADF))?
{
Handle::Scheme(KernelSchemes::User(UserScheme { inner })) => {
let inner = inner.clone();
assert!(scheme_id == inner.scheme_id);
let scheme = scheme_id;
let params = unsafe { user_buf.read_exact::<NewFdParams>()? };
return Ok(OpenResult::External(Arc::new(RwLock::new(
FileDescription {
scheme,
number: params.number,
offset: params.offset,
flags: params.flags as u32,
internal_flags: InternalFlags::from_extra0(params.internal_flags)
.ok_or(Error::new(EINVAL))?,
},
))));
}
Handle::SchemeCreationCapability => (),
_ => return Err(Error::new(EBADF)),
};
>>>>>>> master
const EXPECTED: &[u8] = b"create-scheme";
let mut buf = [0u8; EXPECTED.len()];
@@ -810,7 +861,10 @@ pub struct CallerCtx {
pub pid: usize,
pub uid: u32,
pub gid: u32,
<<<<<<< HEAD
pub groups: alloc::vec::Vec<u32>,
=======
>>>>>>> master
}
impl CallerCtx {
pub fn filter_uid_gid(self, euid: u32, egid: u32) -> Self {
@@ -819,7 +873,10 @@ impl CallerCtx {
pid: self.pid,
uid: euid,
gid: egid,
<<<<<<< HEAD
groups: self.groups,
=======
>>>>>>> master
}
} else {
self
@@ -1,3 +1,4 @@
<<<<<<< HEAD
use alloc::{
collections::VecDeque,
string::{String, ToString},
@@ -5,6 +6,10 @@ use alloc::{
vec::Vec,
};
use core::sync::atomic::{AtomicUsize, Ordering};
=======
use alloc::{collections::VecDeque, sync::Arc, vec::Vec};
use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
>>>>>>> master
use syscall::{data::GlobalSchemes, CallFlags};
@@ -19,6 +24,7 @@ use crate::{
sync::{CleanLockToken, Mutex, RwLock, WaitCondition, L1},
syscall::{
data::Stat,
<<<<<<< HEAD
error::{
Error, Result, EAGAIN, EBADF, EEXIST, EINVAL, EINTR, ENOENT, ENOTDIR, EPIPE,
},
@@ -26,12 +32,17 @@ use crate::{
EventFlags, EVENT_READ, EVENT_WRITE, MODE_FIFO, O_ACCMODE, O_DIRECTORY,
O_NONBLOCK, O_RDONLY, O_RDWR, O_STAT, O_WRONLY,
},
=======
error::{Error, Result, EAGAIN, EBADF, EINTR, EINVAL, ENOENT, EPIPE},
flag::{EventFlags, EVENT_READ, EVENT_WRITE, MODE_FIFO, O_NONBLOCK},
>>>>>>> master
usercopy::{UserSliceRo, UserSliceRw, UserSliceWo},
},
};
use super::{CallerCtx, KernelScheme, OpenResult, SchemeExt, StrOrBytes};
<<<<<<< HEAD
static PIPE_NEXT_ID: AtomicUsize = AtomicUsize::new(1);
#[derive(Clone)]
@@ -73,10 +84,24 @@ struct NamedPipe {
static HANDLES: RwLock<L1, HashMap<usize, Handle>> =
RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new()));
static NAMED_PIPES: RwLock<L1, HashMap<String, Arc<NamedPipe>>> =
=======
// TODO: Preallocate a number of scheme IDs, since there can only be *one* root namespace, and
// therefore only *one* pipe scheme.
static PIPE_NEXT_ID: AtomicUsize = AtomicUsize::new(0);
enum Handle {
Pipe(Arc<Pipe>),
SchemeRoot,
}
// TODO: SLOB?
static PIPES: RwLock<L1, HashMap<usize, Handle>> =
>>>>>>> master
RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new()));
const MAX_QUEUE_SIZE: usize = 65536;
<<<<<<< HEAD
fn next_id() -> usize {
PIPE_NEXT_ID.fetch_add(1, Ordering::Relaxed)
}
@@ -229,11 +254,40 @@ pub fn open_named_pipe(path: &str, flags: usize, token: &mut CleanLockToken) ->
pub fn unlink_named_pipe(path: &str, token: &mut CleanLockToken) -> bool {
NAMED_PIPES.write(token.token()).remove(path).is_some()
=======
// In almost all places where Rust (and LLVM) uses pointers, they are limited to nonnegative isize,
// so this is fine.
const WRITE_NOT_READ_BIT: usize = 1;
fn from_raw_id(id: usize) -> (bool, usize) {
(id & WRITE_NOT_READ_BIT != 0, id & !WRITE_NOT_READ_BIT)
}
pub fn pipe(token: &mut CleanLockToken) -> Result<(usize, usize)> {
// Bit 0 is used for WRITE_NOT_READ_BIT
let id = PIPE_NEXT_ID.fetch_add(2, Ordering::Relaxed);
PIPES.write(token.token()).insert(
id,
Handle::Pipe(Arc::new(Pipe {
queue: Mutex::new(VecDeque::new()),
read_condition: WaitCondition::new(),
write_condition: WaitCondition::new(),
writer_is_alive: AtomicBool::new(true),
reader_is_alive: AtomicBool::new(true),
has_run_dup: AtomicBool::new(false),
fd_queue: Mutex::new(VecDeque::new()),
})),
);
Ok((id, id | WRITE_NOT_READ_BIT))
>>>>>>> master
}
pub struct PipeScheme;
impl PipeScheme {
<<<<<<< HEAD
fn get_endpoint(id: usize, token: &mut CleanLockToken) -> Result<EndpointHandle> {
HANDLES
.read(token.token())
@@ -241,6 +295,15 @@ impl PipeScheme {
.and_then(|handle| match handle {
Handle::Endpoint(endpoint) => Some(endpoint.clone()),
Handle::SchemeRoot => None,
=======
fn get_pipe(key: usize, token: &mut CleanLockToken) -> Result<Arc<Pipe>> {
PIPES
.read(token.token())
.get(&key)
.and_then(|handle| match handle {
Handle::Pipe(pipe) => Some(Arc::clone(pipe)),
_ => None,
>>>>>>> master
})
.ok_or(Error::new(EBADF))
}
@@ -248,17 +311,25 @@ impl PipeScheme {
impl KernelScheme for PipeScheme {
fn scheme_root(&self, token: &mut CleanLockToken) -> Result<usize> {
<<<<<<< HEAD
let id = next_id();
HANDLES.write(token.token()).insert(id, Handle::SchemeRoot);
Ok(id)
}
=======
let id = PIPE_NEXT_ID.fetch_add(2, Ordering::Relaxed);
PIPES.write(token.token()).insert(id, Handle::SchemeRoot);
Ok(id)
}
>>>>>>> master
fn fevent(
&self,
id: usize,
flags: EventFlags,
token: &mut CleanLockToken,
) -> Result<EventFlags> {
<<<<<<< HEAD
let endpoint = Self::get_endpoint(id, token)?;
let mut ready = EventFlags::empty();
@@ -275,6 +346,24 @@ impl KernelScheme for PipeScheme {
&& flags.contains(EVENT_READ)
&& (!endpoint.pipe.queue.lock(token.token()).is_empty()
|| endpoint.pipe.writer_count.load(Ordering::Acquire) == 0)
=======
let (is_writer_not_reader, key) = from_raw_id(id);
let pipe = Self::get_pipe(key, token)?;
let mut ready = EventFlags::empty();
if is_writer_not_reader
&& flags.contains(EVENT_WRITE)
&& (pipe.queue.lock(token.token()).len() <= MAX_QUEUE_SIZE
|| !pipe.reader_is_alive.load(Ordering::Acquire))
{
ready |= EventFlags::EVENT_WRITE;
}
if !is_writer_not_reader
&& flags.contains(EVENT_READ)
&& (!pipe.queue.lock(token.token()).is_empty()
|| !pipe.writer_is_alive.load(Ordering::Acquire))
>>>>>>> master
{
ready |= EventFlags::EVENT_READ;
}
@@ -283,6 +372,7 @@ impl KernelScheme for PipeScheme {
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
<<<<<<< HEAD
let handle = HANDLES
.write(token.token())
.remove(&id)
@@ -325,6 +415,48 @@ impl KernelScheme for PipeScheme {
}
drop_wait_conditions_if_possible(endpoint.pipe, token);
=======
let (is_write_not_read, key) = from_raw_id(id);
let pipe = Self::get_pipe(key, token)?;
let scheme_id = GlobalSchemes::Pipe.scheme_id();
let can_remove = if is_write_not_read {
pipe.writer_is_alive.store(false, Ordering::SeqCst);
event::trigger(scheme_id, key, EVENT_READ, token);
pipe.read_condition.notify(token);
!pipe.reader_is_alive.load(Ordering::SeqCst)
} else {
pipe.reader_is_alive.store(false, Ordering::SeqCst);
event::trigger(scheme_id, key | WRITE_NOT_READ_BIT, EVENT_WRITE, token);
pipe.write_condition.notify(token);
!pipe.writer_is_alive.load(Ordering::SeqCst)
};
if can_remove {
let handle = PIPES.write(token.token()).remove(&key);
if let Some(Handle::Pipe(pipe)) = handle
&& let Some(pipe) = Arc::into_inner(pipe)
{
{
pipe.read_condition.into_drop(token);
}
{
pipe.write_condition.into_drop(token);
}
}
}
if let Some(pipe) = Arc::into_inner(pipe) {
{
pipe.read_condition.into_drop(token);
}
{
pipe.write_condition.into_drop(token);
}
>>>>>>> master
}
Ok(())
@@ -337,9 +469,15 @@ impl KernelScheme for PipeScheme {
_ctx: CallerCtx,
token: &mut CleanLockToken,
) -> Result<OpenResult> {
<<<<<<< HEAD
let endpoint = Self::get_endpoint(old_id, token)?;
if !endpoint.kind.can_read() {
=======
let (is_writer_not_reader, key) = from_raw_id(old_id);
if is_writer_not_reader {
>>>>>>> master
return Err(Error::new(EBADF));
}
@@ -349,6 +487,7 @@ impl KernelScheme for PipeScheme {
return Err(Error::new(EINVAL));
}
<<<<<<< HEAD
Ok(OpenResult::SchemeLocal(
open_endpoint(
Arc::clone(&endpoint.pipe),
@@ -360,6 +499,19 @@ impl KernelScheme for PipeScheme {
))
}
=======
let pipe = Self::get_pipe(key, token)?;
if pipe.has_run_dup.swap(true, Ordering::SeqCst) {
return Err(Error::new(EBADF));
}
Ok(OpenResult::SchemeLocal(
key | WRITE_NOT_READ_BIT,
InternalFlags::empty(),
))
}
>>>>>>> master
fn kopenat(
&self,
id: usize,
@@ -369,6 +521,7 @@ impl KernelScheme for PipeScheme {
_ctx: CallerCtx,
token: &mut CleanLockToken,
) -> Result<OpenResult> {
<<<<<<< HEAD
let is_scheme_root = {
let handles = HANDLES.read(token.token());
match handles.get(&id) {
@@ -410,6 +563,42 @@ impl KernelScheme for PipeScheme {
),
InternalFlags::empty(),
))
=======
let (_, key) = from_raw_id(id);
{
let guard = PIPES.read(token.token());
if let Some(Handle::SchemeRoot) = guard.get(&key) {
} else if let Some(Handle::Pipe(pipe_arc)) = guard.get(&key) {
let pipe = Arc::clone(pipe_arc);
drop(guard);
if user_buf.as_bytes() == b"write" {
return Err(Error::new(EINVAL));
}
if pipe.has_run_dup.swap(true, Ordering::SeqCst) {
return Err(Error::new(EBADF));
}
return Ok(OpenResult::SchemeLocal(
key | WRITE_NOT_READ_BIT,
InternalFlags::empty(),
));
} else {
return Err(Error::new(EBADF));
}
}
let path = user_buf.as_str().or(Err(Error::new(EINVAL)))?;
if !path.trim_start_matches('/').is_empty() {
return Err(Error::new(ENOENT));
}
let (read_id, _) = pipe(token)?;
Ok(OpenResult::SchemeLocal(read_id, InternalFlags::empty()))
>>>>>>> master
}
fn kread(
@@ -420,6 +609,7 @@ impl KernelScheme for PipeScheme {
_stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
<<<<<<< HEAD
let endpoint = Self::get_endpoint(id, token)?;
if !endpoint.kind.can_read() {
@@ -429,6 +619,18 @@ impl KernelScheme for PipeScheme {
loop {
let vec = endpoint.pipe.queue.lock(token.token());
let (mut vec, mut lock_token) = vec.into_split();
=======
let (is_write_not_read, key) = from_raw_id(id);
if is_write_not_read {
return Err(Error::new(EBADF));
}
let pipe = Self::get_pipe(key, token)?;
loop {
let vec = pipe.queue.lock(token.token());
let (mut vec, mut token) = vec.into_split();
>>>>>>> master
let (s1, s2) = vec.as_slices();
let s1_count = core::cmp::min(user_buf.len(), s1.len());
@@ -448,6 +650,7 @@ impl KernelScheme for PipeScheme {
let _ = vec.drain(..bytes_read);
if bytes_read > 0 {
<<<<<<< HEAD
drop(vec);
drop(lock_token);
trigger_matching(&endpoint.pipe, false, true, EVENT_WRITE, token);
@@ -471,11 +674,34 @@ impl KernelScheme for PipeScheme {
.read_condition
.wait(vec, "PipeRead::read", &mut lock_token)
{
=======
event::trigger_locked(
GlobalSchemes::Pipe.scheme_id(),
key | WRITE_NOT_READ_BIT,
EVENT_WRITE,
token.token(),
);
pipe.write_condition.notify_locked(token.token());
return Ok(bytes_read);
} else if user_buf.is_empty() {
return Ok(0);
}
if !pipe.writer_is_alive.load(Ordering::SeqCst) {
return Ok(0);
} else if fcntl_flags & O_NONBLOCK as u32 != 0 {
return Err(Error::new(EAGAIN));
} else if !pipe.read_condition.wait(vec, "PipeRead::read", &mut token) {
>>>>>>> master
return Err(Error::new(EINTR));
}
}
}
<<<<<<< HEAD
=======
>>>>>>> master
fn kwrite(
&self,
id: usize,
@@ -484,6 +710,7 @@ impl KernelScheme for PipeScheme {
_stored_flags: u32,
token: &mut CleanLockToken,
) -> Result<usize> {
<<<<<<< HEAD
let endpoint = Self::get_endpoint(id, token)?;
if !endpoint.kind.can_write() {
@@ -495,6 +722,20 @@ impl KernelScheme for PipeScheme {
let (mut vec, mut lock_token) = vec.into_split();
if endpoint.pipe.reader_count.load(Ordering::Relaxed) == 0 {
=======
let (is_write_not_read, key) = from_raw_id(id);
if !is_write_not_read {
return Err(Error::new(EBADF));
}
let pipe = Self::get_pipe(key, token)?;
loop {
let vec = pipe.queue.lock(token.token());
let (mut vec, mut token) = vec.into_split();
if !pipe.reader_is_alive.load(Ordering::Relaxed) {
>>>>>>> master
return Err(Error::new(EPIPE));
}
@@ -509,6 +750,10 @@ impl KernelScheme for PipeScheme {
let mut bytes_written = 0;
<<<<<<< HEAD
=======
// TODO: Modify VecDeque so that the unwritten portions can be accessed directly?
>>>>>>> master
for (idx, chunk) in src_buf.in_variable_chunks(TMPBUF_SIZE).enumerate() {
let chunk_byte_count = match chunk.copy_common_bytes_to_slice(&mut tmp_buf) {
Ok(c) => c,
@@ -520,6 +765,7 @@ impl KernelScheme for PipeScheme {
}
if bytes_written > 0 {
<<<<<<< HEAD
drop(vec);
drop(lock_token);
trigger_matching(&endpoint.pipe, true, false, EVENT_READ, token);
@@ -529,21 +775,40 @@ impl KernelScheme for PipeScheme {
}
if user_buf.is_empty() {
=======
event::trigger_locked(
GlobalSchemes::Pipe.scheme_id(),
key,
EVENT_READ,
token.token(),
);
pipe.read_condition.notify_locked(token.token());
return Ok(bytes_written);
} else if user_buf.is_empty() {
>>>>>>> master
return Ok(0);
}
if fcntl_flags & O_NONBLOCK as u32 != 0 {
return Err(Error::new(EAGAIN));
<<<<<<< HEAD
}
if !endpoint
.pipe
.write_condition
.wait(vec, "PipeWrite::write", &mut lock_token)
=======
} else if !pipe
.write_condition
.wait(vec, "PipeWrite::write", &mut token)
>>>>>>> master
{
return Err(Error::new(EINTR));
}
}
}
<<<<<<< HEAD
fn kfpath(&self, id: usize, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<usize> {
let endpoint = Self::get_endpoint(id, token)?;
@@ -560,12 +825,24 @@ impl KernelScheme for PipeScheme {
buf.copy_exactly(&Stat {
st_mode: MODE_FIFO | mode,
=======
fn kfpath(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<usize> {
//TODO: construct useful path?
buf.copy_common_bytes_from_slice("/scheme/pipe/".as_bytes())
}
fn kfstat(&self, _id: usize, buf: UserSliceWo, _token: &mut CleanLockToken) -> Result<()> {
buf.copy_exactly(&Stat {
st_mode: MODE_FIFO | 0o666,
>>>>>>> master
..Default::default()
})?;
Ok(())
}
<<<<<<< HEAD
=======
>>>>>>> master
fn kfdwrite(
&self,
id: usize,
@@ -575,6 +852,7 @@ impl KernelScheme for PipeScheme {
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
<<<<<<< HEAD
let endpoint = Self::get_endpoint(id, token)?;
if !endpoint.kind.can_write() {
@@ -586,6 +864,25 @@ impl KernelScheme for PipeScheme {
let (mut vec, mut lock_token) = vec.into_split();
if endpoint.pipe.reader_count.load(Ordering::Relaxed) == 0 {
=======
let (is_write_not_read, key) = from_raw_id(id);
if !is_write_not_read {
return Err(Error::new(EBADF));
}
let pipe = match Self::get_pipe(key, token) {
Ok(p) => p,
Err(e) => {
return Err(e);
}
};
loop {
let vec = pipe.fd_queue.lock(token.token());
let (mut vec, mut token) = vec.into_split();
if !pipe.reader_is_alive.load(Ordering::Relaxed) {
>>>>>>> master
return Err(Error::new(EPIPE));
}
if descs.is_empty() {
@@ -605,24 +902,43 @@ impl KernelScheme for PipeScheme {
let fds_written = vec.len() - before_len;
if fds_written > 0 {
<<<<<<< HEAD
drop(vec);
drop(lock_token);
trigger_matching(&endpoint.pipe, true, false, EVENT_READ, token);
endpoint.pipe.read_condition.notify(token);
=======
event::trigger_locked(
GlobalSchemes::Pipe.scheme_id(),
key,
EVENT_READ,
token.token(),
);
pipe.read_condition.notify_locked(token.token());
>>>>>>> master
return Ok(fds_written);
}
<<<<<<< HEAD
if !endpoint
.pipe
.write_condition
.wait(vec, "PipeWrite::write", &mut lock_token)
=======
if !pipe
.write_condition
.wait(vec, "PipeWrite::write", &mut token)
>>>>>>> master
{
return Err(Error::new(EINTR));
}
}
}
<<<<<<< HEAD
=======
>>>>>>> master
fn kfdread(
&self,
id: usize,
@@ -631,19 +947,38 @@ impl KernelScheme for PipeScheme {
_metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
<<<<<<< HEAD
let endpoint = Self::get_endpoint(id, token)?;
if !endpoint.kind.can_read() {
return Err(Error::new(EBADF));
}
=======
let (is_write_not_read, key) = from_raw_id(id);
if is_write_not_read {
return Err(Error::new(EBADF));
}
let pipe = match Self::get_pipe(key, token) {
Ok(p) => p,
Err(e) => {
return Err(e);
}
};
>>>>>>> master
if payload.is_empty() {
return Ok(0);
}
loop {
<<<<<<< HEAD
let vec = endpoint.pipe.fd_queue.lock(token.token());
let (mut vec, mut lock_token) = vec.into_split();
=======
let vec = pipe.fd_queue.lock(token.token());
let (mut vec, mut token) = vec.into_split();
>>>>>>> master
let fds_available = vec.len();
let max_fds_read = payload.len() / size_of::<usize>();
@@ -656,13 +991,18 @@ impl KernelScheme for PipeScheme {
fds_to_transfer,
payload,
flags.contains(CallFlags::FD_CLOEXEC),
<<<<<<< HEAD
&mut lock_token,
=======
&mut token,
>>>>>>> master
)?;
} else {
bulk_add_fds(
fds_to_transfer,
payload,
flags.contains(CallFlags::FD_CLOEXEC),
<<<<<<< HEAD
&mut lock_token,
)?;
}
@@ -671,10 +1011,24 @@ impl KernelScheme for PipeScheme {
drop(lock_token);
trigger_matching(&endpoint.pipe, false, true, EVENT_WRITE, token);
endpoint.pipe.write_condition.notify(token);
=======
&mut token,
)?;
}
event::trigger_locked(
GlobalSchemes::Pipe.scheme_id(),
key | WRITE_NOT_READ_BIT,
EVENT_WRITE,
token.token(),
);
pipe.write_condition.notify_locked(token.token());
>>>>>>> master
return Ok(fds_to_read);
}
<<<<<<< HEAD
if endpoint.pipe.writer_count.load(Ordering::SeqCst) == 0 {
return Ok(0);
}
@@ -683,6 +1037,11 @@ impl KernelScheme for PipeScheme {
.read_condition
.wait(vec, "PipeRead::read", &mut lock_token)
{
=======
if !pipe.writer_is_alive.load(Ordering::SeqCst) {
return Ok(0);
} else if !pipe.read_condition.wait(vec, "PipeRead::read", &mut token) {
>>>>>>> master
return Err(Error::new(EINTR));
}
}
@@ -690,6 +1049,7 @@ impl KernelScheme for PipeScheme {
}
pub struct Pipe {
<<<<<<< HEAD
read_condition: WaitCondition,
write_condition: WaitCondition,
queue: Mutex<L1, VecDeque<u8>>,
@@ -710,3 +1070,13 @@ impl Pipe {
}
}
}
=======
read_condition: WaitCondition, // signals whether there are available bytes to read
write_condition: WaitCondition, // signals whether there is room for additional bytes
queue: Mutex<L1, VecDeque<u8>>,
reader_is_alive: AtomicBool, // starts set, unset when reader closes
writer_is_alive: AtomicBool, // starts set, unset when writer closes
has_run_dup: AtomicBool,
fd_queue: Mutex<L1, VecDeque<Arc<LockedFileDescription>>>,
}
>>>>>>> master
@@ -105,7 +105,10 @@ enum ContextHandle {
// Attr handles, to set ens/euid/egid/pid.
Authority,
Attr,
<<<<<<< HEAD
Groups,
=======
>>>>>>> master
Status {
privileged: bool,
@@ -262,7 +265,10 @@ impl ProcScheme {
let handle = match actual_name {
"attrs" => ContextHandle::Attr,
"status" => ContextHandle::Status { privileged: true },
<<<<<<< HEAD
"groups" => ContextHandle::Groups,
=======
>>>>>>> master
_ => return Err(Error::new(ENOENT)),
};
@@ -308,11 +314,14 @@ impl ProcScheme {
let id = NonZeroUsize::new(NEXT_ID.fetch_add(1, Ordering::Relaxed))
.ok_or(Error::new(EMFILE))?;
let context = context::spawn(true, Some(id), ret, token)?;
<<<<<<< HEAD
{
let parent_groups =
context::current().read(token.token()).groups.clone();
context.write(token.token()).groups = parent_groups;
}
=======
>>>>>>> master
HANDLES.write(token.token()).insert(
id.get(),
Handle {
@@ -432,7 +441,10 @@ impl KernelScheme for ProcScheme {
}
fn close(&self, id: usize, token: &mut CleanLockToken) -> Result<()> {
<<<<<<< HEAD
let mut inner_token = unsafe { CleanLockToken::new() };
=======
>>>>>>> master
let handle = HANDLES
.write(token.token())
.remove(&id)
@@ -460,7 +472,13 @@ impl KernelScheme for ProcScheme {
))]
regs.set_arg1(arg1);
<<<<<<< HEAD
Ok(context.set_addr_space(Some(new), inner_token.downgrade()))
=======
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
Ok(context.set_addr_space(Some(new), token.downgrade()))
>>>>>>> master
})?;
if let Some(old_ctx) = old_ctx
&& let Some(addrspace) = Arc::into_inner(old_ctx)
@@ -499,7 +517,10 @@ impl KernelScheme for ProcScheme {
consume: bool,
token: &mut CleanLockToken,
) -> Result<usize> {
<<<<<<< HEAD
let mut inner_token = unsafe { CleanLockToken::new() };
=======
>>>>>>> master
let handle = HANDLES
.read(token.token())
.get(&id)
@@ -590,7 +611,13 @@ impl KernelScheme for ProcScheme {
};
// TODO: Allocated or AllocatedShared?
let addrsp = AddrSpace::current()?;
<<<<<<< HEAD
let page = addrsp.acquire_write(inner_token.downgrade()).mmap_anywhere(
=======
// TODO: Lock ordering violation
let mut token = unsafe { CleanLockToken::new() };
let page = addrsp.acquire_write(token.downgrade()).mmap_anywhere(
>>>>>>> master
&addrsp,
NonZeroUsize::new(1).unwrap(),
MapFlags::PROT_READ | MapFlags::PROT_WRITE,
@@ -854,6 +881,7 @@ impl KernelScheme for ProcScheme {
}
}
fn extract_scheme_number(fd: usize, token: &mut CleanLockToken) -> Result<(KernelSchemes, usize)> {
<<<<<<< HEAD
let desc = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
@@ -865,6 +893,19 @@ fn extract_scheme_number(fd: usize, token: &mut CleanLockToken) -> Result<(Kerne
};
let scheme = desc.get_scheme(token)?;
let number = desc.number;
=======
let (scheme_id, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let file_descriptor = context
.get_file(FileHandle::from(fd), &mut token)
.ok_or(Error::new(EBADF))?;
let desc = file_descriptor.description.read(token.token());
(desc.scheme, desc.number)
};
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
>>>>>>> master
Ok((scheme, number))
}
@@ -1276,6 +1317,7 @@ impl ContextHandle {
guard.prio = (info.prio as usize).min(39);
Ok(size_of::<ProcSchemeAttrs>())
}
<<<<<<< HEAD
Self::Groups => {
const NGROUPS_MAX: usize = 65536;
if buf.len() % size_of::<u32>() != 0 {
@@ -1309,6 +1351,8 @@ impl ContextHandle {
}
Ok(count * size_of::<u32>())
}
=======
>>>>>>> master
ContextHandle::OpenViaDup => {
let mut args = buf.usizes();
@@ -1513,6 +1557,7 @@ impl ContextHandle {
debug_name,
})
}
<<<<<<< HEAD
Self::Groups => {
let c = &context.read(token.token());
let max = buf.len() / size_of::<u32>();
@@ -1522,6 +1567,8 @@ impl ContextHandle {
}
Ok(count * size_of::<u32>())
}
=======
>>>>>>> master
ContextHandle::Sighandler => {
let data = match context.read(token.token()).sig {
Some(ref sig) => SetSighandlerData {
@@ -45,11 +45,14 @@ enum Handle {
data: Arc<RwLock<L1, Option<Vec<u8>>>>,
},
SchemeRoot,
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Msr {
cpu: usize,
msr: u32,
},
=======
>>>>>>> master
}
#[derive(Clone, Copy)]
@@ -138,6 +141,7 @@ impl KernelScheme for SysScheme {
let id = HANDLES.write(token.token()).insert(Handle::TopLevel);
Ok(OpenResult::SchemeLocal(id, InternalFlags::POSITIONED))
<<<<<<< HEAD
} else if path.starts_with("msr/") {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
{
@@ -160,6 +164,8 @@ impl KernelScheme for SysScheme {
{
Err(Error::new(ENOENT))
}
=======
>>>>>>> master
} else {
//Have to iterate to get the path without allocation
let entry = FILES
@@ -187,8 +193,11 @@ impl KernelScheme for SysScheme {
Handle::TopLevel => return Ok(0),
Handle::Resource { kind, data, .. } => (*kind, data.clone()),
Handle::SchemeRoot => return Err(Error::new(EBADF)),
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Handle::Msr { .. } => return Ok(0),
=======
>>>>>>> master
}
};
if matches!(kind, Kind::Wr(_)) {
@@ -217,6 +226,7 @@ impl KernelScheme for SysScheme {
Handle::TopLevel => "",
Handle::Resource { path, .. } => path,
Handle::SchemeRoot => return Err(Error::new(EBADF)),
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Handle::Msr { cpu, msr } => {
const FIRST: &[u8] = b"sys:msr/";
@@ -227,6 +237,8 @@ impl KernelScheme for SysScheme {
}
return Ok(bytes_read);
}
=======
>>>>>>> master
};
const FIRST: &[u8] = b"sys:";
@@ -254,6 +266,7 @@ impl KernelScheme for SysScheme {
let (kind, data_lock) = {
match HANDLES.read(token.token()).get(id)? {
Handle::Resource { kind, data, .. } => (*kind, data.clone()),
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Handle::Msr { cpu, msr } => {
if *cpu != crate::cpu_id().get() as usize {
@@ -263,6 +276,8 @@ impl KernelScheme for SysScheme {
let data = format!("{:016x}\n", val).into_bytes();
return buffer.copy_common_bytes_from_slice(&data[pos..]);
}
=======
>>>>>>> master
_ => return Err(Error::new(EBADF)),
}
};
@@ -301,6 +316,7 @@ impl KernelScheme for SysScheme {
let len = buffer.copy_common_bytes_to_slice(&mut intermediate)?;
(*handler, intermediate, len)
}
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Handle::Msr { cpu, msr } => {
if *cpu != crate::cpu_id().get() as usize {
@@ -313,6 +329,8 @@ impl KernelScheme for SysScheme {
unsafe { x86::msr::wrmsr(*msr, val); }
return Ok(len);
}
=======
>>>>>>> master
Handle::SchemeRoot => return Err(Error::new(EBADF)),
};
handler(&intermediate[..len], token)
@@ -329,8 +347,12 @@ impl KernelScheme for SysScheme {
return Ok(0);
};
match HANDLES.read(token.token()).get(id)? {
<<<<<<< HEAD
Handle::Resource { .. }
| Handle::Msr { .. } => Err(Error::new(ENOTDIR)),
=======
Handle::Resource { .. } => Err(Error::new(ENOTDIR)),
>>>>>>> master
Handle::TopLevel => {
let mut buf = DirentBuf::new(buf, header_size).ok_or(Error::new(EIO))?;
for (this_idx, (name, _)) in FILES.iter().enumerate().skip(first_index) {
@@ -354,6 +376,7 @@ impl KernelScheme for SysScheme {
Handle::Resource { kind, data, .. } => Some((*kind, data.clone())),
Handle::TopLevel => None,
Handle::SchemeRoot => return Err(Error::new(EBADF)),
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
Handle::Msr { .. } => {
let stat = Stat {
@@ -366,6 +389,8 @@ impl KernelScheme for SysScheme {
buf.copy_exactly(&stat)?;
return Ok(());
}
=======
>>>>>>> master
}
};
let stat = if let Some((kind, data_lock)) = stat_base {
@@ -80,7 +80,10 @@ const ONE: NonZeroUsize = match NonZeroUsize::new(1) {
Some(one) => one,
None => unreachable!(),
};
<<<<<<< HEAD
const MAX_SPURIOUS_WAKEUPS: usize = 100;
=======
>>>>>>> master
enum ParsedCqe {
TriggerFevent {
@@ -210,8 +213,11 @@ impl UserInner {
caller_responsible: &mut PageSpan,
token: &mut CleanLockToken,
) -> Result<Response> {
<<<<<<< HEAD
let mut remaining_spurious_wakeups = MAX_SPURIOUS_WAKEUPS;
=======
>>>>>>> master
{
// Disable preemption to avoid context switches between setting the
// process state and sending the scheme request. The process is made
@@ -264,10 +270,14 @@ impl UserInner {
};
let states = self.states.lock(token.token());
<<<<<<< HEAD
let (mut states, mut state_token) = states.into_split();
let mut timed_out_descriptions = None;
let mut remove_state = false;
let mut timed_out = false;
=======
let (mut states, mut token) = states.into_split();
>>>>>>> master
match states.get_mut(sqe.tag as usize) {
// invalid state
None => return Err(Error::new(EBADFD)),
@@ -280,6 +290,7 @@ impl UserInner {
fds,
} => {
let maybe_eintr =
<<<<<<< HEAD
eintr_if_sigkill(&mut callee_responsible, &mut state_token.token());
if maybe_eintr.is_ok() {
@@ -309,6 +320,26 @@ impl UserInner {
.write(state_token.token())
.block("UserInner::call (woken up after cancelation request)");
}
=======
eintr_if_sigkill(&mut callee_responsible, &mut token.token());
*o = State::Waiting {
canceling: true,
callee_responsible,
context,
fds,
};
maybe_eintr?;
context::current()
.write(token.token())
.block("UserInner::call (woken up after cancelation request)");
// We do not want to drop the lock before blocking
// as if we get preempted in between we might miss a
// wakeup.
drop(states);
>>>>>>> master
}
// spurious wakeup
State::Waiting {
@@ -317,6 +348,7 @@ impl UserInner {
context,
mut callee_responsible,
} => {
<<<<<<< HEAD
let current_context = context::current();
let maybe_eintr =
eintr_if_sigkill(&mut callee_responsible, &mut state_token);
@@ -387,6 +419,62 @@ impl UserInner {
.write(token.token())
.block("UserInner::call (spurious wakeup)");
}
=======
let maybe_eintr = eintr_if_sigkill(&mut callee_responsible, &mut token);
let current_context = context::current();
*o = State::Waiting {
// Currently we treat all spurious wakeups to have the same behavior
// as signals (i.e., we send a cancellation request). It is not something
// that should happen, but it certainly can happen, for example if a context
// is awoken through its thread handle without setting any sig bits, or if the
// caller clears its own sig bits. If it actually is a signal, then it is the
// intended behavior.
canceling: true,
fds,
context,
callee_responsible,
};
maybe_eintr?;
// We do not want to preempt between sending the
// cancellation and blocking again where we might
// miss a wakeup.
let mut preempt = PreemptGuardL1::new(&current_context, &mut token);
let token = preempt.token();
self.todo.send_locked(
Sqe {
opcode: Opcode::Cancel as u8,
sqe_flags: SqeFlags::ONEWAY,
tag: sqe.tag,
..Default::default()
},
token.token(),
);
event::trigger_locked(
self.root_id,
self.scheme_id.get(),
EVENT_READ,
token.token(),
);
// 1. If cancellation was requested and arrived
// before the scheme processed the request, an
// acknowledgement will be sent back after the
// cancellation is processed and we will be woken up
// again. State will be State::Responded then.
//
// 2. If cancellation was requested but the scheme
// already processed the request, we will receive
// the actual response next and woken up again.
// State will be State::Responded then.
context::current()
.write(token.token())
.block("UserInner::call (spurious wakeup)");
drop(states);
>>>>>>> master
}
// invalid state
@@ -401,6 +489,7 @@ impl UserInner {
}
},
}
<<<<<<< HEAD
if let Some(descriptions) = timed_out_descriptions {
drop(states);
@@ -412,10 +501,13 @@ impl UserInner {
if timed_out {
return Err(Error::new(ETIMEDOUT));
}
=======
>>>>>>> master
}
}
}
<<<<<<< HEAD
fn collect_descriptions_to_close(
fds: Vec<Arc<LockedFileDescription>>,
) -> Vec<FileDescription> {
@@ -465,6 +557,8 @@ impl UserInner {
}
}
=======
>>>>>>> master
/// Map a readable structure to the scheme's userspace and return the
/// pointer
#[must_use = "copying back to head/tail buffers can fail"]
@@ -1376,7 +1470,10 @@ impl UserInner {
}
pub fn into_drop(self, token: &mut CleanLockToken) {
<<<<<<< HEAD
self.fail_pending_calls(token);
=======
>>>>>>> master
self.todo.condition.into_drop(token);
}
}
@@ -74,16 +74,24 @@ impl MemoryEntry {
}
struct MemoryMap {
<<<<<<< HEAD
entries: [MemoryEntry; 1024],
=======
entries: [MemoryEntry; 512],
>>>>>>> master
size: usize,
}
impl MemoryMap {
fn register(&mut self, base: usize, size: usize, kind: BootloaderMemoryKind) {
if self.size >= self.entries.len() {
<<<<<<< HEAD
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
unsafe { core::arch::asm!("out dx, al", in("dx") 0x3F8u16, in("al") b'!', options(nostack, preserves_flags)); }
panic!("Early memory map overflow at entry {} (max {})", self.size, self.entries.len());
=======
panic!("Early memory map overflow!");
>>>>>>> master
}
let start = if kind == BootloaderMemoryKind::Free {
align_up(base)
@@ -136,7 +144,11 @@ static MEMORY_MAP: SyncUnsafeCell<MemoryMap> = SyncUnsafeCell::new(MemoryMap {
start: 0,
end: 0,
kind: BootloaderMemoryKind::Null,
<<<<<<< HEAD
}; 1024],
=======
}; 512],
>>>>>>> master
size: 0,
});
@@ -325,6 +337,7 @@ unsafe fn map_memory<A: Arch>(areas: &[MemoryArea], mut bump_allocator: &mut Bum
}
}
<<<<<<< HEAD
let kernel_area = match (*MEMORY_MAP.get()).kernel() {
Some(area) => area,
None => {
@@ -335,6 +348,9 @@ unsafe fn map_memory<A: Arch>(areas: &[MemoryArea], mut bump_allocator: &mut Bum
}
}
};
=======
let kernel_area = (*MEMORY_MAP.get()).kernel().unwrap();
>>>>>>> master
let kernel_base = kernel_area.start;
let kernel_size = kernel_area.end.saturating_sub(kernel_area.start);
// Map kernel at KERNEL_OFFSET
@@ -149,6 +149,7 @@ static BOOTSTRAP: spin::Once<Bootstrap> = spin::Once::new();
pub(crate) static AP_READY: AtomicBool = AtomicBool::new(false);
static BSP_READY: AtomicBool = AtomicBool::new(false);
<<<<<<< HEAD
#[cold]
fn halt_boot(message: &str) -> ! {
print!("{message}");
@@ -158,6 +159,8 @@ fn halt_boot(message: &str) -> ! {
}
}
=======
>>>>>>> master
/// This is the kernel entry point for the primary CPU. The arch crate is responsible for calling this
pub(crate) fn kmain(bootstrap: Bootstrap) -> ! {
let mut token = unsafe { CleanLockToken::new() };
@@ -189,7 +192,13 @@ pub(crate) fn kmain(bootstrap: Bootstrap) -> ! {
context.euid = 0;
context.egid = 0;
}
<<<<<<< HEAD
Err(_err) => halt_boot("FATAL: failed to spawn first userspace process userspace_init\n"),
=======
Err(err) => {
panic!("failed to spawn userspace_init: {:?}", err);
}
>>>>>>> master
}
run_userspace(&mut token)
@@ -1,6 +1,9 @@
pub use self::{ordered::*, wait_condition::WaitCondition, wait_queue::WaitQueue};
<<<<<<< HEAD
pub mod mcs;
=======
>>>>>>> master
pub mod ordered;
pub mod wait_condition;
pub mod wait_queue;
@@ -52,9 +52,13 @@
//! *g1 = 12;
//! ```
use alloc::sync::Arc;
<<<<<<< HEAD
use core::cell::UnsafeCell;
use core::marker::PhantomData;
use core::ptr;
=======
use core::marker::PhantomData;
>>>>>>> master
use crate::percpu::PercpuBlock;
@@ -734,6 +738,7 @@ impl<L: Level, T> Drop for ArcRwLockWriteGuard<L, T> {
/// This function can only be called if no lock is held by the calling thread/task
#[inline]
pub fn check_no_locks(_: LockToken<'_, L0>) {}
<<<<<<< HEAD
// ---------------------------------------------------------------------------
// MCS-based fair mutex (McsMutex)
@@ -874,3 +879,5 @@ impl<L: Level, T> Drop for McsRawGuard<'_, L, T> {
self.lock.raw.release(&percpu.mcs_sched_node);
}
}
=======
>>>>>>> master
+102 -180
View File
@@ -2,7 +2,7 @@
use core::num::NonZeroUsize;
use alloc::{format, string::{String, ToString}, sync::Arc, vec::Vec};
use alloc::{string::String, sync::Arc, vec::Vec};
use redox_path::RedoxPath;
use crate::{
@@ -12,9 +12,9 @@ use crate::{
memory::{AddrSpace, GenericFlusher, Grant, PageSpan, TlbShootdownActions},
},
memory::{Page, VirtualAddress, PAGE_SIZE},
scheme::{self, pipe, FileHandle, KernelScheme, OpenResult, SchemeExt, StrOrBytes},
scheme::{self, FileHandle, KernelScheme, OpenResult, StrOrBytes},
sync::{CleanLockToken, RwLock},
syscall::{data::{GlobalSchemes, Stat}, error::*, flag::*},
syscall::{data::Stat, error::*, flag::*},
};
use super::usercopy::{UserSlice, UserSliceRo, UserSliceRw, UserSliceWo};
@@ -45,7 +45,7 @@ pub fn file_op_generic_ext<T>(
(file, desc)
};
let scheme = desc.get_scheme(token)?;
let scheme = scheme::get_scheme(token.token(), desc.scheme)?;
op(&*scheme, file.description, desc, token)
}
@@ -62,32 +62,55 @@ pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result<String>
// TODO: Define elsewhere
const PATH_MAX: usize = PAGE_SIZE;
fn fifo_path_key(scheme_id: scheme::SchemeId, number: usize, path: &str) -> String {
if path.starts_with('/') {
path.to_string()
} else {
format!("@fifo:{}:{}:{}", scheme_id.get(), number, path)
}
}
fn install_open_result(
scheme_id: scheme::SchemeId,
pub fn openat(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
open_result: OpenResult,
fcntl_flags: u32,
euid: u32,
egid: u32,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let new_description = match open_result {
OpenResult::SchemeLocal(number, internal_flags) => Arc::new(RwLock::new(
FileDescription::new(
scheme_id,
number,
0,
(flags & !O_CLOEXEC) as u32,
internal_flags,
token,
),
)),
OpenResult::External(desc) => desc,
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
let (scheme_id, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?;
let desc = pipe.description.read(token.token());
(desc.scheme, desc.number)
};
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
let new_description = {
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
let res = scheme.kopenat(
number,
StrOrBytes::from_str(&path_buf),
flags,
fcntl_flags,
caller_ctx,
token,
);
match res? {
OpenResult::SchemeLocal(number, internal_flags) => {
Arc::new(RwLock::new(FileDescription {
offset: 0,
internal_flags,
scheme: scheme_id,
number,
flags: (flags & !O_CLOEXEC) as u32,
}))
}
OpenResult::External(desc) => desc,
}
};
let current_lock = context::current();
@@ -103,102 +126,6 @@ fn install_open_result(
)
.ok_or(Error::new(EMFILE))
}
fn path_exists_in_scheme(
scheme: &dyn KernelScheme,
number: usize,
path: &str,
caller_ctx: scheme::CallerCtx,
token: &mut CleanLockToken,
) -> Result<bool> {
match scheme.kopenat(number, StrOrBytes::from_str(path), O_STAT, 0, caller_ctx, token) {
Ok(OpenResult::SchemeLocal(number, _)) => {
let _ = scheme.close(number, token);
Ok(true)
}
Ok(OpenResult::External(_)) => Ok(true),
Err(err) if err.errno == ENOENT => Ok(false),
Err(err) => Err(err),
}
}
pub fn openat(
fh: FileHandle,
raw_path: UserSliceRo,
flags: usize,
fcntl_flags: u32,
euid: u32,
egid: u32,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
let desc = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut context_token) = current.token_split();
let pipe = context
.get_file(fh, &mut context_token)
.ok_or(Error::new(EBADF))?;
*pipe.description.read(context_token.token())
};
let scheme = desc.get_scheme(token)?;
let number = desc.number;
let scheme_id = desc.scheme;
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
let fifo_mode_requested = flags & MODE_FIFO as usize == MODE_FIFO as usize;
let fifo_key = fifo_path_key(scheme_id, number, &path_buf);
if pipe::named_pipe_exists(&fifo_key, token) {
if flags & O_EXCL == O_EXCL && flags & O_CREAT == O_CREAT {
return Err(Error::new(EEXIST));
}
if fifo_mode_requested && flags & O_CREAT == O_CREAT {
return Err(Error::new(EEXIST));
}
let pipe_number = pipe::open_named_pipe(&fifo_key, flags, token)?
.ok_or(Error::new(ENOENT))?;
return install_open_result(
GlobalSchemes::Pipe.scheme_id(),
flags,
OpenResult::SchemeLocal(pipe_number, InternalFlags::empty()),
token,
);
}
if fifo_mode_requested && flags & O_CREAT == O_CREAT {
if path_exists_in_scheme(&*scheme, number, &path_buf, caller_ctx, token)? {
return Err(Error::new(EEXIST));
}
let mode = u16::try_from(flags & 0o7777).map_err(|_| Error::new(EINVAL))?;
let pipe_number = pipe::create_named_pipe(&fifo_key, &path_buf, mode, flags, token)?;
return install_open_result(
GlobalSchemes::Pipe.scheme_id(),
flags,
OpenResult::SchemeLocal(pipe_number, InternalFlags::empty()),
token,
);
}
let open_result = scheme.kopenat(
number,
StrOrBytes::from_str(&path_buf),
flags,
fcntl_flags,
caller_ctx,
token,
)?;
install_open_result(scheme_id, flags, open_result, token)
}
/// Unlinkat syscall
pub fn unlinkat(
fh: FileHandle,
@@ -210,27 +137,22 @@ pub fn unlinkat(
) -> Result<()> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
let desc = {
let (number, scheme_id) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut context_token) = current.token_split();
let pipe = context
.get_file(fh, &mut context_token)
.ok_or(Error::new(EBADF))?;
*pipe.description.read(context_token.token())
let (context, mut token) = current.token_split();
let pipe = context.get_file(fh, &mut token).ok_or(Error::new(EBADF))?;
let desc = pipe.description.read(token.token());
(desc.number, desc.scheme)
};
let number = desc.number;
let scheme = desc.get_scheme(token)?;
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
let caller_ctx = context::current()
.read(token.token())
.caller_ctx()
.filter_uid_gid(euid, egid);
if pipe::unlink_named_pipe(&fifo_path_key(desc.scheme, number, &path_buf), token) {
return Ok(());
}
/*
let mut path_buf = BorrowedHtBuf::head()?;
let path = path_buf.use_for_string(raw_path)?;
@@ -277,18 +199,17 @@ fn duplicate_file(
let description = { *file.description.read(token.token()) };
let new_description = {
let scheme = description.get_scheme(token)?;
let scheme = scheme::get_scheme(token.token(), description.scheme)?;
match scheme.kdup(description.number, user_buf, caller_ctx, token)? {
OpenResult::SchemeLocal(number, internal_flags) => {
Arc::new(RwLock::new(FileDescription::new(
description.scheme,
number,
0,
description.flags,
Arc::new(RwLock::new(FileDescription {
offset: 0,
internal_flags,
token,
)))
scheme: description.scheme,
number,
flags: description.flags,
}))
}
OpenResult::External(desc) => desc,
}
@@ -375,10 +296,11 @@ fn call_normal(
}
.ok_or(Error::new(EBADF))?;
let (scheme, number) = {
let desc = *file.description.read(token.token());
(desc.get_scheme(token)?, desc.number)
let (scheme_id, number) = {
let desc = file.description.read(token.token());
(desc.scheme, desc.number)
};
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
if flags.contains(CallFlags::STD_FS) {
scheme.translate_std_fs_call(number, file.description, payload, flags, metadata, token)
@@ -419,28 +341,28 @@ fn fdwrite_inner(
) -> Result<usize> {
// TODO: Ensure deadlocks can't happen
let (scheme, number, descs_to_send) = {
let desc = {
let (scheme, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut context_token) = current.token_split();
let (context, mut token) = current.token_split();
let file_descriptor = context
.get_file(socket, &mut context_token)
.get_file(socket, &mut token)
.ok_or(Error::new(EBADF))?;
*file_descriptor.description.read(context_token.token())
let desc = &file_descriptor.description.read(token.token());
(desc.scheme, desc.number)
};
let scheme = desc.get_scheme(token)?;
let number = desc.number;
let scheme = scheme::get_scheme(token.token(), scheme)?;
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut context_token) = current.token_split();
let (context, mut token) = current.token_split();
(
scheme,
number,
if flags.contains(CallFlags::FD_CLONE) {
context.bulk_get_files(&target_fds, &mut context_token)
context.bulk_get_files(&target_fds, &mut token)
} else {
context.bulk_remove_files(&target_fds, &mut context_token)
context.bulk_remove_files(&target_fds, &mut token)
}?
.into_iter()
.map(|f| f.description)
@@ -473,22 +395,18 @@ fn call_fdread(
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let desc = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut context_token) = current.token_split();
let file_descriptor = context
.get_file(fd, &mut context_token)
.ok_or(Error::new(EBADF))?;
*file_descriptor.description.read(context_token.token())
};
let (scheme, number) = {
let scheme = desc.get_scheme(token)?;
let number = desc.number;
(
scheme,
number,
)
let (scheme, number) = {
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut token) = current.token_split();
let file_descriptor = context.get_file(fd, &mut token).ok_or(Error::new(EBADF))?;
let desc = file_descriptor.description.read(token.token());
(desc.scheme, desc.number)
};
let scheme = scheme::get_scheme(token.token(), scheme)?;
(scheme, number)
};
scheme.kfdread(number, payload, flags, metadata, token)
@@ -522,9 +440,9 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
}
.ok_or(Error::new(EBADF))?;
let (number, flags, desc) = {
let desc = *file.description.read(token.token());
(desc.number, desc.flags, desc)
let (scheme_id, number, flags) = {
let desc = file.description.write(token.token());
(desc.scheme, desc.number, desc.flags)
};
if cmd == F_DUPFD || cmd == F_DUPFD_CLOEXEC {
@@ -542,7 +460,7 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken)
// Communicate fcntl with scheme
if cmd != F_GETFD && cmd != F_SETFD {
let scheme = desc.get_scheme(token)?;
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
scheme.fcntl(number, cmd, arg, token)?;
};
@@ -600,11 +518,13 @@ pub fn flink(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken)
let path = RedoxPath::from_absolute(&path_buf).ok_or(Error::new(EINVAL))?;
let (_, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
let (number, scheme) = {
let desc = *file.description.read(token.token());
(desc.number, desc.get_scheme(token)?)
let (number, scheme_id) = {
let desc = file.description.read(token.token());
(desc.number, desc.scheme)
};
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
// TODO: Check EXDEV.
/*
if scheme_id != description.scheme {
@@ -634,11 +554,13 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken
let path = RedoxPath::from_absolute(&path_buf).ok_or(Error::new(EINVAL))?;
let (_, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
let (number, scheme) = {
let desc = *file.description.read(token.token());
(desc.number, desc.get_scheme(token)?)
let (number, scheme_id) = {
let desc = file.description.read(token.token());
(desc.number, desc.scheme)
};
let scheme = scheme::get_scheme(token.token(), scheme_id)?;
// TODO: Check EXDEV.
/*
if scheme_id != description.scheme {
@@ -28,11 +28,6 @@ use crate::{
sync::CleanLockToken,
};
/// Local syscall numbers not yet in the redox_syscall crate.
/// These are allocated from the 987+ range to avoid collisions with crate numbers.
pub const SYS_SCHED_SETAFFINITY: usize = 987;
pub const SYS_SCHED_GETAFFINITY: usize = 988;
/// Debug
pub mod debug;
@@ -225,10 +220,6 @@ pub fn syscall(
unlinkat(fd, UserSlice::ro(c, d)?, e, f as _, g as _, token).map(|()| 0)
}
SYS_YIELD => sched_yield(token).map(|()| 0),
// P17-3: CPU affinity syscalls. Numbers allocated locally (not yet in redox_syscall crate).
SYS_SCHED_SETAFFINITY => sched_setaffinity(b, UserSlice::ro(c, d)?, token),
SYS_SCHED_GETAFFINITY => sched_getaffinity(b, UserSlice::wo(c, d)?, token),
SYS_NANOSLEEP => nanosleep(
UserSlice::ro(b, size_of::<TimeSpec>())?,
UserSlice::wo(c, size_of::<TimeSpec>())?.none_if_null(),
@@ -11,7 +11,6 @@ use crate::{
memory::{AddrSpace, Grant, PageSpan},
ContextRef,
},
cpu_set::RawMask,
event,
sync::{CleanLockToken, RwLock},
syscall::flag::{EventFlags, O_CREAT, O_RDWR},
@@ -272,95 +271,24 @@ unsafe fn bootstrap_mem(bootstrap: &crate::startup::Bootstrap) -> &'static [u8]
}
fn insert_fd(scheme: SchemeId, number: usize, cloexec: bool, token: &mut CleanLockToken) -> usize {
let description = Arc::new(RwLock::new(FileDescription::new(
scheme,
number,
0,
(O_CREAT | O_RDWR) as u32,
InternalFlags::empty(),
token,
)));
let current_lock = context::current();
let mut current = current_lock.read(token.token());
let (context, mut context_token) = current.token_split();
let (context, mut token) = current.token_split();
context
.add_file_min(
FileDescriptor {
description,
description: Arc::new(RwLock::new(FileDescription {
scheme,
number,
offset: 0,
flags: (O_CREAT | O_RDWR) as u32,
internal_flags: InternalFlags::empty(),
})),
cloexec,
},
syscall::flag::UPPER_FDTBL_TAG + scheme.get(),
&mut context_token,
&mut token,
)
.expect("failed to insert fd to current context")
.get()
}
/// Set CPU affinity mask for a process.
///
/// # Arguments (syscall ABI)
/// - `pid`: Process ID (0 = current process; other PIDs not yet supported)
/// - `mask_ptr`: Pointer to a `RawMask` (32 bytes on 64-bit, 256-bit bitmap)
/// - `mask_len`: Length of mask in bytes (must equal `size_of::<RawMask>()`)
pub fn sched_setaffinity(
pid: usize,
mask_ptr: super::usercopy::UserSliceRo,
token: &mut CleanLockToken,
) -> Result<usize> {
// Validate mask size
if mask_ptr.len() != core::mem::size_of::<RawMask>() {
return Err(Error::new(super::error::EINVAL));
}
// pid == 0 means current process
let target = if pid == 0 {
context::current()
} else {
// TODO: Support PID-based lookup (requires context list iteration
// with lock token downgrades). For now, only pid=0 is supported.
return Err(Error::new(super::error::ESRCH));
};
// Read mask from userspace
let raw_mask: RawMask = unsafe { mask_ptr.read_exact() }?;
// Apply to context's affinity mask
let mut ctx = target.write(token.token());
ctx.sched_affinity.override_from(&raw_mask);
Ok(0)
}
/// Get CPU affinity mask for a process.
///
/// # Arguments (syscall ABI)
/// - `pid`: Process ID (0 = current process; other PIDs not yet supported)
/// - `mask_ptr`: Pointer to a `RawMask` buffer (32 bytes on 64-bit)
/// - `mask_len`: Length of buffer in bytes (must equal `size_of::<RawMask>()`)
///
/// # Returns
/// Number of bytes written to mask_ptr on success.
pub fn sched_getaffinity(
pid: usize,
mask_ptr: super::usercopy::UserSliceWo,
token: &mut CleanLockToken,
) -> Result<usize> {
// Validate mask size
if mask_ptr.len() != core::mem::size_of::<RawMask>() {
return Err(Error::new(super::error::EINVAL));
}
// pid == 0 means current process
let target = if pid == 0 {
context::current()
} else {
return Err(Error::new(super::error::ESRCH));
};
let ctx = target.read(token.token());
let raw_mask = ctx.sched_affinity.to_raw();
mask_ptr.copy_common_bytes_from_slice(crate::cpu_set::mask_as_bytes(&raw_mask))?;
Ok(core::mem::size_of::<RawMask>())
}