diff --git a/build.rs b/build.rs index 65c227e384..3b68df9eb5 100644 --- a/build.rs +++ b/build.rs @@ -1,8 +1,6 @@ use rustc_cfg::Cfg; +use std::{env, path::Path, process::Command}; use toml::Table; -use std::env; -use std::path::Path; -use std::process::Command; fn parse_kconfig(arch: &str) -> Option<()> { println!("cargo:rerun-if-changed=config.toml"); @@ -14,9 +12,16 @@ fn parse_kconfig(arch: &str) -> Option<()> { let config_str = std::fs::read_to_string("config.toml").unwrap(); let root: Table = toml::from_str(&config_str).unwrap(); - let altfeatures = root.get("arch")?.as_table().unwrap() - .get(arch)?.as_table().unwrap() - .get("features")?.as_table().unwrap(); + let altfeatures = root + .get("arch")? + .as_table() + .unwrap() + .get(arch)? + .as_table() + .unwrap() + .get("features")? + .as_table() + .unwrap(); for (name, value) in altfeatures { let choice = value.as_str().unwrap(); @@ -45,30 +50,34 @@ fn main() { .target("aarch64-unknown-redox") .compile("early_init"); */ - }, + } "x86" => { println!("cargo:rerun-if-changed=src/asm/x86/trampoline.asm"); let status = Command::new("nasm") - .arg("-f").arg("bin") - .arg("-o").arg(format!("{}/trampoline", out_dir)) + .arg("-f") + .arg("bin") + .arg("-o") + .arg(format!("{}/trampoline", out_dir)) .arg("src/asm/x86/trampoline.asm") .status() .expect("failed to run nasm"); - if ! status.success() { + if !status.success() { panic!("nasm failed with exit status {}", status); } - }, + } "x86_64" => { println!("cargo:rerun-if-changed=src/asm/x86_64/trampoline.asm"); let status = Command::new("nasm") - .arg("-f").arg("bin") - .arg("-o").arg(format!("{}/trampoline", out_dir)) + .arg("-f") + .arg("bin") + .arg("-o") + .arg(format!("{}/trampoline", out_dir)) .arg("src/asm/x86_64/trampoline.asm") .status() .expect("failed to run nasm"); - if ! status.success() { + if !status.success() { panic!("nasm failed with exit status {}", status); } } diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000000..3c4f56848f --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,21 @@ +blank_lines_lower_bound = 0 +blank_lines_upper_bound = 1 +brace_style = "SameLineWhere" +disable_all_formatting = false +edition = "2018" +empty_item_single_line = true +fn_single_line = false +force_explicit_abi = true +format_strings = false +hard_tabs = false +hide_parse_errors = false +imports_granularity = "Crate" +imports_indent = "Block" +imports_layout = "Mixed" +indent_style = "Block" +max_width = 100 +newline_style = "Unix" +skip_children = false +tab_spaces = 4 +trailing_comma = "Vertical" +where_single_line = false diff --git a/src/acpi/hpet.rs b/src/acpi/hpet.rs index a2546b8e37..9931f3e29c 100644 --- a/src/acpi/hpet.rs +++ b/src/acpi/hpet.rs @@ -2,12 +2,12 @@ use core::{mem, ptr}; use core::ptr::{read_volatile, write_volatile}; -use crate::memory::Frame; -use crate::paging::{KernelMapper, PhysicalAddress, PageFlags}; -use crate::paging::entry::EntryFlags; +use crate::{ + memory::Frame, + paging::{entry::EntryFlags, KernelMapper, PageFlags, PhysicalAddress}, +}; -use super::sdt::Sdt; -use super::{ACPI_TABLE, find_sdt}; +use super::{find_sdt, sdt::Sdt, ACPI_TABLE}; #[repr(packed)] #[derive(Clone, Copy, Debug, Default)] @@ -32,7 +32,7 @@ pub struct Hpet { pub hpet_number: u8, pub min_periodic_clk_tick: u16, - pub oem_attribute: u8 + pub oem_attribute: u8, } impl Hpet { @@ -75,13 +75,21 @@ impl GenericAddressStructure { mapper .get_mut() - .expect("KernelMapper locked re-entrant while mapping memory for GenericAddressStructure") - .map_phys(page.start_address(), frame.start_address(), PageFlags::new().write(true).custom_flag(EntryFlags::NO_CACHE.bits(), true)) + .expect( + "KernelMapper locked re-entrant while mapping memory for GenericAddressStructure", + ) + .map_phys( + page.start_address(), + frame.start_address(), + PageFlags::new() + .write(true) + .custom_flag(EntryFlags::NO_CACHE.bits(), true), + ) .expect("failed to map memory for GenericAddressStructure") .flush(); } - pub unsafe fn read_u64(&self, offset: usize) -> u64{ + pub unsafe fn read_u64(&self, offset: usize) -> u64 { read_volatile((crate::HPET_OFFSET + offset) as *const u64) } @@ -96,17 +104,27 @@ impl GenericAddressStructure { let frame = Frame::containing_address(PhysicalAddress::new(self.address as usize)); let (_, result) = mapper .get_mut() - .expect("KernelMapper locked re-entrant while mapping memory for GenericAddressStructure") - .map_linearly(frame.start_address(), PageFlags::new().write(true).custom_flag(EntryFlags::NO_CACHE.bits(), true)) + .expect( + "KernelMapper locked re-entrant while mapping memory for GenericAddressStructure", + ) + .map_linearly( + frame.start_address(), + PageFlags::new() + .write(true) + .custom_flag(EntryFlags::NO_CACHE.bits(), true), + ) .expect("failed to map memory for GenericAddressStructure"); result.flush(); } - pub unsafe fn read_u64(&self, offset: usize) -> u64{ + pub unsafe fn read_u64(&self, offset: usize) -> u64 { read_volatile((self.address as usize + offset + crate::PHYS_OFFSET) as *const u64) } pub unsafe fn write_u64(&mut self, offset: usize, value: u64) { - write_volatile((self.address as usize + offset + crate::PHYS_OFFSET) as *mut u64, value); + write_volatile( + (self.address as usize + offset + crate::PHYS_OFFSET) as *mut u64, + value, + ); } } diff --git a/src/acpi/madt.rs b/src/acpi/madt.rs index a46a5fdc66..076a7d03ac 100644 --- a/src/acpi/madt.rs +++ b/src/acpi/madt.rs @@ -1,23 +1,26 @@ use core::mem; -use crate::memory::{allocate_frames, Frame}; -use crate::paging::{KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch, VirtualAddress}; +use crate::{ + memory::{allocate_frames, Frame}, + paging::{KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch, VirtualAddress}, +}; -use super::sdt::Sdt; -use super::find_sdt; +use super::{find_sdt, sdt::Sdt}; use core::sync::atomic::{AtomicU8, Ordering}; -use crate::device::local_apic::LOCAL_APIC; -use crate::interrupt; -use crate::start::{kstart_ap, CPU_COUNT, AP_READY}; +use crate::{ + device::local_apic::LOCAL_APIC, + interrupt, + start::{kstart_ap, AP_READY, CPU_COUNT}, +}; /// The Multiple APIC Descriptor Table #[derive(Clone, Copy, Debug)] pub struct Madt { sdt: &'static Sdt, pub local_address: u32, - pub flags: u32 + pub flags: u32, } const TRAMPOLINE: usize = 0x8000; @@ -80,85 +83,96 @@ impl Madt { for madt_entry in madt.iter() { println!(" {:?}", madt_entry); match madt_entry { - MadtEntry::LocalApic(ap_local_apic) => if ap_local_apic.id == me { - println!(" This is my local APIC"); - } else { - if ap_local_apic.flags & 1 == 1 { - // Increase CPU ID - CPU_COUNT.fetch_add(1, Ordering::SeqCst); - - // Allocate a stack - let stack_start = allocate_frames(64).expect("no more frames in acpi stack_start").start_address().data() + crate::PHYS_OFFSET; - let stack_end = stack_start + 64 * 4096; - - let ap_ready = (TRAMPOLINE + 8) as *mut u64; - let ap_cpu_id = unsafe { ap_ready.add(1) }; - let ap_page_table = unsafe { ap_ready.add(2) }; - let ap_stack_start = unsafe { ap_ready.add(3) }; - let ap_stack_end = unsafe { ap_ready.add(4) }; - let ap_code = unsafe { ap_ready.add(5) }; - - // Set the ap_ready to 0, volatile - unsafe { - ap_ready.write(0); - ap_cpu_id.write(ap_local_apic.id.into()); - ap_page_table.write(page_table_physaddr as u64); - ap_stack_start.write(stack_start as u64); - ap_stack_end.write(stack_end as u64); - ap_code.write(kstart_ap as u64); - - // TODO: Is this necessary (this fence)? - core::arch::asm!(""); - }; - AP_READY.store(false, Ordering::SeqCst); - - print!(" AP {}:", ap_local_apic.id); - - // Send INIT IPI - { - let mut icr = 0x4500; - if local_apic.x2 { - icr |= (ap_local_apic.id as u64) << 32; - } else { - icr |= (ap_local_apic.id as u64) << 56; - } - print!(" IPI..."); - local_apic.set_icr(icr); - } - - // Send START IPI - { - //Start at 0x0800:0000 => 0x8000. Hopefully the bootloader code is still there - let ap_segment = (TRAMPOLINE >> 12) & 0xFF; - let mut icr = 0x4600 | ap_segment as u64; - - if local_apic.x2 { - icr |= (ap_local_apic.id as u64) << 32; - } else { - icr |= (ap_local_apic.id as u64) << 56; - } - - print!(" SIPI..."); - local_apic.set_icr(icr); - } - - // Wait for trampoline ready - print!(" Wait..."); - while unsafe { (*ap_ready.cast::()).load(Ordering::SeqCst) } == 0 { - interrupt::pause(); - } - print!(" Trampoline..."); - while ! AP_READY.load(Ordering::SeqCst) { - interrupt::pause(); - } - println!(" Ready"); - - unsafe { RmmA::invalidate_all(); } + MadtEntry::LocalApic(ap_local_apic) => { + if ap_local_apic.id == me { + println!(" This is my local APIC"); } else { - println!(" CPU Disabled"); + if ap_local_apic.flags & 1 == 1 { + // Increase CPU ID + CPU_COUNT.fetch_add(1, Ordering::SeqCst); + + // Allocate a stack + let stack_start = allocate_frames(64) + .expect("no more frames in acpi stack_start") + .start_address() + .data() + + crate::PHYS_OFFSET; + let stack_end = stack_start + 64 * 4096; + + let ap_ready = (TRAMPOLINE + 8) as *mut u64; + let ap_cpu_id = unsafe { ap_ready.add(1) }; + let ap_page_table = unsafe { ap_ready.add(2) }; + let ap_stack_start = unsafe { ap_ready.add(3) }; + let ap_stack_end = unsafe { ap_ready.add(4) }; + let ap_code = unsafe { ap_ready.add(5) }; + + // Set the ap_ready to 0, volatile + unsafe { + ap_ready.write(0); + ap_cpu_id.write(ap_local_apic.id.into()); + ap_page_table.write(page_table_physaddr as u64); + ap_stack_start.write(stack_start as u64); + ap_stack_end.write(stack_end as u64); + ap_code.write(kstart_ap as u64); + + // TODO: Is this necessary (this fence)? + core::arch::asm!(""); + }; + AP_READY.store(false, Ordering::SeqCst); + + print!(" AP {}:", ap_local_apic.id); + + // Send INIT IPI + { + let mut icr = 0x4500; + if local_apic.x2 { + icr |= (ap_local_apic.id as u64) << 32; + } else { + icr |= (ap_local_apic.id as u64) << 56; + } + print!(" IPI..."); + local_apic.set_icr(icr); + } + + // Send START IPI + { + //Start at 0x0800:0000 => 0x8000. Hopefully the bootloader code is still there + let ap_segment = (TRAMPOLINE >> 12) & 0xFF; + let mut icr = 0x4600 | ap_segment as u64; + + if local_apic.x2 { + icr |= (ap_local_apic.id as u64) << 32; + } else { + icr |= (ap_local_apic.id as u64) << 56; + } + + print!(" SIPI..."); + local_apic.set_icr(icr); + } + + // Wait for trampoline ready + print!(" Wait..."); + while unsafe { + (*ap_ready.cast::()).load(Ordering::SeqCst) + } == 0 + { + interrupt::pause(); + } + print!(" Trampoline..."); + while !AP_READY.load(Ordering::SeqCst) { + interrupt::pause(); + } + println!(" Ready"); + + unsafe { + RmmA::invalidate_all(); + } + } else { + println!(" CPU Disabled"); + } } - }, - _ => () + } + _ => (), } } @@ -176,14 +190,15 @@ impl Madt { } pub fn new(sdt: &'static Sdt) -> Option { - if &sdt.signature == b"APIC" && sdt.data_len() >= 8 { //Not valid if no local address and flags + if &sdt.signature == b"APIC" && sdt.data_len() >= 8 { + //Not valid if no local address and flags let local_address = unsafe { *(sdt.data_address() as *const u32) }; let flags = unsafe { *(sdt.data_address() as *const u32).offset(1) }; Some(Madt { sdt, local_address, - flags + flags, }) } else { None @@ -193,7 +208,7 @@ impl Madt { pub fn iter(&self) -> MadtIter { MadtIter { sdt: self.sdt, - i: 8 // Skip local controller address and flags + i: 8, // Skip local controller address and flags } } } @@ -207,7 +222,7 @@ pub struct MadtLocalApic { /// Local APIC ID pub id: u8, /// Flags. 1 means that the processor is enabled - pub flags: u32 + pub flags: u32, } /// MADT I/O APIC @@ -221,7 +236,7 @@ pub struct MadtIoApic { /// I/O APIC address pub address: u32, /// Global system interrupt base - pub gsi_base: u32 + pub gsi_base: u32, } /// MADT Interrupt Source Override @@ -235,7 +250,7 @@ pub struct MadtIntSrcOverride { /// Global system interrupt base pub gsi_base: u32, /// Flags - pub flags: u16 + pub flags: u16, } /// MADT Entries @@ -247,12 +262,12 @@ pub enum MadtEntry { InvalidIoApic(usize), IntSrcOverride(&'static MadtIntSrcOverride), InvalidIntSrcOverride(usize), - Unknown(u8) + Unknown(u8), } pub struct MadtIter { sdt: &'static Sdt, - i: usize + i: usize, } impl Iterator for MadtIter { @@ -260,26 +275,40 @@ impl Iterator for MadtIter { fn next(&mut self) -> Option { if self.i + 1 < self.sdt.data_len() { let entry_type = unsafe { *(self.sdt.data_address() as *const u8).add(self.i) }; - let entry_len = unsafe { *(self.sdt.data_address() as *const u8).add(self.i + 1) } as usize; + let entry_len = + unsafe { *(self.sdt.data_address() as *const u8).add(self.i + 1) } as usize; if self.i + entry_len <= self.sdt.data_len() { let item = match entry_type { - 0 => if entry_len == mem::size_of::() + 2 { - MadtEntry::LocalApic(unsafe { &*((self.sdt.data_address() + self.i + 2) as *const MadtLocalApic) }) - } else { - MadtEntry::InvalidLocalApic(entry_len) - }, - 1 => if entry_len == mem::size_of::() + 2 { - MadtEntry::IoApic(unsafe { &*((self.sdt.data_address() + self.i + 2) as *const MadtIoApic) }) - } else { - MadtEntry::InvalidIoApic(entry_len) - }, - 2 => if entry_len == mem::size_of::() + 2 { - MadtEntry::IntSrcOverride(unsafe { &*((self.sdt.data_address() + self.i + 2) as *const MadtIntSrcOverride) }) - } else { - MadtEntry::InvalidIntSrcOverride(entry_len) - }, - _ => MadtEntry::Unknown(entry_type) + 0 => { + if entry_len == mem::size_of::() + 2 { + MadtEntry::LocalApic(unsafe { + &*((self.sdt.data_address() + self.i + 2) as *const MadtLocalApic) + }) + } else { + MadtEntry::InvalidLocalApic(entry_len) + } + } + 1 => { + if entry_len == mem::size_of::() + 2 { + MadtEntry::IoApic(unsafe { + &*((self.sdt.data_address() + self.i + 2) as *const MadtIoApic) + }) + } else { + MadtEntry::InvalidIoApic(entry_len) + } + } + 2 => { + if entry_len == mem::size_of::() + 2 { + MadtEntry::IntSrcOverride(unsafe { + &*((self.sdt.data_address() + self.i + 2) + as *const MadtIntSrcOverride) + }) + } else { + MadtEntry::InvalidIntSrcOverride(entry_len) + } + } + _ => MadtEntry::Unknown(entry_type), }; self.i += entry_len; diff --git a/src/acpi/mod.rs b/src/acpi/mod.rs index 4ee210d0f6..69f49601ba 100644 --- a/src/acpi/mod.rs +++ b/src/acpi/mod.rs @@ -1,38 +1,37 @@ //! # ACPI //! Code to parse the ACPI tables -use alloc::string::String; -use alloc::vec::Vec; -use alloc::boxed::Box; +use alloc::{boxed::Box, string::String, vec::Vec}; use hashbrown::HashMap; use spin::{Once, RwLock}; -use crate::log::info; -use crate::paging::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}; +use crate::{ + log::info, + paging::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}, +}; -use self::madt::Madt; -use self::rsdt::Rsdt; -use self::sdt::Sdt; -use self::xsdt::Xsdt; -use self::hpet::Hpet; -use self::rxsdt::Rxsdt; -use self::rsdp::RSDP; +use self::{hpet::Hpet, madt::Madt, rsdp::RSDP, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt}; pub mod hpet; pub mod madt; +mod rsdp; mod rsdt; +mod rxsdt; pub mod sdt; mod xsdt; -mod rxsdt; -mod rsdp; unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::paging::PageMapper) { let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data())); let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data())); for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE { - let (_, flush) = mapper.map_linearly(base.add(page_idx * crate::memory::PAGE_SIZE), PageFlags::new()).expect("failed to linearly map SDT"); + let (_, flush) = mapper + .map_linearly( + base.add(page_idx * crate::memory::PAGE_SIZE), + PageFlags::new(), + ) + .expect("failed to linearly map SDT"); flush.flush(); } } @@ -52,7 +51,11 @@ pub fn get_sdt(sdt_address: usize, mapper: &mut KernelMapper) -> &'static Sdt { sdt = &*(RmmA::phys_to_virt(physaddr).data() as *const Sdt); - map_linearly(physaddr.add(SDT_SIZE), sdt.length as usize - SDT_SIZE, mapper); + map_linearly( + physaddr.add(SDT_SIZE), + sdt.length as usize - SDT_SIZE, + mapper, + ); } sdt } @@ -151,7 +154,7 @@ pub type SdtSignature = (String, [u8; 6], [u8; 8]); pub static SDT_POINTERS: RwLock>> = RwLock::new(None); pub fn find_sdt(name: &str) -> Vec<&'static Sdt> { - let mut sdts: Vec<&'static Sdt> = vec!(); + let mut sdts: Vec<&'static Sdt> = vec![]; if let Some(ref ptrs) = *(SDT_POINTERS.read()) { for (signature, sdt) in ptrs { @@ -165,7 +168,8 @@ pub fn find_sdt(name: &str) -> Vec<&'static Sdt> { } pub fn get_sdt_signature(sdt: &'static Sdt) -> SdtSignature { - let signature = String::from_utf8(sdt.signature.to_vec()).expect("Error converting signature to string"); + let signature = + String::from_utf8(sdt.signature.to_vec()).expect("Error converting signature to string"); (signature, sdt.oem_id, sdt.oem_table_id) } diff --git a/src/acpi/rsdp.rs b/src/acpi/rsdp.rs index 6239d8824d..a208c6459c 100644 --- a/src/acpi/rsdp.rs +++ b/src/acpi/rsdp.rs @@ -1,8 +1,9 @@ -use core::convert::TryFrom; -use core::mem; +use core::{convert::TryFrom, mem}; -use crate::memory::Frame; -use crate::paging::{KernelMapper, Page, PageFlags, PhysicalAddress, VirtualAddress}; +use crate::{ + memory::Frame, + paging::{KernelMapper, Page, PageFlags, PhysicalAddress, VirtualAddress}, +}; /// RSDP #[derive(Copy, Clone, Debug)] @@ -16,7 +17,7 @@ pub struct RSDP { _length: u32, xsdt_address: u64, _extended_checksum: u8, - _reserved: [u8; 3] + _reserved: [u8; 3], } impl RSDP { @@ -36,12 +37,16 @@ impl RSDP { type Item = &'a [u8]; fn next(&mut self) -> Option { - if self.buf.len() < 4 { return None } + if self.buf.len() < 4 { + return None; + } let length_bytes = <[u8; 4]>::try_from(&self.buf[..4]).ok()?; let length = u32::from_ne_bytes(length_bytes) as usize; - if (4 + length as usize) > self.buf.len() { return None } + if (4 + length as usize) > self.buf.len() { + return None; + } let buf = &self.buf[4..4 + length]; self.buf = &self.buf[4 + length..]; @@ -56,24 +61,36 @@ impl RSDP { let rsdp = unsafe { &*(slice.as_ptr() as *const RSDP) }; // TODO: Validate Some(rsdp) - } else { None } + } else { + None + } } // first, find an RSDP for ACPI 2.0 - if let Some(rsdp_2_0) = (Iter { buf: area }.filter_map(slice_to_rsdp).find(|rsdp| rsdp.is_acpi_2_0())) { + if let Some(rsdp_2_0) = (Iter { buf: area } + .filter_map(slice_to_rsdp) + .find(|rsdp| rsdp.is_acpi_2_0())) + { return Some(*rsdp_2_0); } // secondly, find an RSDP for ACPI 1.0 - if let Some(rsdp_1_0) = (Iter { buf: area }.filter_map(slice_to_rsdp).find(|rsdp| rsdp.is_acpi_1_0())) { + if let Some(rsdp_1_0) = (Iter { buf: area } + .filter_map(slice_to_rsdp) + .find(|rsdp| rsdp.is_acpi_1_0())) + { return Some(*rsdp_1_0); } None } - pub fn get_rsdp(mapper: &mut KernelMapper, already_supplied_rsdps: Option<(u64, u64)>) -> Option { + pub fn get_rsdp( + mapper: &mut KernelMapper, + already_supplied_rsdps: Option<(u64, u64)>, + ) -> Option { if let Some((base, size)) = already_supplied_rsdps { - let area = unsafe { core::slice::from_raw_parts(base as usize as *const u8, size as usize) }; + let area = + unsafe { core::slice::from_raw_parts(base as usize as *const u8, size as usize) }; Self::get_already_supplied_rsdps(area).or_else(|| Self::get_rsdp_by_searching(mapper)) } else { Self::get_rsdp_by_searching(mapper) @@ -89,9 +106,18 @@ impl RSDP { let start_frame = Frame::containing_address(PhysicalAddress::new(start_addr)); let end_frame = Frame::containing_address(PhysicalAddress::new(end_addr)); for frame in Frame::range_inclusive(start_frame, end_frame) { - let page = Page::containing_address(VirtualAddress::new(frame.start_address().data())); + let page = + Page::containing_address(VirtualAddress::new(frame.start_address().data())); let result = unsafe { - mapper.get_mut().expect("KernelMapper locked re-entrant while locating RSDPs").map_phys(page.start_address(), frame.start_address(), PageFlags::new()).expect("failed to map page while searching for RSDP") + mapper + .get_mut() + .expect("KernelMapper locked re-entrant while locating RSDPs") + .map_phys( + page.start_address(), + frame.start_address(), + PageFlags::new(), + ) + .expect("failed to map page while searching for RSDP") }; result.flush(); } @@ -101,7 +127,7 @@ impl RSDP { } fn search(start_addr: usize, end_addr: usize) -> Option { - for i in 0 .. (end_addr + 1 - start_addr)/16 { + for i in 0..(end_addr + 1 - start_addr) / 16 { let rsdp = unsafe { &*((start_addr + i * 16) as *const RSDP) }; if &rsdp.signature == b"RSD PTR " { return Some(*rsdp); diff --git a/src/acpi/rsdt.rs b/src/acpi/rsdt.rs index 750c464714..3a1a66fbf0 100644 --- a/src/acpi/rsdt.rs +++ b/src/acpi/rsdt.rs @@ -1,9 +1,7 @@ -use core::convert::TryFrom; -use core::mem; use alloc::boxed::Box; +use core::{convert::TryFrom, mem}; -use super::sdt::Sdt; -use super::rxsdt::Rxsdt; +use super::{rxsdt::Rxsdt, sdt::Sdt}; #[derive(Debug)] pub struct Rsdt(&'static Sdt); @@ -17,33 +15,28 @@ impl Rsdt { } } pub fn as_slice(&self) -> &[u8] { - let length = usize::try_from(self.0.length) - .expect("expected 32-bit length to fit within usize"); + let length = + usize::try_from(self.0.length).expect("expected 32-bit length to fit within usize"); - unsafe { - core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) - } + unsafe { core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) } } } impl Rxsdt for Rsdt { fn iter(&self) -> Box> { - Box::new(RsdtIter { - sdt: self.0, - i: 0 - }) + Box::new(RsdtIter { sdt: self.0, i: 0 }) } } pub struct RsdtIter { sdt: &'static Sdt, - i: usize + i: usize, } impl Iterator for RsdtIter { type Item = usize; fn next(&mut self) -> Option { - if self.i < self.sdt.data_len()/mem::size_of::() { + if self.i < self.sdt.data_len() / mem::size_of::() { let item = unsafe { *(self.sdt.data_address() as *const u32).add(self.i) }; self.i += 1; Some(item as usize) diff --git a/src/acpi/rxsdt.rs b/src/acpi/rxsdt.rs index ffcb869832..bca5dd0f75 100644 --- a/src/acpi/rxsdt.rs +++ b/src/acpi/rxsdt.rs @@ -2,8 +2,7 @@ use alloc::boxed::Box; use crate::paging::KernelMapper; -use super::sdt::Sdt; -use super::get_sdt; +use super::{get_sdt, sdt::Sdt}; pub trait Rxsdt { fn iter(&self) -> Box>; @@ -15,7 +14,12 @@ pub trait Rxsdt { } } - fn find(&self, signature: [u8; 4], oem_id: [u8; 6], oem_table_id: [u8; 8]) -> Option<&'static Sdt> { + fn find( + &self, + signature: [u8; 4], + oem_id: [u8; 6], + oem_table_id: [u8; 8], + ) -> Option<&'static Sdt> { for sdt in self.iter() { let sdt = unsafe { &*(sdt as *const Sdt) }; diff --git a/src/acpi/sdt.rs b/src/acpi/sdt.rs index ea856272f7..1dbca3f262 100644 --- a/src/acpi/sdt.rs +++ b/src/acpi/sdt.rs @@ -1,18 +1,17 @@ -use core::mem; -use core::slice; +use core::{mem, slice}; #[derive(Copy, Clone, Debug)] #[repr(packed)] pub struct Sdt { - pub signature: [u8; 4], - pub length: u32, - pub revision: u8, - pub checksum: u8, - pub oem_id: [u8; 6], - pub oem_table_id: [u8; 8], - pub oem_revision: u32, - pub creator_id: u32, - pub creator_revision: u32 + pub signature: [u8; 4], + pub length: u32, + pub revision: u8, + pub checksum: u8, + pub oem_id: [u8; 6], + pub oem_table_id: [u8; 8], + pub oem_revision: u32, + pub creator_id: u32, + pub creator_revision: u32, } impl Sdt { @@ -36,7 +35,12 @@ impl Sdt { unsafe { slice::from_raw_parts(self.data_address() as *const u8, self.data_len()) } } - pub fn match_pattern(&self, signature: [u8; 4], oem_id: [u8; 6], oem_table_id: [u8; 8]) -> bool{ + pub fn match_pattern( + &self, + signature: [u8; 4], + oem_id: [u8; 6], + oem_table_id: [u8; 8], + ) -> bool { self.signature == signature && self.oem_id == oem_id && self.oem_table_id == oem_table_id } } diff --git a/src/acpi/xsdt.rs b/src/acpi/xsdt.rs index 6dc69b663d..dd6736a050 100644 --- a/src/acpi/xsdt.rs +++ b/src/acpi/xsdt.rs @@ -1,9 +1,7 @@ -use core::convert::TryFrom; -use core::mem; use alloc::boxed::Box; +use core::{convert::TryFrom, mem}; -use super::sdt::Sdt; -use super::rxsdt::Rxsdt; +use super::{rxsdt::Rxsdt, sdt::Sdt}; #[derive(Debug)] pub struct Xsdt(&'static Sdt); @@ -17,33 +15,28 @@ impl Xsdt { } } pub fn as_slice(&self) -> &[u8] { - let length = usize::try_from(self.0.length) - .expect("expected 32-bit length to fit within usize"); + let length = + usize::try_from(self.0.length).expect("expected 32-bit length to fit within usize"); - unsafe { - core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) - } + unsafe { core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) } } } impl Rxsdt for Xsdt { fn iter(&self) -> Box> { - Box::new(XsdtIter { - sdt: self.0, - i: 0 - }) + Box::new(XsdtIter { sdt: self.0, i: 0 }) } } pub struct XsdtIter { sdt: &'static Sdt, - i: usize + i: usize, } impl Iterator for XsdtIter { type Item = usize; fn next(&mut self) -> Option { - if self.i < self.sdt.data_len()/mem::size_of::() { + if self.i < self.sdt.data_len() / mem::size_of::() { let item = unsafe { *(self.sdt.data_address() as *const u64).add(self.i) }; self.i += 1; Some(item as usize) diff --git a/src/allocator/linked_list.rs b/src/allocator/linked_list.rs index 496f8ff6d4..77278d3a1c 100644 --- a/src/allocator/linked_list.rs +++ b/src/allocator/linked_list.rs @@ -1,5 +1,7 @@ -use core::alloc::{GlobalAlloc, Layout}; -use core::ptr::{self, NonNull}; +use core::{ + alloc::{GlobalAlloc, Layout}, + ptr::{self, NonNull}, +}; use linked_list_allocator::Heap; use spin::Mutex; @@ -21,10 +23,18 @@ unsafe impl GlobalAlloc for Allocator { match heap.allocate_first_fit(layout) { Err(()) => { let size = heap.size(); - super::map_heap(&mut KernelMapper::lock(), crate::KERNEL_HEAP_OFFSET + size, crate::KERNEL_HEAP_SIZE); + super::map_heap( + &mut KernelMapper::lock(), + crate::KERNEL_HEAP_OFFSET + size, + crate::KERNEL_HEAP_SIZE, + ); heap.extend(crate::KERNEL_HEAP_SIZE); - }, - other => return other.ok().map_or(ptr::null_mut(), |allocation| allocation.as_ptr()), + } + other => { + return other + .ok() + .map_or(ptr::null_mut(), |allocation| allocation.as_ptr()) + } } } panic!("__rust_allocate: heap not initialized"); diff --git a/src/allocator/mod.rs b/src/allocator/mod.rs index 4da36eb300..a3fceaafd7 100644 --- a/src/allocator/mod.rs +++ b/src/allocator/mod.rs @@ -1,26 +1,34 @@ +use crate::paging::{mapper::PageFlushAll, KernelMapper, Page, PageFlags, VirtualAddress}; use rmm::Flusher; -use crate::paging::{KernelMapper, Page, PageFlags, VirtualAddress, mapper::PageFlushAll}; -#[cfg(not(feature="slab"))] +#[cfg(not(feature = "slab"))] pub use self::linked_list::Allocator; -#[cfg(feature="slab")] +#[cfg(feature = "slab")] pub use self::slab::Allocator; -#[cfg(not(feature="slab"))] +#[cfg(not(feature = "slab"))] mod linked_list; -#[cfg(feature="slab")] +#[cfg(feature = "slab")] mod slab; unsafe fn map_heap(mapper: &mut KernelMapper, offset: usize, size: usize) { - let mapper = mapper.get_mut().expect("failed to obtain exclusive access to KernelMapper while extending heap"); + let mapper = mapper + .get_mut() + .expect("failed to obtain exclusive access to KernelMapper while extending heap"); let mut flush_all = PageFlushAll::new(); let heap_start_page = Page::containing_address(VirtualAddress::new(offset)); - let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size-1)); + let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1)); for page in Page::range_inclusive(heap_start_page, heap_end_page) { - let result = mapper.map(page.start_address(), PageFlags::new().write(true).global(cfg!(not(feature = "pti")))) + let result = mapper + .map( + page.start_address(), + PageFlags::new() + .write(true) + .global(cfg!(not(feature = "pti"))), + ) .expect("failed to map kernel heap"); flush_all.consume(result); } diff --git a/src/allocator/slab.rs b/src/allocator/slab.rs index 9ada9e6899..2185645a5d 100644 --- a/src/allocator/slab.rs +++ b/src/allocator/slab.rs @@ -1,6 +1,6 @@ use core::alloc::{Alloc, AllocErr, Layout}; -use spin::Mutex; use slab_allocator::Heap; +use spin::Mutex; static HEAP: Mutex> = Mutex::new(None); diff --git a/src/arch/aarch64/consts.rs b/src/arch/aarch64/consts.rs index ec27a3fbc8..5a3c672c0c 100644 --- a/src/arch/aarch64/consts.rs +++ b/src/arch/aarch64/consts.rs @@ -8,15 +8,15 @@ pub const PML4_MASK: usize = 0x0000_ff80_0000_0000; /// Offset of recursive paging (deprecated, but still reserved) pub const RECURSIVE_PAGE_OFFSET: usize = (-(PML4_SIZE as isize)) as usize; -pub const RECURSIVE_PAGE_PML4: usize = (RECURSIVE_PAGE_OFFSET & PML4_MASK)/PML4_SIZE; +pub const RECURSIVE_PAGE_PML4: usize = (RECURSIVE_PAGE_OFFSET & PML4_MASK) / PML4_SIZE; /// Offset of kernel pub const KERNEL_OFFSET: usize = RECURSIVE_PAGE_OFFSET - PML4_SIZE; -pub const KERNEL_PML4: usize = (KERNEL_OFFSET & PML4_MASK)/PML4_SIZE; +pub const KERNEL_PML4: usize = (KERNEL_OFFSET & PML4_MASK) / PML4_SIZE; /// Offset to kernel heap pub const KERNEL_HEAP_OFFSET: usize = KERNEL_OFFSET - PML4_SIZE; -pub const KERNEL_HEAP_PML4: usize = (KERNEL_HEAP_OFFSET & PML4_MASK)/PML4_SIZE; +pub const KERNEL_HEAP_PML4: usize = (KERNEL_HEAP_OFFSET & PML4_MASK) / PML4_SIZE; /// Size of kernel heap pub const KERNEL_HEAP_SIZE: usize = 1 * 1024 * 1024; // 1 MB @@ -25,7 +25,7 @@ pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE; /// Offset to kernel percpu variables pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_TMP_MISC_OFFSET - PML4_SIZE; -pub const KERNEL_PERCPU_PML4: usize = (KERNEL_PERCPU_OFFSET & PML4_MASK)/PML4_SIZE; +pub const KERNEL_PERCPU_PML4: usize = (KERNEL_PERCPU_OFFSET & PML4_MASK) / PML4_SIZE; /// Size of kernel percpu variables pub const KERNEL_PERCPU_SHIFT: u8 = 16; // 2^16 = 64 KiB pub const KERNEL_PERCPU_SIZE: usize = 1_usize << KERNEL_PERCPU_SHIFT; @@ -33,7 +33,7 @@ pub const KERNEL_PERCPU_SIZE: usize = 1_usize << KERNEL_PERCPU_SHIFT; /// Offset of physmap // This needs to match RMM's PHYS_OFFSET pub const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; -pub const PHYS_PML4: usize = (PHYS_OFFSET & PML4_MASK)/PML4_SIZE; +pub const PHYS_PML4: usize = (PHYS_OFFSET & PML4_MASK) / PML4_SIZE; /// End offset of the user image, i.e. kernel start pub const USER_END_OFFSET: usize = 256 * PML4_SIZE; diff --git a/src/arch/aarch64/debug.rs b/src/arch/aarch64/debug.rs index a1d0260ec1..a5fe11b35c 100644 --- a/src/arch/aarch64/debug.rs +++ b/src/arch/aarch64/debug.rs @@ -1,15 +1,12 @@ use core::fmt; use spin::MutexGuard; -use crate::log::{LOG, Log}; +use crate::log::{Log, LOG}; -#[cfg(feature = "graphical_debug")] -use crate::devices::graphical_debug::{DEBUG_DISPLAY, DebugDisplay}; #[cfg(feature = "serial_debug")] -use super::device::{ - serial::COM1, - uart_pl011::SerialPort, -}; +use super::device::{serial::COM1, uart_pl011::SerialPort}; +#[cfg(feature = "graphical_debug")] +use crate::devices::graphical_debug::{DebugDisplay, DEBUG_DISPLAY}; pub struct Writer<'a> { log: MutexGuard<'a, Option>, diff --git a/src/arch/aarch64/device/cpu/mod.rs b/src/arch/aarch64/device/cpu/mod.rs index 6e88a9f26e..6d9ca8cdf6 100644 --- a/src/arch/aarch64/device/cpu/mod.rs +++ b/src/arch/aarch64/device/cpu/mod.rs @@ -1,6 +1,6 @@ use core::fmt::{Result, Write}; -use crate::device::cpu::registers::{control_regs}; +use crate::device::cpu::registers::control_regs; pub mod registers; @@ -29,28 +29,15 @@ enum ImplementerID { } const IMPLEMENTERS: [&'static str; 12] = [ - "Unknown", - "Arm", - "Broadcom", - "Cavium", - "Digital", - "Infineon", - "Motorola", - "Nvidia", - "AMCC", - "Qualcomm", - "Marvell", - "Intel", + "Unknown", "Arm", "Broadcom", "Cavium", "Digital", "Infineon", "Motorola", "Nvidia", "AMCC", + "Qualcomm", "Marvell", "Intel", ]; - enum VariantID { Unknown, } -const VARIANTS: [&'static str; 1] = [ - "Unknown", -]; +const VARIANTS: [&'static str; 1] = ["Unknown"]; enum ArchitectureID { Unknown, @@ -63,16 +50,8 @@ enum ArchitectureID { V6, } -const ARCHITECTURES: [&'static str; 8] = [ - "Unknown", - "v4", - "v4T", - "v5", - "v5T", - "v5TE", - "v5TEJ", - "v6", -]; +const ARCHITECTURES: [&'static str; 8] = + ["Unknown", "v4", "v4T", "v5", "v5T", "v5TE", "v5TEJ", "v6"]; enum PartNumberID { Unknown, @@ -106,11 +85,7 @@ enum RevisionID { Thunder1_1, } -const REVISIONS: [&'static str; 3] = [ - "Unknown", - "Thunder-1.0", - "Thunder-1.1", -]; +const REVISIONS: [&'static str; 3] = ["Unknown", "Thunder-1.0", "Thunder-1.1"]; struct CpuInfo { implementer: &'static str, @@ -177,7 +152,7 @@ impl CpuInfo { _ => REVISIONS[RevisionID::Unknown as usize], }; val - }, + } _ => REVISIONS[RevisionID::Unknown as usize], }; diff --git a/src/arch/aarch64/device/generic_timer.rs b/src/arch/aarch64/device/generic_timer.rs index cadf690b14..0a0610e10f 100644 --- a/src/arch/aarch64/device/generic_timer.rs +++ b/src/arch/aarch64/device/generic_timer.rs @@ -1,19 +1,15 @@ +use crate::log::{debug, info}; use alloc::boxed::Box; -use crate::log::{info, debug}; +use crate::{ + arch::device::irqchip::IRQ_CHIP, context, context::timeout, + device::cpu::registers::control_regs, dtb::DTB_BINARY, init::device_tree::find_compatible_node, + interrupt::irq::trigger, time, +}; use alloc::vec::Vec; use byteorder::{ByteOrder, BE}; -use crate::arch::device::irqchip::IRQ_CHIP; -use crate::context::timeout; -use crate::device::cpu::registers::control_regs; -use crate::init::device_tree::find_compatible_node; -use crate::interrupt::irq::trigger; -use crate::time; -use crate::context; -use crate::dtb::DTB_BINARY; -use super::irqchip::InterruptHandler; -use super::irqchip::register_irq; +use super::irqchip::{register_irq, InterruptHandler}; bitflags! { struct TimerCtrlFlags: u32 { @@ -24,19 +20,28 @@ bitflags! { } pub unsafe fn init() { - let mut timer = GenericTimer{ clk_freq: 0, reload_count: 0}; + let mut timer = GenericTimer { + clk_freq: 0, + reload_count: 0, + }; timer.init(); let data = DTB_BINARY.get().unwrap(); let fdt = fdt::DeviceTree::new(data).unwrap(); if let Some(node) = find_compatible_node(&fdt, "arm,armv7-timer") { - let interrupts = node.properties().find(|p| p.name.contains("interrupts")).unwrap(); + let interrupts = node + .properties() + .find(|p| p.name.contains("interrupts")) + .unwrap(); let mut intr_data = Vec::new(); for chunk in interrupts.data.chunks(4) { let val = BE::read_u32(chunk); intr_data.push(val); } let mut ic_idx = IRQ_CHIP.irq_chip_list.root_idx; - if let Some(interrupt_parent) = node.properties().find(|p| p.name.contains("interrupt-parent")) { + if let Some(interrupt_parent) = node + .properties() + .find(|p| p.name.contains("interrupt-parent")) + { let phandle = BE::read_u32(interrupt_parent.data); let mut i = 0; while i < IRQ_CHIP.irq_chip_list.chips.len() { @@ -49,7 +54,10 @@ pub unsafe fn init() { } } //PHYS_NONSECURE_PPI only - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx].ic.irq_xlate(&intr_data, 1).unwrap(); + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(&intr_data, 1) + .unwrap(); info!("generic_timer virq = {}", virq); register_irq(virq as u32, Box::new(timer)); IRQ_CHIP.irq_enable(virq as u32); @@ -109,7 +117,6 @@ impl GenericTimer { impl InterruptHandler for GenericTimer { fn irq_handler(&mut self, irq: u32) { - self.clear_irq(); { *time::OFFSET.lock() += self.clk_freq as u128; @@ -119,7 +126,9 @@ impl InterruptHandler for GenericTimer { context::switch::tick(); - unsafe { trigger(irq); } + unsafe { + trigger(irq); + } self.reload_count(); } } diff --git a/src/arch/aarch64/device/irqchip/gic.rs b/src/arch/aarch64/device/irqchip/gic.rs index eeaeadaf1f..9635f54657 100644 --- a/src/arch/aarch64/device/irqchip/gic.rs +++ b/src/arch/aarch64/device/irqchip/gic.rs @@ -1,11 +1,16 @@ use core::ptr::{read_volatile, write_volatile}; -use fdt::{DeviceTree, Node}; use byteorder::{ByteOrder, BE}; +use fdt::{DeviceTree, Node}; -use crate::init::device_tree::find_compatible_node; -use crate::log::{info, debug}; -use syscall::{Result, error::{Error, EINVAL}}; +use crate::{ + init::device_tree::find_compatible_node, + log::{debug, info}, +}; +use syscall::{ + error::{Error, EINVAL}, + Result, +}; use super::{InterruptController, IrqDesc}; @@ -35,11 +40,13 @@ impl GenericInterruptController { ncpus: 0, nirqs: 0, }; - let gic_cpu_if = GicCpuIf { - address: 0, - }; + let gic_cpu_if = GicCpuIf { address: 0 }; - GenericInterruptController { gic_dist_if, gic_cpu_if, irq_range: (0, 0) } + GenericInterruptController { + gic_dist_if, + gic_cpu_if, + irq_range: (0, 0), + } } pub fn parse(fdt: &DeviceTree) -> Result<(usize, usize, usize, usize)> { if let Some(node) = find_compatible_node(fdt, "arm,cortex-a15-gic") { @@ -76,11 +83,18 @@ impl GenericInterruptController { } impl InterruptController for GenericInterruptController { - fn irq_init(&mut self, fdt: &DeviceTree, irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize) -> Result> { - let (dist_addr, dist_size, cpu_addr, cpu_size) = match GenericInterruptController::parse(fdt) { - Ok(regs) => regs, - Err(err) => return Err(err), - }; + fn irq_init( + &mut self, + fdt: &DeviceTree, + irq_desc: &mut [IrqDesc; 1024], + ic_idx: usize, + irq_idx: &mut usize, + ) -> Result> { + let (dist_addr, dist_size, cpu_addr, cpu_size) = + match GenericInterruptController::parse(fdt) { + Ok(regs) => regs, + Err(err) => return Err(err), + }; unsafe { self.gic_cpu_if.init(crate::PHYS_OFFSET + cpu_addr); @@ -92,7 +106,11 @@ impl InterruptController for GenericInterruptController { self.gic_cpu_if.write(GICC_PMR, 0xff); } let idx = *irq_idx; - let cnt = if self.gic_dist_if.nirqs > 1024 { 1024 } else { self.gic_dist_if.nirqs as usize }; + let cnt = if self.gic_dist_if.nirqs > 1024 { + 1024 + } else { + self.gic_dist_if.nirqs as usize + }; let mut i: usize = 0; //only support linear irq map now. while i < cnt && (idx + i < 1024) { @@ -145,9 +163,7 @@ impl InterruptController for GenericInterruptController { } } - fn irq_handler(&mut self, _irq: u32) { - - } + fn irq_handler(&mut self, _irq: u32) {} } pub struct GicDistIf { @@ -166,7 +182,10 @@ impl GicDistIf { let typer = self.read(GICD_TYPER); self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1; self.nirqs = ((typer & 0x1f) + 1) * 32; - info!("gic: Distributor supports {:?} CPUs and {:?} IRQs", self.ncpus, self.nirqs); + info!( + "gic: Distributor supports {:?} CPUs and {:?} IRQs", + self.ncpus, self.nirqs + ); // Set all SPIs to level triggered for irq in (32..self.nirqs).step_by(16) { diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs index af7298090b..d49360cb4d 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs @@ -1,16 +1,20 @@ +use alloc::{boxed::Box, vec::Vec}; use core::ptr::{read_volatile, write_volatile}; -use alloc::boxed::Box; -use alloc::vec::Vec; +use crate::arch::device::irqchip::IRQ_CHIP; use byteorder::{ByteOrder, BE}; use fdt::{DeviceTree, Node}; -use crate::arch::device::irqchip::IRQ_CHIP; -use crate::init::device_tree::find_compatible_node; -use crate::log::{info, debug, error}; -use syscall::{Result, error::{Error, EINVAL}}; +use crate::{ + init::device_tree::find_compatible_node, + log::{debug, error, info}, +}; +use syscall::{ + error::{Error, EINVAL}, + Result, +}; -use super::{InterruptController, IrqDesc, InterruptHandler}; +use super::{InterruptController, InterruptHandler, IrqDesc}; #[inline(always)] fn ffs(num: u32) -> u32 { @@ -39,7 +43,7 @@ fn ffs(num: u32) -> u32 { x >>= 1; r += 1; } - + r } @@ -59,10 +63,12 @@ pub struct Bcm2835ArmInterruptController { pub irq_range: (usize, usize), } - impl Bcm2835ArmInterruptController { pub fn new() -> Self { - Bcm2835ArmInterruptController { address: 0, irq_range: (0, 0) } + Bcm2835ArmInterruptController { + address: 0, + irq_range: (0, 0), + } } pub fn parse(fdt: &DeviceTree) -> Result<(usize, usize, Option)> { //TODO: try to parse dtb using stable library @@ -79,10 +85,16 @@ impl Bcm2835ArmInterruptController { let base = BE::read_u32(base); let size = BE::read_u32(size); let mut ret_virq = None; - - if let Some(interrupt_parent) = node.properties().find(|p| p.name.contains("interrupt-parent")) { + + if let Some(interrupt_parent) = node + .properties() + .find(|p| p.name.contains("interrupt-parent")) + { let phandle = BE::read_u32(interrupt_parent.data); - let interrupts = node.properties().find(|p| p.name.contains("interrupts")).unwrap(); + let interrupts = node + .properties() + .find(|p| p.name.contains("interrupts")) + .unwrap(); let mut intr_data = Vec::new(); for chunk in interrupts.data.chunks(4) { let val = BE::read_u32(chunk); @@ -99,7 +111,10 @@ impl Bcm2835ArmInterruptController { i += 1; } //PHYS_NONSECURE_PPI only - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx].ic.irq_xlate(&intr_data, 0).unwrap(); + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(&intr_data, 0) + .unwrap(); info!("bcm2835arm_ctrl virq = {}", virq); ret_virq = Some(virq); } @@ -127,7 +142,13 @@ impl Bcm2835ArmInterruptController { } impl InterruptController for Bcm2835ArmInterruptController { - fn irq_init(&mut self, fdt: &DeviceTree, irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize) -> Result> { + fn irq_init( + &mut self, + fdt: &DeviceTree, + irq_desc: &mut [IrqDesc; 1024], + ic_idx: usize, + irq_idx: &mut usize, + ) -> Result> { let (base, size, virq) = match Bcm2835ArmInterruptController::parse(fdt) { Ok((a, b, c)) => (a, b, c), Err(_) => return Err(Error::new(EINVAL)), @@ -161,8 +182,17 @@ impl InterruptController for Bcm2835ArmInterruptController { let sources = unsafe { self.read(PENDING_0) }; let pending_num = ffs(sources) - 1; let fast_irq = [ - 7 + 32, 9 + 32, 10 + 32, 18 + 32, 19 + 32, - 21 + 64, 22 + 64, 23 + 64, 24 + 64, 25 + 64, 30 + 64 + 7 + 32, + 9 + 32, + 10 + 32, + 18 + 32, + 19 + 32, + 21 + 64, + 22 + 64, + 23 + 64, + 24 + 64, + 25 + 64, + 30 + 64, ]; //fast irq @@ -172,66 +202,76 @@ impl InterruptController for Bcm2835ArmInterruptController { let pending_num = ffs(sources & 0x3ff) - 1; match pending_num { - num@0..=7 => { - return num - }, + num @ 0..=7 => return num, 8 => { let sources1 = unsafe { self.read(PENDING_1) }; let irq_0_31 = ffs(sources1) - 1; return irq_0_31 + 32; - }, + } 9 => { let sources2 = unsafe { self.read(PENDING_2) }; let irq_32_63 = ffs(sources2) - 1; return irq_32_63 + 64; - }, + } num => { - error!("unexpected irq pending in BASIC PENDING: 0x{}, sources = 0x{:08x}", num, sources); + error!( + "unexpected irq pending in BASIC PENDING: 0x{}, sources = 0x{:08x}", + num, sources + ); return num; } } } - fn irq_eoi(&mut self, _irq_num: u32) { - - } + fn irq_eoi(&mut self, _irq_num: u32) {} fn irq_enable(&mut self, irq_num: u32) { debug!("bcm2835 enable {} {}", irq_num, irq_num & 0x1f); match irq_num { - num @0..=31 => { + num @ 0..=31 => { let val = 1 << num; - unsafe { self.write(ENABLE_0, val); } - }, - num @32..=63 => { + unsafe { + self.write(ENABLE_0, val); + } + } + num @ 32..=63 => { let val = 1 << (num & 0x1f); - unsafe { self.write(ENABLE_1, val); } - }, - num @64..=95 => { + unsafe { + self.write(ENABLE_1, val); + } + } + num @ 64..=95 => { let val = 1 << (num & 0x1f); - unsafe { self.write(ENABLE_2, val); } - }, + unsafe { + self.write(ENABLE_2, val); + } + } _ => return, } } fn irq_disable(&mut self, irq_num: u32) { match irq_num { - num @0..=31 => { + num @ 0..=31 => { let val = 1 << num; - unsafe { self.write(DISABLE_0, val); } - }, - num @32..=63 => { + unsafe { + self.write(DISABLE_0, val); + } + } + num @ 32..=63 => { let val = 1 << (num & 0x1f); - unsafe { self.write(DISABLE_1, val); } - }, - num @64..=95 => { + unsafe { + self.write(DISABLE_1, val); + } + } + num @ 64..=95 => { let val = 1 << (num & 0x1f); - unsafe { self.write(DISABLE_2, val); } - }, + unsafe { + self.write(DISABLE_2, val); + } + } _ => return, } - } fn irq_xlate(&mut self, irq_data: &[u32], idx: usize) -> Result { //assert interrupt-cells == 0x2 @@ -260,7 +300,6 @@ impl InterruptController for Bcm2835ArmInterruptController { } fn irq_handler(&mut self, _irq: u32) { - unsafe { let irq = self.irq_ack(); if let Some(virq) = self.irq_to_virq(irq) && virq < 1024 { @@ -277,7 +316,6 @@ impl InterruptController for Bcm2835ArmInterruptController { impl InterruptHandler for Bcm2835ArmInterruptController { fn irq_handler(&mut self, _irq: u32) { - unsafe { let irq = self.irq_ack(); if let Some(virq) = self.irq_to_virq(irq) && virq < 1024 { diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs index 430cb1c709..f8488ef369 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs @@ -1,11 +1,19 @@ -use core::{ptr::{read_volatile, write_volatile}, arch::asm}; +use core::{ + arch::asm, + ptr::{read_volatile, write_volatile}, +}; use byteorder::{ByteOrder, BE}; use fdt::{DeviceTree, Node}; -use crate::init::device_tree::find_compatible_node; -use crate::log::{info, debug}; -use syscall::{Result, error::{Error, EINVAL}}; +use crate::{ + init::device_tree::find_compatible_node, + log::{debug, info}, +}; +use syscall::{ + error::{Error, EINVAL}, + Result, +}; use super::{InterruptController, IrqDesc}; @@ -55,7 +63,7 @@ fn ffs(num: u32) -> u32 { x >>= 1; r += 1; } - + r } @@ -65,10 +73,13 @@ pub struct Bcm2836ArmInterruptController { pub active_cpu: u32, } - impl Bcm2836ArmInterruptController { pub fn new() -> Self { - Bcm2836ArmInterruptController { address: 0, irq_range: (0, 0), active_cpu: 0 } + Bcm2836ArmInterruptController { + address: 0, + irq_range: (0, 0), + active_cpu: 0, + } } pub fn parse(fdt: &DeviceTree) -> Result<(usize, usize)> { //TODO: try to parse dtb using stable library @@ -84,7 +95,7 @@ impl Bcm2836ArmInterruptController { let (base, size) = reg.data.split_at(4); let base = BE::read_u32(base); let size = BE::read_u32(size); - + Ok((base as usize, size as usize)) } @@ -111,7 +122,13 @@ impl Bcm2836ArmInterruptController { } impl InterruptController for Bcm2836ArmInterruptController { - fn irq_init(&mut self, fdt: &DeviceTree, irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize) -> Result> { + fn irq_init( + &mut self, + fdt: &DeviceTree, + irq_desc: &mut [IrqDesc; 1024], + ic_idx: usize, + irq_idx: &mut usize, + ) -> Result> { let (base, size) = match Bcm2836ArmInterruptController::parse(fdt) { Ok((a, b)) => (a, b), Err(_) => return Err(Error::new(EINVAL)), @@ -145,15 +162,15 @@ impl InterruptController for Bcm2836ArmInterruptController { fn irq_ack(&mut self) -> u32 { let mut cpuid: usize = 0; - unsafe { asm!("mrs {}, mpidr_el1", out(reg) cpuid); } + unsafe { + asm!("mrs {}, mpidr_el1", out(reg) cpuid); + } let cpu = cpuid as u32 & 0x3; let sources: u32 = unsafe { self.read(LOCAL_IRQ_PENDING + 4 * cpu) }; ffs(sources) - 1 } - fn irq_eoi(&mut self, _irq_num: u32) { - - } + fn irq_eoi(&mut self, _irq_num: u32) {} fn irq_enable(&mut self, irq_num: u32) { match irq_num { @@ -167,7 +184,7 @@ impl InterruptController for Bcm2836ArmInterruptController { }, LOCAL_IRQ_GPU_FAST => { //GPU IRQ always enable - }, + } _ => { //ignore } @@ -178,7 +195,9 @@ impl InterruptController for Bcm2836ArmInterruptController { match irq_num { LOCAL_IRQ_CNTPNSIRQ => unsafe { let mut cpuid: usize = 0; - unsafe { asm!("mrs {}, mpidr_el1", out(reg) cpuid); } + unsafe { + asm!("mrs {}, mpidr_el1", out(reg) cpuid); + } let mut cpu = cpuid as u32 & 0x3; let mut reg_val = self.read(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu); reg_val &= !0x2; @@ -186,7 +205,7 @@ impl InterruptController for Bcm2836ArmInterruptController { }, LOCAL_IRQ_GPU_FAST => { //GPU IRQ always enable - }, + } _ => { //ignore } @@ -213,7 +232,5 @@ impl InterruptController for Bcm2836ArmInterruptController { } } - fn irq_handler(&mut self, irq: u32) { - - } + fn irq_handler(&mut self, irq: u32) {} } diff --git a/src/arch/aarch64/device/irqchip/mod.rs b/src/arch/aarch64/device/irqchip/mod.rs index c3fd464dc1..c532a3d0b3 100644 --- a/src/arch/aarch64/device/irqchip/mod.rs +++ b/src/arch/aarch64/device/irqchip/mod.rs @@ -1,10 +1,16 @@ -use alloc::{boxed::Box, vec::Vec, string::{String, ToString}}; +use alloc::{ + boxed::Box, + string::{String, ToString}, + vec::Vec, +}; use byteorder::{ByteOrder, BE}; -use syscall::Result; use fdt::DeviceTree; +use syscall::Result; -use crate::log::{debug, error}; -use crate::init::device_tree::travel_interrupt_ctrl; +use crate::{ + init::device_tree::travel_interrupt_ctrl, + log::{debug, error}, +}; mod gic; mod irq_bcm2835; @@ -18,7 +24,13 @@ pub const IRQ_TYPE_LEVEL_HIGH: u32 = 4; pub const IRQ_TYPE_LEVEL_LOW: u32 = 8; pub trait InterruptController { - fn irq_init(&mut self, fdt: &DeviceTree, irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize) -> Result>; + fn irq_init( + &mut self, + fdt: &DeviceTree, + irq_desc: &mut [IrqDesc; 1024], + ic_idx: usize, + irq_idx: &mut usize, + ) -> Result>; fn irq_ack(&mut self) -> u32; fn irq_eoi(&mut self, irq_num: u32); fn irq_enable(&mut self, irq_num: u32); @@ -38,7 +50,7 @@ pub struct IrqChipItem { pub parent_phandle: Option, pub intr_cell_size: u32, pub parent: Option, //parent idx in chiplist - pub childs: Vec, //child idx in chiplist + pub childs: Vec, //child idx in chiplist pub interrupts: Vec, pub ic: Box, } @@ -51,10 +63,10 @@ pub struct IrqChipList { pub struct IrqDescItem { pub idx: usize, - pub ic_idx: usize, //ic idx in irq chip list + pub ic_idx: usize, //ic idx in irq chip list pub child_ic_idx: Option, //ic idx in irq chip list - pub ic_irq: u32, //hwirq in ic - pub used: bool, + pub ic_irq: u32, //hwirq in ic + pub used: bool, } pub struct IrqDesc { @@ -66,27 +78,50 @@ impl IrqChipList { fn init_inner1(&mut self, fdt: &fdt::DeviceTree) { let root_node = fdt.nodes().nth(0).unwrap(); let mut idx = 0; - let intr = root_node.properties().find(|p| p.name.contains("interrupt-parent")).unwrap(); + let intr = root_node + .properties() + .find(|p| p.name.contains("interrupt-parent")) + .unwrap(); let root_intr_parent = BE::read_u32(&intr.data); debug!("root parent = 0x{:08x}", root_intr_parent); self.root_phandle = root_intr_parent; for node in fdt.nodes() { - if node.properties().find(|p| p.name.contains("interrupt-controller")).is_some() { - let compatible = node.properties().find(|p| p.name.contains("compatible")).unwrap(); - let phandle = node.properties().find(|p| p.name.contains("phandle")).unwrap(); - let intr_cells = node.properties().find(|p| p.name.contains("#interrupt-cells")).unwrap(); - let _intr = node.properties().find(|p| p.name.contains("interrupt-parent")); + if node + .properties() + .find(|p| p.name.contains("interrupt-controller")) + .is_some() + { + let compatible = node + .properties() + .find(|p| p.name.contains("compatible")) + .unwrap(); + let phandle = node + .properties() + .find(|p| p.name.contains("phandle")) + .unwrap(); + let intr_cells = node + .properties() + .find(|p| p.name.contains("#interrupt-cells")) + .unwrap(); + let _intr = node + .properties() + .find(|p| p.name.contains("interrupt-parent")); let _intr_data = node.properties().find(|p| p.name.contains("interrupts")); let s = core::str::from_utf8(compatible.data).unwrap(); - debug!("{}, compatible = {}, #interrupt-cells = 0x{:08x}, phandle = 0x{:08x}", node.name, s, BE::read_u32(intr_cells.data), - BE::read_u32(phandle.data)); + debug!( + "{}, compatible = {}, #interrupt-cells = 0x{:08x}, phandle = 0x{:08x}", + node.name, + s, + BE::read_u32(intr_cells.data), + BE::read_u32(phandle.data) + ); let mut item = IrqChipItem { compatible: s.to_string(), phandle: BE::read_u32(phandle.data), parent_phandle: None, - intr_cell_size: BE::read_u32(intr_cells.data), + intr_cell_size: BE::read_u32(intr_cells.data), parent: None, childs: Vec::new(), interrupts: Vec::new(), @@ -111,7 +146,6 @@ impl IrqChipList { self.chips.push(item); idx += 1; - } } } @@ -135,7 +169,6 @@ impl IrqChipList { } x += 1; } - } fn init_inner3(&mut self, fdt: &fdt::DeviceTree, irq_desc: &mut [IrqDesc; 1024]) { @@ -146,13 +179,14 @@ impl IrqChipList { while !queue.is_empty() { let cur_idx = queue.pop().unwrap(); queue.extend_from_slice(&self.chips[cur_idx].childs); - let virq = self.chips[cur_idx].ic.irq_init(fdt, irq_desc, cur_idx, &mut irq_idx); + let virq = self.chips[cur_idx] + .ic + .irq_init(fdt, irq_desc, cur_idx, &mut irq_idx); if let Ok(Some(virq)) = virq { irq_desc[virq].basic.child_ic_idx = Some(cur_idx); } } } - } pub struct IrqChipCore { @@ -164,7 +198,9 @@ pub struct IrqChipCore { impl IrqChipCore { pub fn irq_ack(&mut self) -> u32 { - self.irq_chip_list.chips[self.irq_chip_list.root_idx].ic.irq_ack() + self.irq_chip_list.chips[self.irq_chip_list.root_idx] + .ic + .irq_ack() } pub fn irq_eoi(&mut self, virq: u32) { @@ -192,7 +228,9 @@ impl IrqChipCore { } pub fn irq_to_virq(&mut self, hwirq: u32) -> Option { - self.irq_chip_list.chips[self.irq_chip_list.root_idx].ic.irq_to_virq(hwirq) + self.irq_chip_list.chips[self.irq_chip_list.root_idx] + .ic + .irq_to_virq(hwirq) } pub fn init(&mut self, fdt: &DeviceTree) { @@ -202,7 +240,6 @@ impl IrqChipCore { self.irq_chip_list.init_inner1(fdt); self.irq_chip_list.init_inner2(); self.irq_chip_list.init_inner3(fdt, &mut self.irq_desc); - } pub fn new_ic(ic_str: &str) -> Option> { @@ -229,7 +266,7 @@ const INIT_IRQ_DESC: IrqDesc = IrqDesc { }, handler: INIT_HANDLER, }; -pub static mut IRQ_CHIP: IrqChipCore = IrqChipCore { +pub static mut IRQ_CHIP: IrqChipCore = IrqChipCore { irq_chip_list: IrqChipList { chips: Vec::new(), root_phandle: 0, @@ -239,7 +276,6 @@ pub static mut IRQ_CHIP: IrqChipCore = IrqChipCore { irq_desc: [INIT_IRQ_DESC; 1024], }; - pub fn init(fdt: &DeviceTree) { travel_interrupt_ctrl(fdt); unsafe { @@ -250,13 +286,13 @@ pub fn init(fdt: &DeviceTree) { pub fn register_irq(virq: u32, handler: Box) { if virq >= 1024 { error!("irq {} exceed 1024!!!", virq); - return ; + return; } unsafe { if IRQ_CHIP.irq_desc[virq as usize].handler.is_some() { error!("irq {} has already been registered!", virq); - return ; + return; } IRQ_CHIP.irq_desc[virq as usize].handler = Some(handler); diff --git a/src/arch/aarch64/device/mod.rs b/src/arch/aarch64/device/mod.rs index 9b3de8edf4..b6ffd59b4f 100644 --- a/src/arch/aarch64/device/mod.rs +++ b/src/arch/aarch64/device/mod.rs @@ -1,15 +1,17 @@ use core::arch::asm; -use crate::memory::Frame; -use crate::paging::{KernelMapper, PhysicalAddress, Page, PageFlags, VirtualAddress}; -use crate::dtb::DTB_BINARY; -use crate::log::info; +use crate::{ + dtb::DTB_BINARY, + log::info, + memory::Frame, + paging::{KernelMapper, Page, PageFlags, PhysicalAddress, VirtualAddress}, +}; pub mod cpu; -pub mod irqchip; pub mod generic_timer; -pub mod serial; +pub mod irqchip; pub mod rtc; +pub mod serial; pub mod uart_pl011; pub unsafe fn init() { @@ -28,8 +30,7 @@ pub unsafe fn init_noncore() { rtc::init(); } -pub unsafe fn init_ap() { -} +pub unsafe fn init_ap() {} //map physical addr X to virtual addr PHYS_OFFSET + X pub unsafe fn io_mmap(addr: usize, io_size: usize) { @@ -38,13 +39,18 @@ pub unsafe fn io_mmap(addr: usize, io_size: usize) { let start_frame = Frame::containing_address(PhysicalAddress::new(addr)); let end_frame = Frame::containing_address(PhysicalAddress::new(addr + io_size - 1)); for frame in Frame::range_inclusive(start_frame, end_frame) { - let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::PHYS_OFFSET)); + let page = Page::containing_address(VirtualAddress::new( + frame.start_address().data() + crate::PHYS_OFFSET, + )); mapper .get_mut() .expect("failed to access KernelMapper for mapping GIC distributor") - .map_phys(page.start_address(), frame.start_address(), PageFlags::new().write(true)) + .map_phys( + page.start_address(), + frame.start_address(), + PageFlags::new().write(true), + ) .expect("failed to map GIC distributor") .flush(); } - } diff --git a/src/arch/aarch64/device/rtc.rs b/src/arch/aarch64/device/rtc.rs index b47d7bf735..075fd07932 100644 --- a/src/arch/aarch64/device/rtc.rs +++ b/src/arch/aarch64/device/rtc.rs @@ -1,8 +1,10 @@ use core::ptr::{read_volatile, write_volatile}; -use crate::memory::Frame; -use crate::paging::{KernelMapper, PhysicalAddress, Page, PageFlags, TableKind, VirtualAddress}; -use crate::time; +use crate::{ + memory::Frame, + paging::{KernelMapper, Page, PageFlags, PhysicalAddress, TableKind, VirtualAddress}, + time, +}; static RTC_DR: u32 = 0x000; static RTC_MR: u32 = 0x004; @@ -13,9 +15,7 @@ static RTC_RIS: u32 = 0x014; static RTC_MIS: u32 = 0x018; static RTC_ICR: u32 = 0x01c; -static mut PL031_RTC: Pl031rtc = Pl031rtc { - address: 0, -}; +static mut PL031_RTC: Pl031rtc = Pl031rtc { address: 0 }; pub unsafe fn init() { PL031_RTC.init(); @@ -34,11 +34,17 @@ impl Pl031rtc { let end_frame = Frame::containing_address(PhysicalAddress::new(0x09010000 + 0x1000 - 1)); for frame in Frame::range_inclusive(start_frame, end_frame) { - let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::PHYS_OFFSET)); + let page = Page::containing_address(VirtualAddress::new( + frame.start_address().data() + crate::PHYS_OFFSET, + )); mapper .get_mut() .expect("failed to access KernelMapper for mapping RTC") - .map_phys(page.start_address(), frame.start_address(), PageFlags::new().write(true)) + .map_phys( + page.start_address(), + frame.start_address(), + PageFlags::new().write(true), + ) .expect("failed to map RTC") .flush(); } diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs index ed8326fdf7..7220fb0ed1 100644 --- a/src/arch/aarch64/device/serial.rs +++ b/src/arch/aarch64/device/serial.rs @@ -1,31 +1,33 @@ use alloc::boxed::Box; use spin::Mutex; -use crate::{device::uart_pl011::SerialPort, interrupt::irq::trigger}; -use crate::init::device_tree; -use crate::log::{info, debug}; +use crate::{ + device::uart_pl011::SerialPort, + init::device_tree, + interrupt::irq::trigger, + log::{debug, info}, +}; -use super::irqchip::{register_irq, IRQ_CHIP, InterruptHandler}; -use crate::dtb::DTB_BINARY; -use byteorder::{ByteOrder, BE}; -use crate::init::device_tree::find_compatible_node; +use super::irqchip::{register_irq, InterruptHandler, IRQ_CHIP}; +use crate::{dtb::DTB_BINARY, init::device_tree::find_compatible_node}; use alloc::vec::Vec; +use byteorder::{ByteOrder, BE}; pub static COM1: Mutex> = Mutex::new(None); -pub struct Com1Irq { -} +pub struct Com1Irq {} impl InterruptHandler for Com1Irq { fn irq_handler(&mut self, irq: u32) { if let Some(ref mut serial_port) = *COM1.lock() { serial_port.receive(); }; - unsafe { trigger(irq); } + unsafe { + trigger(irq); + } } } - pub unsafe fn init_early(dtb_base: usize, dtb_size: usize) { if COM1.lock().is_some() { // Hardcoded UART @@ -47,14 +49,20 @@ pub unsafe fn init() { let data = DTB_BINARY.get().unwrap(); let fdt = fdt::DeviceTree::new(data).unwrap(); if let Some(node) = find_compatible_node(&fdt, "arm,pl011") { - let interrupts = node.properties().find(|p| p.name.contains("interrupts")).unwrap(); + let interrupts = node + .properties() + .find(|p| p.name.contains("interrupts")) + .unwrap(); let mut intr_data = Vec::new(); for chunk in interrupts.data.chunks(4) { let val = BE::read_u32(chunk); intr_data.push(val); } let mut ic_idx = IRQ_CHIP.irq_chip_list.root_idx; - if let Some(interrupt_parent) = node.properties().find(|p| p.name.contains("interrupt-parent")) { + if let Some(interrupt_parent) = node + .properties() + .find(|p| p.name.contains("interrupt-parent")) + { let phandle = BE::read_u32(interrupt_parent.data); let mut i = 0; while i < IRQ_CHIP.irq_chip_list.chips.len() { @@ -66,7 +74,10 @@ pub unsafe fn init() { i += 1; } } - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx].ic.irq_xlate(&intr_data, 0).unwrap(); + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(&intr_data, 0) + .unwrap(); info!("serial_port virq = {}", virq); register_irq(virq as u32, Box::new(Com1Irq {})); IRQ_CHIP.irq_enable(virq as u32); diff --git a/src/arch/aarch64/device/uart_pl011.rs b/src/arch/aarch64/device/uart_pl011.rs index 79d67fbb86..61315e9e21 100644 --- a/src/arch/aarch64/device/uart_pl011.rs +++ b/src/arch/aarch64/device/uart_pl011.rs @@ -135,13 +135,14 @@ impl SerialPort { } pub fn write_reg(&self, register: u8, data: u32) { - unsafe { ptr::write_volatile((self.base + register as usize) as *mut u32, data); } + unsafe { + ptr::write_volatile((self.base + register as usize) as *mut u32, data); + } } pub fn init(&mut self, with_irq: bool) { - if self.skip_init { - return ; + return; } //Disable UART first @@ -175,7 +176,7 @@ impl SerialPort { } pub fn drain_fifo(&mut self) { - for _ in 0..self.fifo_size*2 { + for _ in 0..self.fifo_size * 2 { if self.line_sts().contains(UartFrFlags::RXFE) { break; } @@ -187,7 +188,6 @@ impl SerialPort { let mut flags = self.intr_stats(); let chk_flags = UartRisFlags::RTIS | UartRisFlags::RXIS; while (flags & chk_flags).bits != 0 { - if self.cts_event_walkaround { self.write_reg(self.intr_clr_reg, 0x00); let _ = self.read_reg(self.intr_clr_reg); @@ -213,7 +213,7 @@ impl SerialPort { } pub fn send(&mut self, data: u8) { - while ! self.line_sts().contains(UartFrFlags::TXFE) {} + while !self.line_sts().contains(UartFrFlags::TXFE) {} self.write_reg(self.data_reg, data as u32); } @@ -227,7 +227,6 @@ impl SerialPort { } pub fn enable_irq(&mut self) { - self.clear_all_irqs(); self.drain_fifo(); diff --git a/src/arch/aarch64/init/device_tree/mod.rs b/src/arch/aarch64/init/device_tree/mod.rs index 526be6e694..8836879547 100644 --- a/src/arch/aarch64/init/device_tree/mod.rs +++ b/src/arch/aarch64/init/device_tree/mod.rs @@ -1,11 +1,13 @@ -extern crate fdt; extern crate byteorder; +extern crate fdt; -use crate::log::{info, debug}; -use fdt::Node; -use core::slice; -use crate::memory::MemoryArea; use self::byteorder::{ByteOrder, BE}; +use crate::{ + log::{debug, info}, + memory::MemoryArea, +}; +use core::slice; +use fdt::Node; pub static mut MEMORY_MAP: [MemoryArea; 512] = [MemoryArea { base_addr: 0, @@ -16,29 +18,61 @@ pub static mut MEMORY_MAP: [MemoryArea; 512] = [MemoryArea { pub fn root_cell_sz(dt: &fdt::DeviceTree) -> Option<(u32, u32)> { let root_node = dt.nodes().nth(0).unwrap(); - let address_cells = root_node.properties().find(|p| p.name.contains("#address-cells")).unwrap(); - let size_cells = root_node.properties().find(|p| p.name.contains("#size-cells")).unwrap(); + let address_cells = root_node + .properties() + .find(|p| p.name.contains("#address-cells")) + .unwrap(); + let size_cells = root_node + .properties() + .find(|p| p.name.contains("#size-cells")) + .unwrap(); - Some((BE::read_u32(&size_cells.data), BE::read_u32(&size_cells.data))) + Some(( + BE::read_u32(&size_cells.data), + BE::read_u32(&size_cells.data), + )) } pub fn travel_interrupt_ctrl(fdt: &fdt::DeviceTree) { let root_node = fdt.nodes().nth(0).unwrap(); - let intr = root_node.properties().find(|p| p.name.contains("interrupt-parent")).unwrap(); + let intr = root_node + .properties() + .find(|p| p.name.contains("interrupt-parent")) + .unwrap(); let root_intr_parent = BE::read_u32(&intr.data); debug!("root parent = 0x{:08x}", root_intr_parent); for node in fdt.nodes() { - if node.properties().find(|p| p.name.contains("interrupt-controller")).is_some() { - let compatible = node.properties().find(|p| p.name.contains("compatible")).unwrap(); - let phandle = node.properties().find(|p| p.name.contains("phandle")).unwrap(); - let intr_cells = node.properties().find(|p| p.name.contains("#interrupt-cells")).unwrap(); - let _intr = node.properties().find(|p| p.name.contains("interrupt-parent")); + if node + .properties() + .find(|p| p.name.contains("interrupt-controller")) + .is_some() + { + let compatible = node + .properties() + .find(|p| p.name.contains("compatible")) + .unwrap(); + let phandle = node + .properties() + .find(|p| p.name.contains("phandle")) + .unwrap(); + let intr_cells = node + .properties() + .find(|p| p.name.contains("#interrupt-cells")) + .unwrap(); + let _intr = node + .properties() + .find(|p| p.name.contains("interrupt-parent")); let _intr_data = node.properties().find(|p| p.name.contains("interrupts")); let s = core::str::from_utf8(compatible.data).unwrap(); - debug!("{}, compatible = {}, #interrupt-cells = 0x{:08x}, phandle = 0x{:08x}", node.name, s, BE::read_u32(intr_cells.data), - BE::read_u32(phandle.data)); + debug!( + "{}, compatible = {}, #interrupt-cells = 0x{:08x}, phandle = 0x{:08x}", + node.name, + s, + BE::read_u32(intr_cells.data), + BE::read_u32(phandle.data) + ); if let Some(intr) = _intr { if let Some(intr_data) = _intr_data { debug!("interrupt-parent = 0x{:08x}", BE::read_u32(intr.data)); @@ -53,10 +87,17 @@ pub fn travel_interrupt_ctrl(fdt: &fdt::DeviceTree) { } } -fn memory_ranges(dt: &fdt::DeviceTree, address_cells: usize, size_cells: usize, ranges: &mut [(usize, usize); 10]) -> usize { - +fn memory_ranges( + dt: &fdt::DeviceTree, + address_cells: usize, + size_cells: usize, + ranges: &mut [(usize, usize); 10], +) -> usize { let (memory_node, _memory_cells) = dt.find_node("/memory").unwrap(); - let reg = memory_node.properties().find(|p| p.name.contains("reg")).unwrap(); + let reg = memory_node + .properties() + .find(|p| p.name.contains("reg")) + .unwrap(); let chunk_sz = (address_cells + size_cells) * 4; let chunk_count = (reg.data.len() / chunk_sz); let mut index = 0; @@ -79,8 +120,12 @@ fn memory_ranges(dt: &fdt::DeviceTree, address_cells: usize, size_cells: usize, index } -fn dev_memory_ranges(dt: &fdt::DeviceTree, address_cells: usize, size_cells: usize, ranges: &mut [(usize, usize); 10]) -> usize { - +fn dev_memory_ranges( + dt: &fdt::DeviceTree, + address_cells: usize, + size_cells: usize, + ranges: &mut [(usize, usize); 10], +) -> usize { // work around for qemu-arm64 // dev mem: 128MB - 1GB, see https://github.com/qemu/qemu/blob/master/hw/arm/virt.c for details let root_node = dt.nodes().nth(0).unwrap(); @@ -100,7 +145,10 @@ fn dev_memory_ranges(dt: &fdt::DeviceTree, address_cells: usize, size_cells: usi } let (memory_node, _memory_cells) = dt.find_node("/soc").unwrap(); - let reg = memory_node.properties().find(|p| p.name.contains("ranges")).unwrap(); + let reg = memory_node + .properties() + .find(|p| p.name.contains("ranges")) + .unwrap(); let chunk_sz = (address_cells * 2 + size_cells) * 4; let chunk_count = (reg.data.len() / chunk_sz); let mut index = 0; @@ -137,7 +185,10 @@ fn dev_memory_ranges(dt: &fdt::DeviceTree, address_cells: usize, size_cells: usi return 0; } }; - debug!("dev mem 0x{:08x} 0x{:08x} 0x{:08x}", child_bus_addr, parent_bus_addr, addr_size); + debug!( + "dev mem 0x{:08x} 0x{:08x} 0x{:08x}", + child_bus_addr, parent_bus_addr, addr_size + ); ranges[index] = (parent_bus_addr as usize, addr_size as usize); index += 1; @@ -150,17 +201,33 @@ pub fn diag_uart_range(dtb_base: usize, dtb_size: usize) -> Option<(usize, usize let dt = fdt::DeviceTree::new(data).unwrap(); let (chosen_node, _chosen_cells) = dt.find_node("/chosen").unwrap(); - let stdout_path = chosen_node.properties().find(|p| p.name.contains("stdout-path")).unwrap(); - let uart_node_name = core::str::from_utf8(stdout_path.data).unwrap() + let stdout_path = chosen_node + .properties() + .find(|p| p.name.contains("stdout-path")) + .unwrap(); + let uart_node_name = core::str::from_utf8(stdout_path.data) + .unwrap() .split('/') .nth(1)? .trim_end(); let len = uart_node_name.len(); - let uart_node_name = &uart_node_name[0..len-1]; - let uart_node = dt.nodes().find(|n| n.name.contains(uart_node_name)).unwrap(); - let skip_init = uart_node.properties().find(|p| p.name.contains("skip-init")).is_some(); - let cts_event_walkaround = uart_node.properties().find(|p| p.name.contains("cts-event-walkaround")).is_some(); - let reg = uart_node.properties().find(|p| p.name.contains("reg")).unwrap(); + let uart_node_name = &uart_node_name[0..len - 1]; + let uart_node = dt + .nodes() + .find(|n| n.name.contains(uart_node_name)) + .unwrap(); + let skip_init = uart_node + .properties() + .find(|p| p.name.contains("skip-init")) + .is_some(); + let cts_event_walkaround = uart_node + .properties() + .find(|p| p.name.contains("cts-event-walkaround")) + .is_some(); + let reg = uart_node + .properties() + .find(|p| p.name.contains("reg")) + .unwrap(); let (address_cells, size_cells) = root_cell_sz(&dt).unwrap(); let chunk_sz = (address_cells + size_cells) * 4; @@ -188,7 +255,10 @@ fn compatible_node_present<'a>(dt: &fdt::DeviceTree<'a>, compat_string: &str) -> false } -pub fn find_compatible_node<'a>(dt: &'a fdt::DeviceTree<'a>, compat_string: &str) -> Option> { +pub fn find_compatible_node<'a>( + dt: &'a fdt::DeviceTree<'a>, + compat_string: &str, +) -> Option> { for node in dt.nodes() { if let Some(compatible) = node.properties().find(|p| p.name.contains("compatible")) { let s = core::str::from_utf8(compatible.data).unwrap(); @@ -205,10 +275,14 @@ pub fn fill_env_data(dtb_base: usize, dtb_size: usize, env_base: usize) -> usize let dt = fdt::DeviceTree::new(data).unwrap(); let (chosen_node, _chosen_cells) = dt.find_node("/chosen").unwrap(); - if let Some(bootargs) = chosen_node.properties().find(|p| p.name.contains("bootargs")) { + if let Some(bootargs) = chosen_node + .properties() + .find(|p| p.name.contains("bootargs")) + { let bootargs_len = bootargs.data.len(); - let env_base_slice = unsafe { slice::from_raw_parts_mut(env_base as *mut u8, bootargs_len) }; + let env_base_slice = + unsafe { slice::from_raw_parts_mut(env_base as *mut u8, bootargs_len) }; env_base_slice[..bootargs_len].clone_from_slice(bootargs.data); bootargs_len @@ -222,11 +296,16 @@ pub fn fill_memory_map(dtb_base: usize, dtb_size: usize) { let dt = fdt::DeviceTree::new(data).unwrap(); let (address_cells, size_cells) = root_cell_sz(&dt).unwrap(); - let mut ranges: [(usize, usize); 10] = [(0,0); 10]; + let mut ranges: [(usize, usize); 10] = [(0, 0); 10]; - //in uefi boot mode, ignore memory node, just read the device memory range + //in uefi boot mode, ignore memory node, just read the device memory range //let nranges = memory_ranges(&dt, address_cells as usize, size_cells as usize, &mut ranges); - let nranges = dev_memory_ranges(&dt, address_cells as usize, size_cells as usize, &mut ranges); + let nranges = dev_memory_ranges( + &dt, + address_cells as usize, + size_cells as usize, + &mut ranges, + ); for index in 0..nranges { let (base, size) = ranges[index]; diff --git a/src/arch/aarch64/interrupt/exception.rs b/src/arch/aarch64/interrupt/exception.rs index 4167fb2953..5c92846ec9 100644 --- a/src/arch/aarch64/interrupt/exception.rs +++ b/src/arch/aarch64/interrupt/exception.rs @@ -1,13 +1,12 @@ use rmm::VirtualAddress; -use crate::memory::{ArchIntCtx, GenericPfFlags}; use crate::{ + exception_stack, interrupt::stack_trace, + memory::{ArchIntCtx, GenericPfFlags}, syscall, syscall::flag::*, - with_exception_stack, - exception_stack, }; use super::InterruptStack; @@ -32,7 +31,12 @@ unsafe fn far_el1() -> usize { ret } -unsafe fn instr_data_abort_inner(stack: &mut InterruptStack, from_user: bool, instr_not_data: bool, from: &str) -> bool { +unsafe fn instr_data_abort_inner( + stack: &mut InterruptStack, + from_user: bool, + instr_not_data: bool, + from: &str, +) -> bool { let iss = iss(stack.iret.esr_el1); let fsc = iss & 0x3F; //dbg!(fsc); @@ -46,7 +50,10 @@ unsafe fn instr_data_abort_inner(stack: &mut InterruptStack, from_user: bool, in // TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64 // doesn't appear to require that flag to be set if the read alone would trigger a fault. - flags.set(GenericPfFlags::INVOLVED_WRITE, write_not_read_if_data && !instr_not_data); + flags.set( + GenericPfFlags::INVOLVED_WRITE, + write_not_read_if_data && !instr_not_data, + ); flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data); flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user); @@ -74,7 +81,12 @@ unsafe fn cntvct_el0() -> usize { ret } -unsafe fn instr_trapped_msr_mrs_inner(stack: &mut InterruptStack, from_user: bool, instr_not_data: bool, from: &str) -> bool { +unsafe fn instr_trapped_msr_mrs_inner( + stack: &mut InterruptStack, + from_user: bool, + instr_not_data: bool, + from: &str, +) -> bool { let iss = iss(stack.iret.esr_el1); let res0 = (iss & 0x1C0_0000) >> 22; let op0 = (iss & 0x030_0000) >> 20; @@ -99,7 +111,7 @@ unsafe fn instr_trapped_msr_mrs_inner(stack: &mut InterruptStack, from_user: boo stack.store_reg(rt as usize, reg_val); //skip faulting instruction, A64 instructions are always 32-bits stack.iret.elr_el1 += 4; - return true + return true; } //MRS , CNTPCT_EL0 (0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => { @@ -107,7 +119,7 @@ unsafe fn instr_trapped_msr_mrs_inner(stack: &mut InterruptStack, from_user: boo stack.store_reg(rt as usize, reg_val); //skip faulting instruction, A64 instructions are always 32-bits stack.iret.elr_el1 += 4; - return true + return true; } //MRS , CNTVCT_EL0 (0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => { @@ -115,22 +127,25 @@ unsafe fn instr_trapped_msr_mrs_inner(stack: &mut InterruptStack, from_user: boo stack.store_reg(rt as usize, reg_val); //skip faulting instruction, A64 instructions are always 32-bits stack.iret.elr_el1 += 4; - return true + return true; } - _ => {}, + _ => {} } false } exception_stack!(synchronous_exception_at_el1_with_spx, |stack| { - if !pf_inner(stack, exception_code(stack.iret.esr_el1), "sync_exc_el1_spx") { + if !pf_inner( + stack, + exception_code(stack.iret.esr_el1), + "sync_exc_el1_spx", + ) { println!("Synchronous exception at EL1 with SPx"); if exception_code(stack.iret.esr_el1) == 0b100101 { let far_el1 = far_el1(); println!("FAR_EL1 = 0x{:08x}", far_el1); - } - else if exception_code(stack.iret.esr_el1) == 0b100100 { + } else if exception_code(stack.iret.esr_el1) == 0b100100 { let far_el1 = far_el1(); println!("USER FAR_EL1 = 0x{:08x}", far_el1); } @@ -160,16 +175,23 @@ exception_stack!(synchronous_exception_at_el0, |stack| { match exception_code(stack.iret.esr_el1) { 0b010101 => with_exception_stack!(|stack| { let scratch = &stack.scratch; - syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, stack) + syscall::syscall( + scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, stack, + ) }), - ty => if !pf_inner(stack, ty as u8, "sync_exc_el0") { - log::error!("FATAL: Not an SVC induced synchronous exception (ty={:b})", ty); - println!("FAR_EL1: {:#0x}", far_el1()); - //crate::debugger::debugger(None); - stack.dump(); - stack_trace(); - crate::ksignal(SIGSEGV); + ty => { + if !pf_inner(stack, ty as u8, "sync_exc_el0") { + log::error!( + "FATAL: Not an SVC induced synchronous exception (ty={:b})", + ty + ); + println!("FAR_EL1: {:#0x}", far_el1()); + //crate::debugger::debugger(None); + stack.dump(); + stack_trace(); + crate::ksignal(SIGSEGV); + } } } }); diff --git a/src/arch/aarch64/interrupt/handler.rs b/src/arch/aarch64/interrupt/handler.rs index 0d0be46533..f43a2019be 100644 --- a/src/arch/aarch64/interrupt/handler.rs +++ b/src/arch/aarch64/interrupt/handler.rs @@ -87,9 +87,9 @@ impl PreservedRegisters { #[derive(Default)] #[repr(packed)] pub struct IretRegisters { - // occurred - // The exception vector disambiguates at which EL the interrupt - pub sp_el0: usize, // Shouldn't be used if interrupt occurred at EL1 + // occurred + // The exception vector disambiguates at which EL the interrupt + pub sp_el0: usize, // Shouldn't be used if interrupt occurred at EL1 pub esr_el1: usize, pub spsr_el1: usize, pub elr_el1: usize, @@ -238,12 +238,14 @@ impl InterruptStack { 28 => self.preserved.x28 = val, 29 => self.preserved.x29 = val, 30 => self.preserved.x30 = val, - _ => {}, + _ => {} } } //TODO - pub fn is_singlestep(&self) -> bool { false } + pub fn is_singlestep(&self) -> bool { + false + } pub fn set_singlestep(&mut self, singlestep: bool) {} } @@ -276,7 +278,8 @@ macro_rules! function { #[macro_export] macro_rules! push_scratch { - () => { " + () => { + " // Push scratch registers str x18, [sp, #-16]! stp x16, x17, [sp, #-16]! @@ -288,12 +291,14 @@ macro_rules! push_scratch { stp x4, x5, [sp, #-16]! stp x2, x3, [sp, #-16]! stp x0, x1, [sp, #-16]! - " }; + " + }; } #[macro_export] macro_rules! pop_scratch { - () => { " + () => { + " // Pop scratch registers ldp x0, x1, [sp], #16 ldp x2, x3, [sp], #16 @@ -305,12 +310,14 @@ macro_rules! pop_scratch { ldp x14, x15, [sp], #16 ldp x16, x17, [sp], #16 ldr x18, [sp], #16 - " }; + " + }; } #[macro_export] macro_rules! push_preserved { - () => { " + () => { + " // Push preserved registers stp x29, x30, [sp, #-16]! stp x27, x28, [sp, #-16]! @@ -318,12 +325,14 @@ macro_rules! push_preserved { stp x23, x24, [sp, #-16]! stp x21, x22, [sp, #-16]! stp x19, x20, [sp, #-16]! - " }; + " + }; } #[macro_export] macro_rules! pop_preserved { - () => { " + () => { + " // Pop preserved registers ldp x19, x20, [sp], #16 ldp x21, x22, [sp], #16 @@ -331,12 +340,14 @@ macro_rules! pop_preserved { ldp x25, x26, [sp], #16 ldp x27, x28, [sp], #16 ldp x29, x30, [sp], #16 - " }; + " + }; } #[macro_export] macro_rules! push_special { - () => { " + () => { + " mrs x14, spsr_el1 mrs x15, elr_el1 stp x14, x15, [sp, #-16]! @@ -344,12 +355,14 @@ macro_rules! push_special { mrs x14, sp_el0 mrs x15, esr_el1 stp x14, x15, [sp, #-16]! - " }; + " + }; } #[macro_export] macro_rules! pop_special { - () => { " + () => { + " ldp x14, x15, [sp], 16 msr esr_el1, x15 msr sp_el0, x14 @@ -357,7 +370,8 @@ macro_rules! pop_special { ldp x14, x15, [sp], 16 msr elr_el1, x15 msr spsr_el1, x14 - " }; + " + }; } #[macro_export] diff --git a/src/arch/aarch64/interrupt/irq.rs b/src/arch/aarch64/interrupt/irq.rs index a91f9480f7..3e6ad8ad9e 100644 --- a/src/arch/aarch64/interrupt/irq.rs +++ b/src/arch/aarch64/interrupt/irq.rs @@ -1,6 +1,5 @@ //use crate::device::generic_timer::{GENTIMER}; -use crate::device::irqchip::IRQ_CHIP; -use crate::device::serial::COM1; +use crate::device::{irqchip::IRQ_CHIP, serial::COM1}; pub struct IrqDesc { pub virq: u32, @@ -33,10 +32,9 @@ exception_stack!(irq_at_el1, |stack| { } }); - //TODO pub unsafe fn trigger(irq: u32) { - extern { + extern "C" { fn irq_trigger(irq: u32); } diff --git a/src/arch/aarch64/interrupt/mod.rs b/src/arch/aarch64/interrupt/mod.rs index a8ac466901..9ddc914683 100644 --- a/src/arch/aarch64/interrupt/mod.rs +++ b/src/arch/aarch64/interrupt/mod.rs @@ -12,8 +12,7 @@ pub mod trace; use crate::LogicalCpuId; -pub use self::handler::InterruptStack; -pub use self::trace::stack_trace; +pub use self::{handler::InterruptStack, trace::stack_trace}; /// Clear interrupts #[inline(always)] diff --git a/src/arch/aarch64/interrupt/syscall.rs b/src/arch/aarch64/interrupt/syscall.rs index 311a2ae881..906d0ffc05 100644 --- a/src/arch/aarch64/interrupt/syscall.rs +++ b/src/arch/aarch64/interrupt/syscall.rs @@ -1,15 +1,14 @@ use crate::{ - arch::{interrupt::InterruptStack}, - context, - syscall, - syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_PRE_SYSCALL, PTRACE_STOP_POST_SYSCALL}, + arch::interrupt::InterruptStack, + context, syscall, + syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL}, }; #[no_mangle] -pub unsafe extern fn do_exception_unhandled() {} +pub unsafe extern "C" fn do_exception_unhandled() {} #[no_mangle] -pub unsafe extern fn do_exception_synchronous() {} +pub unsafe extern "C" fn do_exception_synchronous() {} #[allow(dead_code)] #[repr(packed)] @@ -57,7 +56,7 @@ pub struct SyscallStack { #[macro_export] macro_rules! with_exception_stack { (|$stack:ident| $code:block) => {{ - let $stack = &mut *$stack; - (*$stack).scratch.x0 = $code; - }} + let $stack = &mut *$stack; + (*$stack).scratch.x0 = $code; + }}; } diff --git a/src/arch/aarch64/interrupt/trace.rs b/src/arch/aarch64/interrupt/trace.rs index 212b13ded5..cf7572c1ae 100644 --- a/src/arch/aarch64/interrupt/trace.rs +++ b/src/arch/aarch64/interrupt/trace.rs @@ -18,7 +18,8 @@ pub unsafe fn stack_trace() { for _frame in 0..64 { if let Some(pc_fp) = fp.checked_add(mem::size_of::()) { if mapper.translate(VirtualAddress::new(fp)).is_some() - && mapper.translate(VirtualAddress::new(pc_fp)).is_some() { + && mapper.translate(VirtualAddress::new(pc_fp)).is_some() + { let pc = *(pc_fp as *const usize); if pc == 0 { println!(" {:>016x}: EMPTY RETURN", fp); @@ -41,11 +42,12 @@ pub unsafe fn stack_trace() { //TODO: Do not create Elf object for every symbol lookup #[inline(never)] pub unsafe fn symbol_trace(addr: usize) { - use core::slice; - use core::sync::atomic::Ordering; + use core::{slice, sync::atomic::Ordering}; - use crate::elf::Elf; - use crate::start::{KERNEL_BASE, KERNEL_SIZE}; + use crate::{ + elf::Elf, + start::{KERNEL_BASE, KERNEL_SIZE}, + }; let kernel_ptr = (KERNEL_BASE.load(Ordering::SeqCst) + crate::KERNEL_OFFSET) as *const u8; let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst)); @@ -70,76 +72,80 @@ pub unsafe fn symbol_trace(addr: usize) { for sym in symbols { if sym::st_type(sym.st_info) == sym::STT_FUNC && addr >= sym.st_value as usize - && addr < (sym.st_value + sym.st_size) as usize - { - println!(" {:>016X}+{:>04X}", sym.st_value, addr - sym.st_value as usize); + && addr < (sym.st_value + sym.st_size) as usize + { + println!( + " {:>016X}+{:>04X}", + sym.st_value, + addr - sym.st_value as usize + ); - if let Some(strtab) = strtab_opt { - let start = strtab.sh_offset as usize + sym.st_name as usize; - let mut end = start; - while end < elf.data.len() { - let b = elf.data[end]; - end += 1; - if b == 0 { - break; - } - } - - if end > start { - let sym_name = &elf.data[start .. end]; - - print!(" "); - - if sym_name.starts_with(b"_ZN") { - // Skip _ZN - let mut i = 3; - let mut first = true; - while i < sym_name.len() { - // E is the end character - if sym_name[i] == b'E' { - break; - } - - // Parse length string - let mut len = 0; - while i < sym_name.len() { - let b = sym_name[i]; - if b >= b'0' && b <= b'9' { - i += 1; - len *= 10; - len += (b - b'0') as usize; - } else { - break; - } - } - - // Print namespace seperator, if required - if first { - first = false; - } else { - print!("::"); - } - - // Print name string - let end = i + len; - while i < sym_name.len() && i < end { - print!("{}", sym_name[i] as char); - i += 1; - } - } - } else { - for &b in sym_name.iter() { - print!("{}", b as char); - } - } - - println!(""); - } + if let Some(strtab) = strtab_opt { + let start = strtab.sh_offset as usize + sym.st_name as usize; + let mut end = start; + while end < elf.data.len() { + let b = elf.data[end]; + end += 1; + if b == 0 { + break; } } + + if end > start { + let sym_name = &elf.data[start..end]; + + print!(" "); + + if sym_name.starts_with(b"_ZN") { + // Skip _ZN + let mut i = 3; + let mut first = true; + while i < sym_name.len() { + // E is the end character + if sym_name[i] == b'E' { + break; + } + + // Parse length string + let mut len = 0; + while i < sym_name.len() { + let b = sym_name[i]; + if b >= b'0' && b <= b'9' { + i += 1; + len *= 10; + len += (b - b'0') as usize; + } else { + break; + } + } + + // Print namespace seperator, if required + if first { + first = false; + } else { + print!("::"); + } + + // Print name string + let end = i + len; + while i < sym_name.len() && i < end { + print!("{}", sym_name[i] as char); + i += 1; + } + } + } else { + for &b in sym_name.iter() { + print!("{}", b as char); + } + } + + println!(""); + } + } + } } } - }, + } Err(_e) => { println!("WTF ?"); } diff --git a/src/arch/aarch64/misc.rs b/src/arch/aarch64/misc.rs index 1e510c0d47..f2828bc6ac 100644 --- a/src/arch/aarch64/misc.rs +++ b/src/arch/aarch64/misc.rs @@ -1,6 +1,8 @@ -use crate::LogicalCpuId; -use crate::paging::{RmmA, RmmArch}; -use crate::percpu::PercpuBlock; +use crate::{ + paging::{RmmA, RmmArch}, + percpu::PercpuBlock, + LogicalCpuId, +}; impl PercpuBlock { pub fn current() -> &'static Self { diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 473aafca16..7bf4c04e35 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -46,7 +46,8 @@ pub use arch_copy_to_user as arch_copy_from_user; #[link_section = ".usercopy-fns"] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { // x0, x1, x2 - core::arch::asm!(" + core::arch::asm!( + " mov x4, x0 mov x0, 0 2: @@ -63,13 +64,18 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - b 2b 3: ret - ", options(noreturn)); + ", + options(noreturn) + ); } pub unsafe fn bootstrap_mem(bootstrap: &crate::Bootstrap) -> &'static [u8] { - use ::rmm::Arch; use crate::memory::PAGE_SIZE; + use ::rmm::Arch; - core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) + core::slice::from_raw_parts( + CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, + bootstrap.page_count * PAGE_SIZE, + ) } pub const KFX_SIZE: usize = 1024; diff --git a/src/arch/aarch64/paging/entry.rs b/src/arch/aarch64/paging/entry.rs index 289ce8d258..d663ebefeb 100644 --- a/src/arch/aarch64/paging/entry.rs +++ b/src/arch/aarch64/paging/entry.rs @@ -11,4 +11,3 @@ bitflags! { const DEV_MEM = 2 << 2; } } - diff --git a/src/arch/aarch64/paging/mapper.rs b/src/arch/aarch64/paging/mapper.rs index 9f7659b4e6..1543c6ff80 100644 --- a/src/arch/aarch64/paging/mapper.rs +++ b/src/arch/aarch64/paging/mapper.rs @@ -4,16 +4,22 @@ use super::RmmA; pub use rmm::{Flusher, PageFlush, PageFlushAll}; -pub struct InactiveFlusher { _inner: () } +pub struct InactiveFlusher { + _inner: (), +} impl InactiveFlusher { // TODO: cpu id - pub fn new() -> Self { Self { _inner: () } } + pub fn new() -> Self { + Self { _inner: () } + } } impl Flusher for InactiveFlusher { fn consume(&mut self, flush: PageFlush) { // TODO: Push to TLB "mailbox" or tell it to reload CR3 if there are too many entries. - unsafe { flush.ignore(); } + unsafe { + flush.ignore(); + } } } impl Drop for InactiveFlusher { diff --git a/src/arch/aarch64/paging/mod.rs b/src/arch/aarch64/paging/mod.rs index bfb00e6b4d..ca73142e25 100644 --- a/src/arch/aarch64/paging/mod.rs +++ b/src/arch/aarch64/paging/mod.rs @@ -7,15 +7,8 @@ use crate::device::cpu::registers::{control_regs, tlb}; use self::mapper::PageFlushAll; -pub use rmm::{ - Arch as RmmArch, - Flusher, - PageFlags, - PhysicalAddress, - TableKind, - VirtualAddress, -}; pub use super::CurrentRmmArch as RmmA; +pub use rmm::{Arch as RmmArch, Flusher, PageFlags, PhysicalAddress, TableKind, VirtualAddress}; pub type PageMapper = rmm::PageMapper; pub use crate::rmm::KernelMapper; @@ -83,7 +76,10 @@ impl Page { } pub fn range_inclusive(start: Page, r#final: Page) -> PageIter { - PageIter { start, end: r#final.next() } + PageIter { + start, + end: r#final.next(), + } } pub fn range_exclusive(start: Page, end: Page) -> PageIter { PageIter { start, end } diff --git a/src/arch/aarch64/rmm.rs b/src/arch/aarch64/rmm.rs index 94e2037e7c..04dae8814a 100644 --- a/src/arch/aarch64/rmm.rs +++ b/src/arch/aarch64/rmm.rs @@ -1,28 +1,15 @@ use core::{ - cmp, - mem, - slice, - sync::atomic::{self, AtomicUsize, Ordering}, cell::SyncUnsafeCell, + cell::SyncUnsafeCell, + cmp, mem, slice, + sync::atomic::{self, AtomicUsize, Ordering}, }; use rmm::{ - KILOBYTE, - MEGABYTE, - Arch, - BuddyAllocator, - BumpAllocator, - FrameAllocator, - FrameCount, - FrameUsage, - MemoryArea, - PageFlags, - PageMapper, - PhysicalAddress, - TableKind, - VirtualAddress, + Arch, BuddyAllocator, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, MemoryArea, + PageFlags, PageMapper, PhysicalAddress, TableKind, VirtualAddress, KILOBYTE, MEGABYTE, }; use spin::Mutex; -use crate::{LogicalCpuId, init::device_tree::MEMORY_MAP, paging::entry::EntryFlags}; +use crate::{init::device_tree::MEMORY_MAP, paging::entry::EntryFlags, LogicalCpuId}; use super::CurrentRmmArch as RmmA; @@ -63,11 +50,16 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { unsafe fn inner( areas: &'static [MemoryArea], - kernel_base: usize, kernel_size_aligned: usize, - stack_base: usize, stack_size_aligned: usize, - env_base: usize, env_size_aligned: usize, - acpi_base: usize, acpi_size_aligned: usize, - initfs_base: usize, initfs_size_aligned: usize, + kernel_base: usize, + kernel_size_aligned: usize, + stack_base: usize, + stack_size_aligned: usize, + env_base: usize, + env_size_aligned: usize, + acpi_base: usize, + acpi_size_aligned: usize, + initfs_base: usize, + initfs_size_aligned: usize, ) -> BuddyAllocator { // First, calculate how much memory we have let mut size = 0; @@ -84,10 +76,8 @@ unsafe fn inner( let mut bump_allocator = BumpAllocator::::new(areas, 0); { - let mut mapper = PageMapper::::create( - TableKind::Kernel, - &mut bump_allocator - ).expect("failed to create Mapper"); + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); // Map all physical areas at PHYS_OFFSET for area in areas.iter() { @@ -95,11 +85,9 @@ unsafe fn inner( let phys = area.base.add(i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -109,19 +97,15 @@ unsafe fn inner( let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE); let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table let virt = A::phys_to_virt(phys); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } @@ -131,11 +115,9 @@ unsafe fn inner( let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } }; @@ -163,7 +145,8 @@ unsafe fn inner( //map dev mem for mem in MEMORY_MAP { if mem._type == 2 { - let size_aligned = ((mem.length as usize + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let size_aligned = + ((mem.length as usize + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let base = mem.base_addr as usize; for i in 0..size_aligned / A::PAGE_SIZE { let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); @@ -171,13 +154,10 @@ unsafe fn inner( // use the same mair_el1 value with bootloader, // mair_el1 == 0x00000000000044FF // set mem_attr == device memory - let flags = page_flags::(virt) - .custom_flag(EntryFlags::DEV_MEM.bits(), true); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flags = page_flags::(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -195,12 +175,10 @@ unsafe fn inner( let phys = PhysicalAddress::new(phys + i * A::PAGE_SIZE); let virt = VirtualAddress::new(virt + i * A::PAGE_SIZE); let flags = PageFlags::new().write(true); - //TODO: Write combining flag - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + //TODO: Write combining flag + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -220,7 +198,10 @@ unsafe fn inner( // Create the physical memory map let offset = bump_allocator.offset(); - log::info!("Permanently used: {} KB", (offset + (KILOBYTE - 1)) / KILOBYTE); + log::info!( + "Permanently used: {} KB", + (offset + (KILOBYTE - 1)) / KILOBYTE + ); BuddyAllocator::::new(bump_allocator).expect("failed to create BuddyAllocator") } @@ -264,10 +245,12 @@ impl core::fmt::Debug for LockedAllocator { } } -static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new([MemoryArea { - base: PhysicalAddress::new(0), - size: 0, -}; 512]); +static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new( + [MemoryArea { + base: PhysicalAddress::new(0), + size: 0, + }; 512], +); static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); // TODO: Share code @@ -300,7 +283,12 @@ pub struct KernelMapper { impl KernelMapper { fn lock_inner(current_processor: usize) -> bool { loop { - match LOCK_OWNER.compare_exchange_weak(NO_PROCESSOR, current_processor, Ordering::Acquire, Ordering::Relaxed) { + match LOCK_OWNER.compare_exchange_weak( + NO_PROCESSOR, + current_processor, + Ordering::Acquire, + Ordering::Relaxed, + ) { Ok(_) => break, // already owned by this hardware thread Err(id) if id == current_processor => break, @@ -314,15 +302,20 @@ impl KernelMapper { prev_count > 0 } - pub unsafe fn lock_for_manual_mapper(current_processor: LogicalCpuId, mapper: crate::paging::PageMapper) -> Self { + pub unsafe fn lock_for_manual_mapper( + current_processor: LogicalCpuId, + mapper: crate::paging::PageMapper, + ) -> Self { let ro = Self::lock_inner(current_processor.get() as usize); - Self { - mapper, - ro, - } + Self { mapper, ro } } pub fn lock_manually(current_processor: LogicalCpuId) -> Self { - unsafe { Self::lock_for_manual_mapper(current_processor, PageMapper::current(TableKind::Kernel, FRAME_ALLOCATOR)) } + unsafe { + Self::lock_for_manual_mapper( + current_processor, + PageMapper::current(TableKind::Kernel, FRAME_ALLOCATOR), + ) + } } pub fn lock() -> Self { Self::lock_manually(crate::cpu_id()) @@ -357,12 +350,18 @@ impl Drop for KernelMapper { } pub unsafe fn init( - kernel_base: usize, kernel_size: usize, - stack_base: usize, stack_size: usize, - env_base: usize, env_size: usize, - acpi_base: usize, acpi_size: usize, - areas_base: usize, areas_size: usize, - initfs_base: usize, initfs_size: usize, + kernel_base: usize, + kernel_size: usize, + stack_base: usize, + stack_size: usize, + env_base: usize, + env_size: usize, + acpi_base: usize, + acpi_size: usize, + areas_base: usize, + areas_size: usize, + initfs_base: usize, + initfs_size: usize, ) { type A = RmmA; @@ -370,24 +369,24 @@ pub unsafe fn init( let real_size = 0x100000; let real_end = real_base + real_size; - let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let kernel_end = kernel_base + kernel_size_aligned; - let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let stack_end = stack_base + stack_size_aligned; - let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let env_end = env_base + env_size_aligned; - let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let acpi_end = acpi_base + acpi_size_aligned; - let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let initfs_end = initfs_base + initfs_size_aligned; let bootloader_areas = slice::from_raw_parts( areas_base as *const BootloaderMemoryEntry, - areas_size / mem::size_of::() + areas_size / mem::size_of::(), ); let areas = &mut *AREAS.get(); @@ -422,42 +421,84 @@ pub unsafe fn init( // Ensure real-mode areas are not used if base < real_end && base + size > real_base { - log::warn!("{:X}:{:X} overlaps with real mode {:X}:{:X}", base, size, real_base, real_size); + log::warn!( + "{:X}:{:X} overlaps with real mode {:X}:{:X}", + base, + size, + real_base, + real_size + ); new_base = cmp::max(new_base, real_end); } // Ensure kernel areas are not used if base < kernel_end && base + size > kernel_base { - log::warn!("{:X}:{:X} overlaps with kernel {:X}:{:X}", base, size, kernel_base, kernel_size); + log::warn!( + "{:X}:{:X} overlaps with kernel {:X}:{:X}", + base, + size, + kernel_base, + kernel_size + ); new_base = cmp::max(new_base, kernel_end); } // Ensure stack areas are not used if base < stack_end && base + size > stack_base { - log::warn!("{:X}:{:X} overlaps with stack {:X}:{:X}", base, size, stack_base, stack_size); + log::warn!( + "{:X}:{:X} overlaps with stack {:X}:{:X}", + base, + size, + stack_base, + stack_size + ); new_base = cmp::max(new_base, stack_end); } // Ensure env areas are not used if base < env_end && base + size > env_base { - log::warn!("{:X}:{:X} overlaps with env {:X}:{:X}", base, size, env_base, env_size); + log::warn!( + "{:X}:{:X} overlaps with env {:X}:{:X}", + base, + size, + env_base, + env_size + ); new_base = cmp::max(new_base, env_end); } // Ensure acpi areas are not used if base < acpi_end && base + size > acpi_base { - log::warn!("{:X}:{:X} overlaps with acpi {:X}:{:X}", base, size, acpi_base, acpi_size); + log::warn!( + "{:X}:{:X} overlaps with acpi {:X}:{:X}", + base, + size, + acpi_base, + acpi_size + ); new_base = cmp::max(new_base, acpi_end); } if base < initfs_end && base + size > initfs_base { - log::warn!("{:X}:{:X} overlaps with initfs {:X}:{:X}", base, size, initfs_base, initfs_size); + log::warn!( + "{:X}:{:X} overlaps with initfs {:X}:{:X}", + base, + size, + initfs_base, + initfs_size + ); new_base = cmp::max(new_base, initfs_end); } if new_base != base { let end = base + size; let new_size = end.checked_sub(new_base).unwrap_or(0); - log::info!("{:X}:{:X} moved to {:X}:{:X}", base, size, new_base, new_size); + log::info!( + "{:X}:{:X} moved to {:X}:{:X}", + base, + size, + new_base, + new_size + ); base = new_base; size = new_size; } @@ -475,11 +516,16 @@ pub unsafe fn init( let allocator = inner::( areas, - kernel_base, kernel_size_aligned, - stack_base, stack_size_aligned, - env_base, env_size_aligned, - acpi_base, acpi_size_aligned, - initfs_base, initfs_size_aligned, + kernel_base, + kernel_size_aligned, + stack_base, + stack_size_aligned, + env_base, + env_size_aligned, + acpi_base, + acpi_size_aligned, + initfs_base, + initfs_size_aligned, ); *INNER_ALLOCATOR.lock() = Some(allocator); } diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index e7edc0aa56..e47cc5c0e0 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -2,21 +2,23 @@ /// It is increcibly unsafe, and should be minimal in nature /// It must create the IDT with the correct entries, those entries are /// defined in other files inside of the `arch` module - use core::slice; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering, AtomicU32}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use crate::memory::{Frame}; -use crate::paging::{Page, PAGE_SIZE, PhysicalAddress, VirtualAddress}; +use crate::{ + memory::Frame, + paging::{Page, PhysicalAddress, VirtualAddress, PAGE_SIZE}, +}; -use crate::{allocator, dtb}; -use crate::device; #[cfg(feature = "graphical_debug")] use crate::devices::graphical_debug; -use crate::init::device_tree; -use crate::interrupt; -use crate::log::{self, info}; -use crate::paging::{self, KernelMapper}; +use crate::{ + allocator, device, dtb, + init::device_tree, + interrupt, + log::{self, info}, + paging::{self, KernelMapper}, +}; /// Test of zero values in BSS. static BSS_TEST_ZERO: usize = 0; @@ -71,7 +73,10 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { } // Convert env to slice - let env = slice::from_raw_parts((crate::PHYS_OFFSET + args.env_base) as *const u8, args.env_size); + let env = slice::from_raw_parts( + (crate::PHYS_OFFSET + args.env_base) as *const u8, + args.env_size, + ); // Set up graphical debug //#[cfg(feature = "graphical_debug")] @@ -90,13 +95,37 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { }); info!("Redox OS starting..."); - info!("Kernel: {:X}:{:X}", {args.kernel_base}, args.kernel_base + args.kernel_size); - info!("Stack: {:X}:{:X}", {args.stack_base}, args.stack_base + args.stack_size); - info!("Env: {:X}:{:X}", {args.env_base}, args.env_base + args.env_size); - info!("RSDPs: {:X}:{:X}", {args.dtb_base}, args.dtb_base + args.dtb_size); - info!("Areas: {:X}:{:X}", {args.areas_base}, args.areas_base + args.areas_size); - info!("Bootstrap: {:X}:{:X}", {args.bootstrap_base}, args.bootstrap_base + args.bootstrap_size); - info!("Bootstrap entry point: {:X}", {args.bootstrap_entry}); + info!( + "Kernel: {:X}:{:X}", + { args.kernel_base }, + args.kernel_base + args.kernel_size + ); + info!( + "Stack: {:X}:{:X}", + { args.stack_base }, + args.stack_base + args.stack_size + ); + info!( + "Env: {:X}:{:X}", + { args.env_base }, + args.env_base + args.env_size + ); + info!( + "RSDPs: {:X}:{:X}", + { args.dtb_base }, + args.dtb_base + args.dtb_size + ); + info!( + "Areas: {:X}:{:X}", + { args.areas_base }, + args.areas_base + args.areas_size + ); + info!( + "Bootstrap: {:X}:{:X}", + { args.bootstrap_base }, + args.bootstrap_base + args.bootstrap_size + ); + info!("Bootstrap entry point: {:X}", { args.bootstrap_entry }); // Setup interrupt handlers core::arch::asm!( @@ -108,8 +137,8 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { ); if args.dtb_base != 0 { - //Try to read device memory map - device_tree::fill_memory_map(crate::PHYS_OFFSET + args.dtb_base, args.dtb_size); + //Try to read device memory map + device_tree::fill_memory_map(crate::PHYS_OFFSET + args.dtb_base, args.dtb_size); } /* NOT USED WITH UEFI @@ -118,12 +147,18 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { // Initialize RMM crate::arch::rmm::init( - args.kernel_base, args.kernel_size, - args.stack_base, args.stack_size, - args.env_base, args.env_size, - args.dtb_base, args.dtb_size, - args.areas_base, args.areas_size, - args.bootstrap_base, args.bootstrap_size, + args.kernel_base, + args.kernel_size, + args.stack_base, + args.stack_size, + args.env_base, + args.env_size, + args.dtb_base, + args.dtb_size, + args.areas_base, + args.areas_size, + args.bootstrap_base, + args.bootstrap_size, ); // Initialize paging @@ -163,7 +198,9 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { BSP_READY.store(true, Ordering::SeqCst); crate::Bootstrap { - base: crate::memory::Frame::containing_address(crate::paging::PhysicalAddress::new(args.bootstrap_base)), + base: crate::memory::Frame::containing_address(crate::paging::PhysicalAddress::new( + args.bootstrap_base, + )), page_count: args.bootstrap_size / crate::memory::PAGE_SIZE, entry: args.bootstrap_entry, env, @@ -182,8 +219,8 @@ pub struct KernelArgsAp { } /// Entry to rust for an AP -pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { - loop{} +pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { + loop {} } #[naked] diff --git a/src/arch/aarch64/stop.rs b/src/arch/aarch64/stop.rs index 6f12001777..e3251f334a 100644 --- a/src/arch/aarch64/stop.rs +++ b/src/arch/aarch64/stop.rs @@ -1,7 +1,7 @@ use core::arch::asm; #[no_mangle] -pub unsafe extern fn kreset() -> ! { +pub unsafe extern "C" fn kreset() -> ! { println!("kreset"); let val: u32 = 0x8400_0009; @@ -16,7 +16,7 @@ pub unsafe fn emergency_reset() -> ! { } #[no_mangle] -pub unsafe extern fn kstop() -> ! { +pub unsafe extern "C" fn kstop() -> ! { println!("kstop"); let val: u32 = 0x8400_0008; diff --git a/src/arch/aarch64/vectors.rs b/src/arch/aarch64/vectors.rs index 629f6e9e43..2b61f9d9a7 100644 --- a/src/arch/aarch64/vectors.rs +++ b/src/arch/aarch64/vectors.rs @@ -1,5 +1,5 @@ core::arch::global_asm!( -" + " // Exception vector stubs // // Unhandled exceptions spin in a wfi loop for the moment @@ -108,4 +108,5 @@ __vec_15: .align 7 exception_vector_end: -"); +" +); diff --git a/src/arch/x86/consts.rs b/src/arch/x86/consts.rs index 7c5f92583d..89309acc8c 100644 --- a/src/arch/x86/consts.rs +++ b/src/arch/x86/consts.rs @@ -3,30 +3,30 @@ // Each PML4 entry references up to 512 GB of memory // The second from the top (510) PML4 is reserved for the kernel - /// Offset of kernel (256 MiB max) - pub const KERNEL_OFFSET: usize = 0xC000_0000; +/// Offset of kernel (256 MiB max) +pub const KERNEL_OFFSET: usize = 0xC000_0000; - // Framebuffer mapped by bootloader to 0xD000_0000 (128 MiB max) +// Framebuffer mapped by bootloader to 0xD000_0000 (128 MiB max) - // Offset to APIC mappings (optional) - pub const LAPIC_OFFSET: usize = 0xD800_0000; - pub const IOAPIC_OFFSET: usize = LAPIC_OFFSET + 4096; - pub const HPET_OFFSET: usize = IOAPIC_OFFSET + 4096; +// Offset to APIC mappings (optional) +pub const LAPIC_OFFSET: usize = 0xD800_0000; +pub const IOAPIC_OFFSET: usize = LAPIC_OFFSET + 4096; +pub const HPET_OFFSET: usize = IOAPIC_OFFSET + 4096; - /// Offset to kernel heap (256 MiB max) - pub const KERNEL_HEAP_OFFSET: usize = 0xE000_0000; - /// Size of kernel heap - pub const KERNEL_HEAP_SIZE: usize = rmm::MEGABYTE; +/// Offset to kernel heap (256 MiB max) +pub const KERNEL_HEAP_OFFSET: usize = 0xE000_0000; +/// Size of kernel heap +pub const KERNEL_HEAP_SIZE: usize = rmm::MEGABYTE; - /// Offset to kernel percpu variables (256 MiB max) - pub const KERNEL_PERCPU_OFFSET: usize = 0xF000_0000; - /// Size of kernel percpu variables - pub const KERNEL_PERCPU_SHIFT: u8 = 16; // 2^16 = 64 KiB - pub const KERNEL_PERCPU_SIZE: usize = 1_usize << KERNEL_PERCPU_SHIFT; +/// Offset to kernel percpu variables (256 MiB max) +pub const KERNEL_PERCPU_OFFSET: usize = 0xF000_0000; +/// Size of kernel percpu variables +pub const KERNEL_PERCPU_SHIFT: u8 = 16; // 2^16 = 64 KiB +pub const KERNEL_PERCPU_SIZE: usize = 1_usize << KERNEL_PERCPU_SHIFT; - /// Offset of physmap (1 GiB max) - // This needs to match RMM's PHYS_OFFSET - pub const PHYS_OFFSET: usize = 0x8000_0000; +/// Offset of physmap (1 GiB max) +// This needs to match RMM's PHYS_OFFSET +pub const PHYS_OFFSET: usize = 0x8000_0000; - /// End offset of the user image, i.e. kernel start - pub const USER_END_OFFSET: usize = 0x8000_0000; +/// End offset of the user image, i.e. kernel start +pub const USER_END_OFFSET: usize = 0x8000_0000; diff --git a/src/arch/x86/debug.rs b/src/arch/x86/debug.rs index 0906c89c62..4013e94f0c 100644 --- a/src/arch/x86/debug.rs +++ b/src/arch/x86/debug.rs @@ -3,24 +3,24 @@ use core::fmt; use spin::Mutex; use spin::MutexGuard; -use crate::log::{LOG, Log}; -#[cfg(feature = "qemu_debug")] -use syscall::io::Io; -#[cfg(any(feature = "qemu_debug", feature = "serial_debug"))] -use crate::syscall::io::Pio; -#[cfg(feature = "lpss_debug")] -use crate::syscall::io::Mmio; #[cfg(any(feature = "lpss_debug", feature = "serial_debug"))] use crate::devices::uart_16550::SerialPort; - -#[cfg(feature = "graphical_debug")] -use crate::devices::graphical_debug::{DEBUG_DISPLAY, DebugDisplay}; +use crate::log::{Log, LOG}; #[cfg(feature = "lpss_debug")] -use super::device::serial::LPSS; +use crate::syscall::io::Mmio; +#[cfg(any(feature = "qemu_debug", feature = "serial_debug"))] +use crate::syscall::io::Pio; +#[cfg(feature = "qemu_debug")] +use syscall::io::Io; + #[cfg(feature = "serial_debug")] use super::device::serial::COM1; +#[cfg(feature = "lpss_debug")] +use super::device::serial::LPSS; #[cfg(feature = "system76_ec_debug")] -use super::device::system76_ec::{SYSTEM76_EC, System76Ec}; +use super::device::system76_ec::{System76Ec, SYSTEM76_EC}; +#[cfg(feature = "graphical_debug")] +use crate::devices::graphical_debug::{DebugDisplay, DEBUG_DISPLAY}; #[cfg(feature = "qemu_debug")] pub static QEMU: Mutex> = Mutex::new(Pio::::new(0x402)); diff --git a/src/arch/x86/device/cpu.rs b/src/arch/x86/device/cpu.rs index 5a5c55b8f6..a24405d3c0 100644 --- a/src/arch/x86/device/cpu.rs +++ b/src/arch/x86/device/cpu.rs @@ -28,101 +28,257 @@ pub fn cpu_info(w: &mut W) -> Result { write!(w, "Features:")?; if let Some(info) = cpuid.get_feature_info() { - if info.has_fpu() { write!(w, " fpu")? }; - if info.has_vme() { write!(w, " vme")? }; - if info.has_de() { write!(w, " de")? }; - if info.has_pse() { write!(w, " pse")? }; - if info.has_tsc() { write!(w, " tsc")? }; - if info.has_msr() { write!(w, " msr")? }; - if info.has_pae() { write!(w, " pae")? }; - if info.has_mce() { write!(w, " mce")? }; + if info.has_fpu() { + write!(w, " fpu")? + }; + if info.has_vme() { + write!(w, " vme")? + }; + if info.has_de() { + write!(w, " de")? + }; + if info.has_pse() { + write!(w, " pse")? + }; + if info.has_tsc() { + write!(w, " tsc")? + }; + if info.has_msr() { + write!(w, " msr")? + }; + if info.has_pae() { + write!(w, " pae")? + }; + if info.has_mce() { + write!(w, " mce")? + }; - if info.has_cmpxchg8b() { write!(w, " cx8")? }; - if info.has_apic() { write!(w, " apic")? }; - if info.has_sysenter_sysexit() { write!(w, " sep")? }; - if info.has_mtrr() { write!(w, " mtrr")? }; - if info.has_pge() { write!(w, " pge")? }; - if info.has_mca() { write!(w, " mca")? }; - if info.has_cmov() { write!(w, " cmov")? }; - if info.has_pat() { write!(w, " pat")? }; + if info.has_cmpxchg8b() { + write!(w, " cx8")? + }; + if info.has_apic() { + write!(w, " apic")? + }; + if info.has_sysenter_sysexit() { + write!(w, " sep")? + }; + if info.has_mtrr() { + write!(w, " mtrr")? + }; + if info.has_pge() { + write!(w, " pge")? + }; + if info.has_mca() { + write!(w, " mca")? + }; + if info.has_cmov() { + write!(w, " cmov")? + }; + if info.has_pat() { + write!(w, " pat")? + }; - if info.has_pse36() { write!(w, " pse36")? }; - if info.has_psn() { write!(w, " psn")? }; - if info.has_clflush() { write!(w, " clflush")? }; - if info.has_ds() { write!(w, " ds")? }; - if info.has_acpi() { write!(w, " acpi")? }; - if info.has_mmx() { write!(w, " mmx")? }; - if info.has_fxsave_fxstor() { write!(w, " fxsr")? }; - if info.has_sse() { write!(w, " sse")? }; + if info.has_pse36() { + write!(w, " pse36")? + }; + if info.has_psn() { + write!(w, " psn")? + }; + if info.has_clflush() { + write!(w, " clflush")? + }; + if info.has_ds() { + write!(w, " ds")? + }; + if info.has_acpi() { + write!(w, " acpi")? + }; + if info.has_mmx() { + write!(w, " mmx")? + }; + if info.has_fxsave_fxstor() { + write!(w, " fxsr")? + }; + if info.has_sse() { + write!(w, " sse")? + }; - if info.has_sse2() { write!(w, " sse2")? }; - if info.has_ss() { write!(w, " ss")? }; - if info.has_htt() { write!(w, " ht")? }; - if info.has_tm() { write!(w, " tm")? }; - if info.has_pbe() { write!(w, " pbe")? }; + if info.has_sse2() { + write!(w, " sse2")? + }; + if info.has_ss() { + write!(w, " ss")? + }; + if info.has_htt() { + write!(w, " ht")? + }; + if info.has_tm() { + write!(w, " tm")? + }; + if info.has_pbe() { + write!(w, " pbe")? + }; - if info.has_sse3() { write!(w, " sse3")? }; - if info.has_pclmulqdq() { write!(w, " pclmulqdq")? }; - if info.has_ds_area() { write!(w, " dtes64")? }; - if info.has_monitor_mwait() { write!(w, " monitor")? }; - if info.has_cpl() { write!(w, " ds_cpl")? }; - if info.has_vmx() { write!(w, " vmx")? }; - if info.has_smx() { write!(w, " smx")? }; - if info.has_eist() { write!(w, " est")? }; + if info.has_sse3() { + write!(w, " sse3")? + }; + if info.has_pclmulqdq() { + write!(w, " pclmulqdq")? + }; + if info.has_ds_area() { + write!(w, " dtes64")? + }; + if info.has_monitor_mwait() { + write!(w, " monitor")? + }; + if info.has_cpl() { + write!(w, " ds_cpl")? + }; + if info.has_vmx() { + write!(w, " vmx")? + }; + if info.has_smx() { + write!(w, " smx")? + }; + if info.has_eist() { + write!(w, " est")? + }; - if info.has_tm2() { write!(w, " tm2")? }; - if info.has_ssse3() { write!(w, " ssse3")? }; - if info.has_cnxtid() { write!(w, " cnxtid")? }; - if info.has_fma() { write!(w, " fma")? }; - if info.has_cmpxchg16b() { write!(w, " cx16")? }; - if info.has_pdcm() { write!(w, " pdcm")? }; - if info.has_pcid() { write!(w, " pcid")? }; - if info.has_dca() { write!(w, " dca")? }; + if info.has_tm2() { + write!(w, " tm2")? + }; + if info.has_ssse3() { + write!(w, " ssse3")? + }; + if info.has_cnxtid() { + write!(w, " cnxtid")? + }; + if info.has_fma() { + write!(w, " fma")? + }; + if info.has_cmpxchg16b() { + write!(w, " cx16")? + }; + if info.has_pdcm() { + write!(w, " pdcm")? + }; + if info.has_pcid() { + write!(w, " pcid")? + }; + if info.has_dca() { + write!(w, " dca")? + }; - if info.has_sse41() { write!(w, " sse4_1")? }; - if info.has_sse42() { write!(w, " sse4_2")? }; - if info.has_x2apic() { write!(w, " x2apic")? }; - if info.has_movbe() { write!(w, " movbe")? }; - if info.has_popcnt() { write!(w, " popcnt")? }; - if info.has_tsc_deadline() { write!(w, " tsc_deadline_timer")? }; - if info.has_aesni() { write!(w, " aes")? }; - if info.has_xsave() { write!(w, " xsave")? }; + if info.has_sse41() { + write!(w, " sse4_1")? + }; + if info.has_sse42() { + write!(w, " sse4_2")? + }; + if info.has_x2apic() { + write!(w, " x2apic")? + }; + if info.has_movbe() { + write!(w, " movbe")? + }; + if info.has_popcnt() { + write!(w, " popcnt")? + }; + if info.has_tsc_deadline() { + write!(w, " tsc_deadline_timer")? + }; + if info.has_aesni() { + write!(w, " aes")? + }; + if info.has_xsave() { + write!(w, " xsave")? + }; - if info.has_oxsave() { write!(w, " xsaveopt")? }; - if info.has_avx() { write!(w, " avx")? }; - if info.has_f16c() { write!(w, " f16c")? }; - if info.has_rdrand() { write!(w, " rdrand")? }; + if info.has_oxsave() { + write!(w, " xsaveopt")? + }; + if info.has_avx() { + write!(w, " avx")? + }; + if info.has_f16c() { + write!(w, " f16c")? + }; + if info.has_rdrand() { + write!(w, " rdrand")? + }; } if let Some(info) = cpuid.get_extended_processor_and_feature_identifiers() { - if info.has_64bit_mode() { write!(w, " lm")? }; - if info.has_rdtscp() { write!(w, " rdtscp")? }; - if info.has_1gib_pages() { write!(w, " pdpe1gb")? }; - if info.has_execute_disable() { write!(w, " nx")? }; - if info.has_syscall_sysret() { write!(w, " syscall")? }; - if info.has_prefetchw() { write!(w, " prefetchw")? }; - if info.has_lzcnt() { write!(w, " lzcnt")? }; - if info.has_lahf_sahf() { write!(w, " lahf_lm")? }; + if info.has_64bit_mode() { + write!(w, " lm")? + }; + if info.has_rdtscp() { + write!(w, " rdtscp")? + }; + if info.has_1gib_pages() { + write!(w, " pdpe1gb")? + }; + if info.has_execute_disable() { + write!(w, " nx")? + }; + if info.has_syscall_sysret() { + write!(w, " syscall")? + }; + if info.has_prefetchw() { + write!(w, " prefetchw")? + }; + if info.has_lzcnt() { + write!(w, " lzcnt")? + }; + if info.has_lahf_sahf() { + write!(w, " lahf_lm")? + }; } if let Some(info) = cpuid.get_advanced_power_mgmt_info() { - if info.has_invariant_tsc() { write!(w, " constant_tsc")? }; + if info.has_invariant_tsc() { + write!(w, " constant_tsc")? + }; } if let Some(info) = cpuid.get_extended_feature_info() { - if info.has_fsgsbase() { write!(w, " fsgsbase")? }; - if info.has_tsc_adjust_msr() { write!(w, " tsc_adjust")? }; - if info.has_bmi1() { write!(w, " bmi1")? }; - if info.has_hle() { write!(w, " hle")? }; - if info.has_avx2() { write!(w, " avx2")? }; - if info.has_smep() { write!(w, " smep")? }; - if info.has_bmi2() { write!(w, " bmi2")? }; - if info.has_rep_movsb_stosb() { write!(w, " erms")? }; - if info.has_invpcid() { write!(w, " invpcid")? }; - if info.has_rtm() { write!(w, " rtm")? }; + if info.has_fsgsbase() { + write!(w, " fsgsbase")? + }; + if info.has_tsc_adjust_msr() { + write!(w, " tsc_adjust")? + }; + if info.has_bmi1() { + write!(w, " bmi1")? + }; + if info.has_hle() { + write!(w, " hle")? + }; + if info.has_avx2() { + write!(w, " avx2")? + }; + if info.has_smep() { + write!(w, " smep")? + }; + if info.has_bmi2() { + write!(w, " bmi2")? + }; + if info.has_rep_movsb_stosb() { + write!(w, " erms")? + }; + if info.has_invpcid() { + write!(w, " invpcid")? + }; + if info.has_rtm() { + write!(w, " rtm")? + }; //if info.has_qm() { write!(w, " qm")? }; - if info.has_fpu_cs_ds_deprecated() { write!(w, " fpu_seg")? }; - if info.has_mpx() { write!(w, " mpx")? }; + if info.has_fpu_cs_ds_deprecated() { + write!(w, " fpu_seg")? + }; + if info.has_mpx() { + write!(w, " mpx")? + }; } writeln!(w)?; diff --git a/src/arch/x86/device/hpet.rs b/src/arch/x86/device/hpet.rs index e15968035e..b52fe87a16 100644 --- a/src/arch/x86/device/hpet.rs +++ b/src/arch/x86/device/hpet.rs @@ -1,5 +1,5 @@ -use crate::acpi::hpet::Hpet; use super::pit; +use crate::acpi::hpet::Hpet; const LEG_RT_CNF: u64 = 2; const ENABLE_CNF: u64 = 1; @@ -27,7 +27,8 @@ pub unsafe fn init(hpet: &mut Hpet) -> bool { { let mut config_word = hpet.base_address.read_u64(GENERAL_CONFIG_OFFSET); config_word &= !(LEG_RT_CNF | ENABLE_CNF); - hpet.base_address.write_u64(GENERAL_CONFIG_OFFSET, config_word); + hpet.base_address + .write_u64(GENERAL_CONFIG_OFFSET, config_word); } let capability = hpet.base_address.read_u64(CAPABILITY_OFFSET); @@ -48,9 +49,11 @@ pub unsafe fn init(hpet: &mut Hpet) -> bool { let counter = hpet.base_address.read_u64(MAIN_COUNTER_OFFSET); let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF; - hpet.base_address.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word); + hpet.base_address + .write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word); // set accumulator value - hpet.base_address.write_u64(T0_COMPARATOR_OFFSET, counter + divisor); + hpet.base_address + .write_u64(T0_COMPARATOR_OFFSET, counter + divisor); // set interval hpet.base_address.write_u64(T0_COMPARATOR_OFFSET, divisor); @@ -58,7 +61,8 @@ pub unsafe fn init(hpet: &mut Hpet) -> bool { { let mut config_word: u64 = hpet.base_address.read_u64(GENERAL_CONFIG_OFFSET); config_word |= LEG_RT_CNF | ENABLE_CNF; - hpet.base_address.write_u64(GENERAL_CONFIG_OFFSET, config_word); + hpet.base_address + .write_u64(GENERAL_CONFIG_OFFSET, config_word); } println!("HPET After Init"); @@ -76,7 +80,10 @@ pub unsafe fn debug(hpet: &mut Hpet) { println!(" clock period: {}", (capability >> 32) as u32); println!(" ID: {:#x}", (capability >> 16) as u16); println!(" LEG_RT_CAP: {}", capability & (1 << 15) == (1 << 15)); - println!(" COUNT_SIZE_CAP: {}", capability & (1 << 13) == (1 << 13)); + println!( + " COUNT_SIZE_CAP: {}", + capability & (1 << 13) == (1 << 13) + ); println!(" timers: {}", (capability >> 8) as u8 & 0x1F); println!(" revision: {}", capability as u8); } @@ -92,7 +99,10 @@ pub unsafe fn debug(hpet: &mut Hpet) { let t0_capabilities = hpet.base_address.read_u64(T0_CONFIG_CAPABILITY_OFFSET); println!(" T0 caps: {:#x}", t0_capabilities); - println!(" interrupt routing: {:#x}", (t0_capabilities >> 32) as u32); + println!( + " interrupt routing: {:#x}", + (t0_capabilities >> 32) as u32 + ); println!(" flags: {:#x}", t0_capabilities as u16); let t0_comparator = hpet.base_address.read_u64(T0_COMPARATOR_OFFSET); diff --git a/src/arch/x86/device/ioapic.rs b/src/arch/x86/device/ioapic.rs index bbd3735864..8b56a07e6d 100644 --- a/src/arch/x86/device/ioapic.rs +++ b/src/arch/x86/device/ioapic.rs @@ -4,15 +4,18 @@ use alloc::vec::Vec; use spin::Mutex; #[cfg(feature = "acpi")] -use crate::acpi::madt::{self, Madt, MadtEntry, MadtIoApic, MadtIntSrcOverride}; +use crate::acpi::madt::{self, Madt, MadtEntry, MadtIntSrcOverride, MadtIoApic}; -use crate::arch::interrupt::irq; -use crate::memory::Frame; -use crate::paging::{KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch, VirtualAddress}; -use crate::paging::entry::EntryFlags; +use crate::{ + arch::interrupt::irq, + memory::Frame, + paging::{ + entry::EntryFlags, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch, + VirtualAddress, + }, +}; -use super::super::cpuid::cpuid; -use super::pic; +use super::{super::cpuid::cpuid, pic}; pub struct IoApicRegs { pointer: *const u32, @@ -131,12 +134,12 @@ pub enum DestinationMode { #[repr(u8)] #[derive(Clone, Copy, Debug)] pub enum DeliveryMode { - Fixed = 0b000, - LowestPriority = 0b001, - Smi = 0b010, - Nmi = 0b100, - Init = 0b101, - ExtInt = 0b111, + Fixed = 0b000, + LowestPriority = 0b001, + Smi = 0b010, + Nmi = 0b100, + Init = 0b101, + ExtInt = 0b111, } #[derive(Clone, Copy, Debug)] @@ -176,7 +179,9 @@ impl fmt::Debug for IoApic { let mut guard = self.0.lock(); let count = guard.max_redirection_table_entries(); - f.debug_list().entries((0..count).map(|i| guard.read_ioredtbl(i))).finish() + f.debug_list() + .entries((0..count).map(|i| guard.read_ioredtbl(i))) + .finish() } } @@ -219,14 +224,10 @@ static mut IOAPICS: Option> = None; static mut SRC_OVERRIDES: Option> = None; pub fn ioapics() -> &'static [IoApic] { - unsafe { - IOAPICS.as_ref().map_or(&[], |vector| &vector[..]) - } + unsafe { IOAPICS.as_ref().map_or(&[], |vector| &vector[..]) } } pub fn src_overrides() -> &'static [Override] { - unsafe { - SRC_OVERRIDES.as_ref().map_or(&[], |vector| &vector[..]) - } + unsafe { SRC_OVERRIDES.as_ref().map_or(&[], |vector| &vector[..]) } } #[cfg(feature = "acpi")] @@ -241,14 +242,24 @@ pub unsafe fn handle_ioapic(mapper: &mut KernelMapper, madt_ioapic: &'static Mad mapper .get_mut() .expect("expected KernelMapper not to be locked re-entrant while mapping I/O APIC memory") - .map_phys(page.start_address(), frame.start_address(), PageFlags::new().write(true).custom_flag(EntryFlags::NO_CACHE.bits(), true)) + .map_phys( + page.start_address(), + frame.start_address(), + PageFlags::new() + .write(true) + .custom_flag(EntryFlags::NO_CACHE.bits(), true), + ) .expect("failed to map I/O APIC") .flush(); let ioapic_registers = page.start_address().data() as *const u32; let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base); - assert_eq!(ioapic.regs.lock().id(), madt_ioapic.id, "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC"); + assert_eq!( + ioapic.regs.lock().id(), + madt_ioapic.id, + "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC" + ); IOAPICS.get_or_insert_with(Vec::new).push(ioapic); } @@ -286,7 +297,11 @@ pub unsafe fn handle_src_override(src_override: &'static MadtIntSrcOverride) { } pub unsafe fn init(active_table: &mut KernelMapper) { - let bsp_apic_id = cpuid().unwrap().get_feature_info().unwrap().initial_local_apic_id(); // TODO: remove unwraps + let bsp_apic_id = cpuid() + .unwrap() + .get_feature_info() + .unwrap() + .initial_local_apic_id(); // TODO: remove unwraps // search the madt for all IOAPICs. #[cfg(feature = "acpi")] @@ -310,7 +325,11 @@ pub unsafe fn init(active_table: &mut KernelMapper) { } } } - println!("I/O APICs: {:?}, overrides: {:?}", ioapics(), src_overrides()); + println!( + "I/O APICs: {:?}, overrides: {:?}", + ioapics(), + src_overrides() + ); // map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it // wouldn't have been disabled due to this I/O APIC) @@ -318,11 +337,21 @@ pub unsafe fn init(active_table: &mut KernelMapper) { let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) { Some(over) => (over.gsi, over.trigger_mode, over.polarity), None => { - if src_overrides().iter().any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq) && !src_overrides().iter().any(|over| over.bus_irq == legacy_irq) { + if src_overrides() + .iter() + .any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq) + && !src_overrides() + .iter() + .any(|over| over.bus_irq == legacy_irq) + { // there's an IRQ conflict, making this legacy IRQ inaccessible. continue; } - (legacy_irq.into(), TriggerMode::ConformsToSpecs, Polarity::ConformsToSpecs) + ( + legacy_irq.into(), + TriggerMode::ConformsToSpecs, + Polarity::ConformsToSpecs, + ) } }; let apic = match find_ioapic(gsi) { @@ -354,7 +383,11 @@ pub unsafe fn init(active_table: &mut KernelMapper) { }; apic.map(redir_tbl_index, map_info); } - println!("I/O APICs: {:?}, overrides: {:?}", ioapics(), src_overrides()); + println!( + "I/O APICs: {:?}, overrides: {:?}", + ioapics(), + src_overrides() + ); irq::set_irq_method(irq::IrqMethod::Apic); // tell the firmware that we're using APIC rather than the default 8259 PIC. @@ -385,7 +418,9 @@ fn resolve(irq: u8) -> u32 { get_override(irq).map_or(u32::from(irq), |over| over.gsi) } fn find_ioapic(gsi: u32) -> Option<&'static IoApic> { - ioapics().iter().find(|apic| gsi >= apic.gsi_start && gsi < apic.gsi_start + u32::from(apic.count)) + ioapics() + .iter() + .find(|apic| gsi >= apic.gsi_start && gsi < apic.gsi_start + u32::from(apic.count)) } pub unsafe fn mask(irq: u8) { diff --git a/src/arch/x86/device/local_apic.rs b/src/arch/x86/device/local_apic.rs index 04654722fa..9821f91cfa 100644 --- a/src/arch/x86/device/local_apic.rs +++ b/src/arch/x86/device/local_apic.rs @@ -1,14 +1,16 @@ -use core::sync::atomic::{self, AtomicU32}; -use core::ptr::{read_volatile, write_volatile}; +use core::{ + ptr::{read_volatile, write_volatile}, + sync::atomic::{self, AtomicU32}, +}; use x86::msr::*; -use crate::paging::{KernelMapper, PhysicalAddress, PageFlags, RmmA, RmmArch, VirtualAddress}; +use crate::paging::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch, VirtualAddress}; use super::super::cpuid::cpuid; pub static mut LOCAL_APIC: LocalApic = LocalApic { address: 0, - x2: false + x2: false, }; pub unsafe fn init(active_table: &mut KernelMapper) { @@ -22,7 +24,7 @@ pub unsafe fn init_ap() { /// Local APIC pub struct LocalApic { pub address: usize, - pub x2: bool + pub x2: bool, } #[derive(Debug)] @@ -42,19 +44,21 @@ pub fn bsp_apic_id() -> Option { impl LocalApic { unsafe fn init(&mut self, mapper: &mut KernelMapper) { - let mapper = mapper.get_mut().expect("expected KernelMapper not to be locked re-entrant while initializing LAPIC"); + let mapper = mapper + .get_mut() + .expect("expected KernelMapper not to be locked re-entrant while initializing LAPIC"); let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000); let virtaddr = VirtualAddress::new(crate::LAPIC_OFFSET); self.address = virtaddr.data(); self.x2 = cpuid().map_or(false, |cpuid| { - cpuid.get_feature_info().map_or(false, |feature_info| { - feature_info.has_x2apic() - }) + cpuid + .get_feature_info() + .map_or(false, |feature_info| feature_info.has_x2apic()) }); - if ! self.x2 { + if !self.x2 { log::info!("Detected xAPIC at {:#x}", physaddr.data()); if let Some((_entry, _, flush)) = mapper.unmap_phys(virtaddr, true) { // Unmap xAPIC page if already mapped @@ -111,15 +115,15 @@ impl LocalApic { if self.x2 { unsafe { rdmsr(IA32_X2APIC_ICR) } } else { - unsafe { - (self.read(0x310) as u64) << 32 | self.read(0x300) as u64 - } + unsafe { (self.read(0x310) as u64) << 32 | self.read(0x300) as u64 } } } pub fn set_icr(&mut self, value: u64) { if self.x2 { - unsafe { wrmsr(IA32_X2APIC_ICR, value); } + unsafe { + wrmsr(IA32_X2APIC_ICR, value); + } } else { unsafe { const PENDING: u32 = 1 << 12; diff --git a/src/arch/x86/device/mod.rs b/src/arch/x86/device/mod.rs index 14efd8b781..3c8d77335a 100644 --- a/src/arch/x86/device/mod.rs +++ b/src/arch/x86/device/mod.rs @@ -1,12 +1,12 @@ pub mod cpu; +#[cfg(feature = "acpi")] +pub mod hpet; pub mod ioapic; pub mod local_apic; pub mod pic; pub mod pit; pub mod rtc; pub mod serial; -#[cfg(feature = "acpi")] -pub mod hpet; #[cfg(feature = "system76_ec_debug")] pub mod system76_ec; @@ -16,7 +16,7 @@ pub unsafe fn init() { pic::init(); local_apic::init(&mut KernelMapper::lock()); } -pub unsafe fn init_after_acpi() { +pub unsafe fn init_after_acpi() { // this will disable the IOAPIC if needed. //ioapic::init(mapper); } @@ -37,7 +37,9 @@ unsafe fn init_hpet() -> bool { } pub unsafe fn init_noncore() { - if false /*TODO: init_hpet()*/ { + if false + /*TODO: init_hpet()*/ + { log::info!("HPET used as system timer"); } else { pit::init(); diff --git a/src/arch/x86/device/pic.rs b/src/arch/x86/device/pic.rs index 12e68a61e9..487229892e 100644 --- a/src/arch/x86/device/pic.rs +++ b/src/arch/x86/device/pic.rs @@ -1,5 +1,7 @@ -use crate::syscall::io::{Io, Pio}; -use crate::arch::interrupt::irq; +use crate::{ + arch::interrupt::irq, + syscall::io::{Io, Pio}, +}; pub static mut MASTER: Pic = Pic::new(0x20); pub static mut SLAVE: Pic = Pic::new(0xA0); diff --git a/src/arch/x86/device/rtc.rs b/src/arch/x86/device/rtc.rs index d68cdbb8a0..0b7bba9e7a 100644 --- a/src/arch/x86/device/rtc.rs +++ b/src/arch/x86/device/rtc.rs @@ -1,5 +1,7 @@ -use crate::syscall::io::{Io, Pio}; -use crate::time; +use crate::{ + syscall::io::{Io, Pio}, + time, +}; pub fn init() { let mut rtc = Rtc::new(); diff --git a/src/arch/x86/device/serial.rs b/src/arch/x86/device/serial.rs index 280125bc12..83a480e92d 100644 --- a/src/arch/x86/device/serial.rs +++ b/src/arch/x86/device/serial.rs @@ -1,7 +1,6 @@ -use crate::devices::uart_16550::SerialPort; #[cfg(feature = "lpss_debug")] use crate::syscall::io::Mmio; -use crate::syscall::io::Pio; +use crate::{devices::uart_16550::SerialPort, syscall::io::Pio}; use spin::Mutex; pub static COM1: Mutex>> = Mutex::new(SerialPort::>::new(0x3F8)); @@ -22,19 +21,24 @@ pub unsafe fn init() { let address = crate::PHYS_OFFSET + 0xFE032000; { - use crate::paging::{ActivePageTable, Page, VirtualAddress, entry::EntryFlags}; - use crate::memory::{Frame, PhysicalAddress}; + use crate::{ + memory::{Frame, PhysicalAddress}, + paging::{entry::EntryFlags, ActivePageTable, Page, VirtualAddress}, + }; let mut active_table = ActivePageTable::new(); let page = Page::containing_address(VirtualAddress::new(address)); - let frame = Frame::containing_address(PhysicalAddress::new(address - crate::PHYS_OFFSET)); - let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE); + let frame = + Frame::containing_address(PhysicalAddress::new(address - crate::PHYS_OFFSET)); + let result = active_table.map_to( + page, + frame, + EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE, + ); result.flush(&mut active_table); } - let lpss = SerialPort::>::new( - crate::PHYS_OFFSET + 0xFE032000 - ); + let lpss = SerialPort::>::new(crate::PHYS_OFFSET + 0xFE032000); lpss.init(); *LPSS.lock() = Some(lpss); diff --git a/src/arch/x86/device/system76_ec.rs b/src/arch/x86/device/system76_ec.rs index 49f3a29134..dca4d5efe3 100644 --- a/src/arch/x86/device/system76_ec.rs +++ b/src/arch/x86/device/system76_ec.rs @@ -13,9 +13,7 @@ pub struct System76Ec { impl System76Ec { pub fn new() -> Option { - let mut system76_ec = Self { - base: 0x0E00, - }; + let mut system76_ec = Self { base: 0x0E00 }; if system76_ec.probe() { Some(system76_ec) } else { diff --git a/src/arch/x86/gdt.rs b/src/arch/x86/gdt.rs index a1d20ef20f..5fe9793996 100644 --- a/src/arch/x86/gdt.rs +++ b/src/arch/x86/gdt.rs @@ -1,16 +1,15 @@ //! Global descriptor table -use core::convert::TryInto; -use core::mem; -use core::ptr::addr_of_mut; +use core::{convert::TryInto, mem, ptr::addr_of_mut}; use crate::LogicalCpuId; -use x86::bits32::task::TaskStateSegment; -use x86::Ring; -use x86::dtables::{self, DescriptorTablePointer}; -use x86::segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector}; -use x86::task; +use x86::{ + bits32::task::TaskStateSegment, + dtables::{self, DescriptorTablePointer}, + segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector}, + task, Ring, +}; use crate::paging::{RmmA, RmmArch, PAGE_SIZE}; @@ -48,28 +47,73 @@ static INIT_GDT: [GdtEntry; 3] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // Kernel data - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), ]; const BASE_GDT: [GdtEntry; 9] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // Kernel data - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // Kernel TLS - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // User (32-bit) code - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // User data - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // User FS (for TLS) - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // User GS (for TLS) - GdtEntry::new(0, 0xFFFFF, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0xFFFFF, + GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE, + ), // TSS GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0), ]; @@ -96,9 +140,10 @@ pub unsafe fn pcr() -> *mut ProcessorControlRegion { #[cfg(feature = "pti")] pub unsafe fn set_tss_stack(stack: usize) { - use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK}; + use super::pti::{PTI_CONTEXT_STACK, PTI_CPU_STACK}; addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16); - addr_of_mut!((*pcr()).tss.0.esp0).write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32); + addr_of_mut!((*pcr()).tss.0.esp0) + .write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32); PTI_CONTEXT_STACK = stack; } @@ -128,7 +173,8 @@ pub unsafe fn init() { /// Initialize GDT and configure percpu. pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR frame"); - let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); + let pcr = + &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); pcr.self_ref = pcr as *const _ as usize; pcr.gdt = BASE_GDT; @@ -179,7 +225,7 @@ pub struct GdtEntry { pub offsetm: u8, pub access: u8, pub flags_limith: u8, - pub offseth: u8 + pub offseth: u8, } impl GdtEntry { @@ -190,14 +236,12 @@ impl GdtEntry { offsetm: (offset >> 16) as u8, access, flags_limith: flags & 0xF0 | ((limit >> 16) as u8) & 0x0F, - offseth: (offset >> 24) as u8 + offseth: (offset >> 24) as u8, } } pub fn offset(&self) -> u32 { - (self.offsetl as u32) | - ((self.offsetm as u32) << 16) | - ((self.offseth as u32) << 24) + (self.offsetl as u32) | ((self.offsetm as u32) << 16) | ((self.offseth as u32) << 24) } pub fn set_offset(&mut self, offset: u32) { diff --git a/src/arch/x86/idt.rs b/src/arch/x86/idt.rs index 73aeededf7..42a765aec9 100644 --- a/src/arch/x86/idt.rs +++ b/src/arch/x86/idt.rs @@ -1,21 +1,24 @@ -use core::num::NonZeroU8; -use core::sync::atomic::{AtomicU32, Ordering}; -use core::mem; +use core::{ + mem, + num::NonZeroU8, + sync::atomic::{AtomicU32, Ordering}, +}; use alloc::boxed::Box; use hashbrown::HashMap; -use x86::segmentation::Descriptor as X86IdtEntry; -use x86::dtables::{self, DescriptorTablePointer}; +use x86::{ + dtables::{self, DescriptorTablePointer}, + segmentation::Descriptor as X86IdtEntry, +}; -use crate::{interrupt::*, LogicalCpuId}; -use crate::ipi::IpiKind; +use crate::{interrupt::*, ipi::IpiKind, LogicalCpuId}; use spin::RwLock; pub static mut INIT_IDTR: DescriptorTablePointer = DescriptorTablePointer { limit: 0, - base: 0 as *const X86IdtEntry + base: 0 as *const X86IdtEntry, }; pub type IdtEntries = [IdtEntry; 256]; @@ -46,7 +49,8 @@ impl Idt { let byte_index = index / 32; let bit = index % 32; - { &self.reservations[usize::from(byte_index)] }.fetch_or(u32::from(reserved) << bit, Ordering::AcqRel); + { &self.reservations[usize::from(byte_index)] } + .fetch_or(u32::from(reserved) << bit, Ordering::AcqRel); } #[inline] pub fn is_reserved_mut(&mut self, index: u8) -> bool { @@ -61,7 +65,8 @@ impl Idt { let byte_index = index / 32; let bit = index % 32; - *{ &mut self.reservations[usize::from(byte_index)] }.get_mut() |= u32::from(reserved) << bit; + *{ &mut self.reservations[usize::from(byte_index)] }.get_mut() |= + u32::from(reserved) << bit; } } @@ -75,7 +80,18 @@ pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool { let byte_index = index / 32; let bit = index % 32; - { &IDTS.read().as_ref().unwrap().get(&cpu_id).unwrap().reservations[usize::from(byte_index)] }.load(Ordering::Acquire) & (1 << bit) != 0 + { + &IDTS + .read() + .as_ref() + .unwrap() + .get(&cpu_id) + .unwrap() + .reservations[usize::from(byte_index)] + } + .load(Ordering::Acquire) + & (1 << bit) + != 0 } #[inline] @@ -83,13 +99,22 @@ pub fn set_reserved(cpu_id: LogicalCpuId, index: u8, reserved: bool) { let byte_index = index / 32; let bit = index % 32; - { &IDTS.read().as_ref().unwrap().get(&cpu_id).unwrap().reservations[usize::from(byte_index)] }.fetch_or(u32::from(reserved) << bit, Ordering::AcqRel); + { + &IDTS + .read() + .as_ref() + .unwrap() + .get(&cpu_id) + .unwrap() + .reservations[usize::from(byte_index)] + } + .fetch_or(u32::from(reserved) << bit, Ordering::AcqRel); } pub fn allocate_interrupt() -> Option { let cpu_id = crate::cpu_id(); for number in 50..=254 { - if ! is_reserved(cpu_id, number) { + if !is_reserved(cpu_id, number) { set_reserved(cpu_id, number, true); return Some(unsafe { NonZeroU8::new_unchecked(number) }); } @@ -120,8 +145,14 @@ pub unsafe fn init() { const fn new_idt_reservations() -> [AtomicU32; 8] { [ - AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0), - AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0) + AtomicU32::new(0), + AtomicU32::new(0), + AtomicU32::new(0), + AtomicU32::new(0), + AtomicU32::new(0), + AtomicU32::new(0), + AtomicU32::new(0), + AtomicU32::new(0), ] } @@ -133,7 +164,9 @@ pub unsafe fn init_paging_post_heap(is_bsp: bool, cpu_id: LogicalCpuId) { if is_bsp { idts_btree.insert(cpu_id, &mut INIT_BSP_IDT); } else { - let idt = idts_btree.entry(cpu_id).or_insert_with(|| Box::leak(Box::new(Idt::new()))); + let idt = idts_btree + .entry(cpu_id) + .or_insert_with(|| Box::leak(Box::new(Idt::new()))); init_generic(is_bsp, idt); } } @@ -238,7 +271,6 @@ pub unsafe fn init_generic(is_bsp: bool, idt: &mut Idt) { current_idt[48].set_func(irq::lapic_timer); current_idt[49].set_func(irq::lapic_error); - // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error. *current_reservations[1].get_mut() |= 0x0003_FFFF; } else { @@ -309,7 +341,11 @@ impl IdtEntry { } pub fn set_ist(&mut self, ist: u8) { - assert_eq!(ist & 0x07, ist, "interrupt stack table must be within 0..=7"); + assert_eq!( + ist & 0x07, + ist, + "interrupt stack table must be within 0..=7" + ); self.zero &= 0xF8; self.zero |= ist; } @@ -321,7 +357,7 @@ impl IdtEntry { } // A function to set the offset more easily - pub fn set_func(&mut self, func: unsafe extern fn()) { + pub fn set_func(&mut self, func: unsafe extern "C" fn()) { self.set_flags(IdtFlags::PRESENT | IdtFlags::RING_0 | IdtFlags::INTERRUPT); self.set_offset((crate::gdt::GDT_KERNEL_CODE as u16) << 3, func as usize); } diff --git a/src/arch/x86/interrupt/exception.rs b/src/arch/x86/interrupt/exception.rs index d0e8f437f5..c70665ff93 100644 --- a/src/arch/x86/interrupt/exception.rs +++ b/src/arch/x86/interrupt/exception.rs @@ -1,18 +1,12 @@ use rmm::TableKind; use x86::irq::PageFaultError; -use crate::memory::GenericPfFlags; -use crate::paging::VirtualAddress; use crate::{ - interrupt::stack_trace, - ptrace, - syscall::flag::*, - - interrupt_stack, - interrupt_error, + interrupt::stack_trace, interrupt_error, interrupt_stack, memory::GenericPfFlags, + paging::VirtualAddress, ptrace, syscall::flag::*, }; -extern { +extern "C" { fn ksignal(signal: usize); } @@ -140,11 +134,26 @@ interrupt_error!(page, |stack| { let arch_flags = PageFaultError::from_bits_truncate(stack.code as u32); let mut generic_flags = GenericPfFlags::empty(); - generic_flags.set(GenericPfFlags::PRESENT, arch_flags.contains(PageFaultError::P)); - generic_flags.set(GenericPfFlags::INVOLVED_WRITE, arch_flags.contains(PageFaultError::WR)); - generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, arch_flags.contains(PageFaultError::US)); - generic_flags.set(GenericPfFlags::INVL, arch_flags.contains(PageFaultError::RSVD)); - generic_flags.set(GenericPfFlags::INSTR_NOT_DATA, arch_flags.contains(PageFaultError::ID)); + generic_flags.set( + GenericPfFlags::PRESENT, + arch_flags.contains(PageFaultError::P), + ); + generic_flags.set( + GenericPfFlags::INVOLVED_WRITE, + arch_flags.contains(PageFaultError::WR), + ); + generic_flags.set( + GenericPfFlags::USER_NOT_SUPERVISOR, + arch_flags.contains(PageFaultError::US), + ); + generic_flags.set( + GenericPfFlags::INVL, + arch_flags.contains(PageFaultError::RSVD), + ); + generic_flags.set( + GenericPfFlags::INSTR_NOT_DATA, + arch_flags.contains(PageFaultError::ID), + ); if crate::memory::page_fault_handler(&mut stack.inner, generic_flags, cr2).is_err() { println!("Page fault: {:>08X} {:#?}", cr2.data(), arch_flags); diff --git a/src/arch/x86/interrupt/handler.rs b/src/arch/x86/interrupt/handler.rs index 1476027b43..b6f911c1e1 100644 --- a/src/arch/x86/interrupt/handler.rs +++ b/src/arch/x86/interrupt/handler.rs @@ -1,7 +1,6 @@ use core::mem; -use crate::memory::ArchIntCtx; -use crate::syscall::IntRegisters; +use crate::{memory::ArchIntCtx, syscall::IntRegisters}; use super::super::flags::*; @@ -50,9 +49,8 @@ pub struct IretRegisters { // The following will only be present if interrupt is raised from another // privilege ring. Otherwise, they are undefined values. // ---- - pub esp: usize, - pub ss: usize + pub ss: usize, } impl IretRegisters { @@ -169,61 +167,73 @@ impl InterruptErrorStack { #[macro_export] macro_rules! push_scratch { - () => { " + () => { + " // Push scratch registers (minus eax) push ecx push edx - " }; + " + }; } #[macro_export] macro_rules! pop_scratch { - () => { " + () => { + " // Pop scratch registers pop edx pop ecx pop eax - " }; + " + }; } #[macro_export] macro_rules! push_preserved { - () => { " + () => { + " // Push preserved registers push ebx push edi push esi push ebp - " }; + " + }; } #[macro_export] macro_rules! pop_preserved { - () => { " + () => { + " // Pop preserved registers pop ebp pop esi pop edi pop ebx - " }; + " + }; } // Must always happen after push_scratch macro_rules! enter_gs { - () => { " + () => { + " // Enter kernel GS segment mov ecx, gs push ecx mov ecx, 0x18 mov gs, ecx - " } + " + }; } // Must always happen before pop_scratch macro_rules! exit_gs { - () => { " + () => { + " // Exit kernel GS segment pop ecx mov gs, ecx - " } + " + }; } #[macro_export] @@ -411,13 +421,16 @@ macro_rules! interrupt_error { } #[naked] unsafe extern "C" fn usercopy_trampoline() { - core::arch::asm!(" + core::arch::asm!( + " pop esi pop edi mov eax, 1 ret - ", options(noreturn)); + ", + options(noreturn) + ); } impl ArchIntCtx for InterruptStack { diff --git a/src/arch/x86/interrupt/ipi.rs b/src/arch/x86/interrupt/ipi.rs index 013dd23b77..a26f8dbd55 100644 --- a/src/arch/x86/interrupt/ipi.rs +++ b/src/arch/x86/interrupt/ipi.rs @@ -1,7 +1,6 @@ use x86::tlb; -use crate::context; -use crate::device::local_apic::LOCAL_APIC; +use crate::{context, device::local_apic::LOCAL_APIC}; interrupt!(wakeup, || { LOCAL_APIC.eoi(); diff --git a/src/arch/x86/interrupt/irq.rs b/src/arch/x86/interrupt/irq.rs index 8fc7cafa08..4531ba3db0 100644 --- a/src/arch/x86/interrupt/irq.rs +++ b/src/arch/x86/interrupt/irq.rs @@ -2,14 +2,21 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::vec::Vec; -use crate::{interrupt, interrupt_stack}; -use crate::context::timeout; -use crate::device::{local_apic, ioapic, pic, pit}; -use crate::device::serial::{COM1, COM2}; -use crate::ipi::{ipi, IpiKind, IpiTarget}; -use crate::scheme::debug::{debug_input, debug_notify}; -use crate::scheme::serio::serio_input; -use crate::{context, time}; +use crate::{ + context, + context::timeout, + device::{ + ioapic, local_apic, pic, pit, + serial::{COM1, COM2}, + }, + interrupt, interrupt_stack, + ipi::{ipi, IpiKind, IpiTarget}, + scheme::{ + debug::{debug_input, debug_notify}, + serio::serio_input, + }, + time, +}; #[repr(u8)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -33,9 +40,13 @@ pub fn spurious_count() -> usize { pub fn spurious_irq_resource() -> syscall::Result> { match irq_method() { IrqMethod::Apic => Ok(Vec::from(&b"(not implemented for APIC yet)"[..])), - IrqMethod::Pic => { - Ok(format!("{}\tIRQ7\n{}\tIRQ15\n{}\ttotal\n", spurious_count_irq7(), spurious_count_irq15(), spurious_count()).into_bytes()) - } + IrqMethod::Pic => Ok(format!( + "{}\tIRQ7\n{}\tIRQ15\n{}\ttotal\n", + spurious_count_irq7(), + spurious_count_irq15(), + spurious_count() + ) + .into_bytes()), } } @@ -55,7 +66,7 @@ fn irq_method() -> IrqMethod { } } -extern { +extern "C" { // triggers irq scheme fn irq_trigger(irq: u8); } @@ -64,7 +75,11 @@ extern { /// scheme user unmasks it ("acknowledges" it). unsafe fn trigger(irq: u8) { match irq_method() { - IrqMethod::Pic => if irq < 16 { pic_mask(irq) }, + IrqMethod::Pic => { + if irq < 16 { + pic_mask(irq) + } + } IrqMethod::Apic => ioapic_mask(irq), } irq_trigger(irq); @@ -74,7 +89,11 @@ unsafe fn trigger(irq: u8) { /// processed the IRQ. pub unsafe fn acknowledge(irq: usize) { match irq_method() { - IrqMethod::Pic => if irq < 16 { pic_unmask(irq) }, + IrqMethod::Pic => { + if irq < 16 { + pic_unmask(irq) + } + } IrqMethod::Apic => ioapic_unmask(irq), } } @@ -82,7 +101,11 @@ pub unsafe fn acknowledge(irq: usize) { /// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one. pub unsafe fn eoi(irq: u8) { match irq_method() { - IrqMethod::Pic => if irq < 16 { pic_eoi(irq) }, + IrqMethod::Pic => { + if irq < 16 { + pic_eoi(irq) + } + } IrqMethod::Apic => lapic_eoi(), } } @@ -242,7 +265,7 @@ interrupt!(ata2, || { if irq_method() == IrqMethod::Pic && pic::SLAVE.isr() & (1 << 7) == 0 { SPURIOUS_COUNT_IRQ15.fetch_add(1, Ordering::Relaxed); pic::MASTER.ack(); - return + return; } trigger(15); eoi(15); @@ -254,7 +277,10 @@ interrupt!(lapic_timer, || { }); interrupt!(lapic_error, || { - println!("Local apic internal error: ESR={:#0x}", local_apic::LOCAL_APIC.esr()); + println!( + "Local apic internal error: ESR={:#0x}", + local_apic::LOCAL_APIC.esr() + ); lapic_eoi(); }); diff --git a/src/arch/x86/interrupt/mod.rs b/src/arch/x86/interrupt/mod.rs index ebd013f23d..7b212b07b4 100644 --- a/src/arch/x86/interrupt/mod.rs +++ b/src/arch/x86/interrupt/mod.rs @@ -9,11 +9,12 @@ pub mod irq; pub mod syscall; pub mod trace; -pub use self::handler::InterruptStack; -pub use self::trace::stack_trace; +pub use self::{handler::InterruptStack, trace::stack_trace}; -pub use super::idt::{available_irqs_iter, is_reserved, set_reserved}; -pub use super::device::local_apic::bsp_apic_id; +pub use super::{ + device::local_apic::bsp_apic_id, + idt::{available_irqs_iter, is_reserved, set_reserved}, +}; /// Clear interrupts #[inline(always)] @@ -53,5 +54,7 @@ pub unsafe fn halt() { /// Safe because it is similar to a NOP, and has no memory effects #[inline(always)] pub fn pause() { - unsafe { core::arch::asm!("pause", options(nomem, nostack)); } + unsafe { + core::arch::asm!("pause", options(nomem, nostack)); + } } diff --git a/src/arch/x86/interrupt/syscall.rs b/src/arch/x86/interrupt/syscall.rs index 0267647158..a341b44e64 100644 --- a/src/arch/x86/interrupt/syscall.rs +++ b/src/arch/x86/interrupt/syscall.rs @@ -1,9 +1,7 @@ use crate::{ arch::{gdt, interrupt::InterruptStack}, - context, - ptrace, - syscall, - syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_PRE_SYSCALL, PTRACE_STOP_POST_SYSCALL}, + context, ptrace, syscall, + syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL}, }; use core::mem::offset_of; use x86::{bits32::task::TaskStateSegment, msr, segmentation::SegmentSelector}; @@ -25,33 +23,44 @@ macro_rules! with_interrupt_stack { } ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); - }} + }}; } interrupt_stack!(syscall, |stack| { with_interrupt_stack!(|stack| { let scratch = &stack.scratch; let preserved = &stack.preserved; - syscall::syscall(scratch.eax, preserved.ebx, scratch.ecx, scratch.edx, preserved.esi, preserved.edi, stack) + syscall::syscall( + scratch.eax, + preserved.ebx, + scratch.ecx, + scratch.edx, + preserved.esi, + preserved.edi, + stack, + ) }) }); #[naked] pub unsafe extern "C" fn clone_ret() { - core::arch::asm!(concat!( - // The address of this instruction is injected by `clone` in process.rs, on - // top of the stack syscall->inner in this file, which is done using the ebp - // register we save there. - // - // The top of our stack here is the address pointed to by ebp, which is: - // - // - the previous ebp - // - the return location - // - // Our goal is to return from the parent function, inner, so we restore - // ebp... - "pop ebp\n", - // ...and we return to the address at the top of the stack - "ret\n", - ), options(noreturn)); + core::arch::asm!( + concat!( + // The address of this instruction is injected by `clone` in process.rs, on + // top of the stack syscall->inner in this file, which is done using the ebp + // register we save there. + // + // The top of our stack here is the address pointed to by ebp, which is: + // + // - the previous ebp + // - the return location + // + // Our goal is to return from the parent function, inner, so we restore + // ebp... + "pop ebp\n", + // ...and we return to the address at the top of the stack + "ret\n", + ), + options(noreturn) + ); } diff --git a/src/arch/x86/interrupt/trace.rs b/src/arch/x86/interrupt/trace.rs index f99a98eeb6..49db2098f3 100644 --- a/src/arch/x86/interrupt/trace.rs +++ b/src/arch/x86/interrupt/trace.rs @@ -3,7 +3,7 @@ use core::{mem, str}; use goblin::elf::sym; use rustc_demangle::demangle; -use crate::{paging::{KernelMapper, VirtualAddress}}; +use crate::paging::{KernelMapper, VirtualAddress}; /// Get a stack trace //TODO: Check for stack being mapped before dereferencing @@ -45,11 +45,12 @@ pub unsafe fn stack_trace() { //TODO: Do not create Elf object for every symbol lookup #[inline(never)] pub unsafe fn symbol_trace(addr: usize) { - use core::slice; - use core::sync::atomic::Ordering; + use core::{slice, sync::atomic::Ordering}; - use crate::elf::Elf; - use crate::start::{KERNEL_BASE, KERNEL_SIZE}; + use crate::{ + elf::Elf, + start::{KERNEL_BASE, KERNEL_SIZE}, + }; let kernel_ptr = (KERNEL_BASE.load(Ordering::SeqCst) + crate::PHYS_OFFSET) as *const u8; let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst)); @@ -65,10 +66,14 @@ pub unsafe fn symbol_trace(addr: usize) { if let Some(symbols) = elf.symbols() { for sym in symbols { if sym::st_type(sym.st_info) == sym::STT_FUNC - && addr >= sym.st_value as usize - && addr < (sym.st_value + sym.st_size) as usize + && addr >= sym.st_value as usize + && addr < (sym.st_value + sym.st_size) as usize { - println!(" {:>016X}+{:>04X}", sym.st_value, addr - sym.st_value as usize); + println!( + " {:>016X}+{:>04X}", + sym.st_value, + addr - sym.st_value as usize + ); if let Some(strtab) = strtab_opt { let start = strtab.sh_offset as usize + sym.st_name as usize; @@ -82,7 +87,7 @@ pub unsafe fn symbol_trace(addr: usize) { } if end > start { - let sym_slice = &elf.data[start .. end - 1]; + let sym_slice = &elf.data[start..end - 1]; if let Ok(sym_name) = str::from_utf8(sym_slice) { println!(" {:#}", demangle(sym_name)); } diff --git a/src/arch/x86/mod.rs b/src/arch/x86/mod.rs index c1448191dc..4528aa3272 100644 --- a/src/arch/x86/mod.rs +++ b/src/arch/x86/mod.rs @@ -42,9 +42,9 @@ pub mod stop; pub mod time; -pub use ::rmm::X86Arch as CurrentRmmArch; +use crate::{memory::PAGE_SIZE, Bootstrap}; use ::rmm::Arch; -use crate::{Bootstrap, memory::PAGE_SIZE}; +pub use ::rmm::X86Arch as CurrentRmmArch; // Flags pub mod flags { @@ -56,7 +56,8 @@ pub mod flags { #[naked] #[link_section = ".usercopy-fns"] pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { - core::arch::asm!(" + core::arch::asm!( + " push edi push esi @@ -70,12 +71,17 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - xor eax, eax ret - ", options(noreturn)); + ", + options(noreturn) + ); } pub use arch_copy_to_user as arch_copy_from_user; pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] { - core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) + core::slice::from_raw_parts( + CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, + bootstrap.page_count * PAGE_SIZE, + ) } pub const KFX_SIZE: usize = 512; diff --git a/src/arch/x86/paging/mapper.rs b/src/arch/x86/paging/mapper.rs index 9f7659b4e6..1543c6ff80 100644 --- a/src/arch/x86/paging/mapper.rs +++ b/src/arch/x86/paging/mapper.rs @@ -4,16 +4,22 @@ use super::RmmA; pub use rmm::{Flusher, PageFlush, PageFlushAll}; -pub struct InactiveFlusher { _inner: () } +pub struct InactiveFlusher { + _inner: (), +} impl InactiveFlusher { // TODO: cpu id - pub fn new() -> Self { Self { _inner: () } } + pub fn new() -> Self { + Self { _inner: () } + } } impl Flusher for InactiveFlusher { fn consume(&mut self, flush: PageFlush) { // TODO: Push to TLB "mailbox" or tell it to reload CR3 if there are too many entries. - unsafe { flush.ignore(); } + unsafe { + flush.ignore(); + } } } impl Drop for InactiveFlusher { diff --git a/src/arch/x86/paging/mod.rs b/src/arch/x86/paging/mod.rs index 59de9519de..e8bb147055 100644 --- a/src/arch/x86/paging/mod.rs +++ b/src/arch/x86/paging/mod.rs @@ -6,15 +6,8 @@ use x86::msr; use self::mapper::PageFlushAll; -pub use rmm::{ - Arch as RmmArch, - Flusher, - PageFlags, - PhysicalAddress, - TableKind, - VirtualAddress, -}; pub use super::CurrentRmmArch as RmmA; +pub use rmm::{Arch as RmmArch, Flusher, PageFlags, PhysicalAddress, TableKind, VirtualAddress}; pub type PageMapper = rmm::PageMapper; pub use crate::rmm::KernelMapper; @@ -111,7 +104,10 @@ impl Page { } pub fn range_inclusive(start: Page, r#final: Page) -> PageIter { - PageIter { start, end: r#final.next() } + PageIter { + start, + end: r#final.next(), + } } pub fn range_exclusive(start: Page, end: Page) -> PageIter { PageIter { start, end } diff --git a/src/arch/x86/pti.rs b/src/arch/x86/pti.rs index 0f5cc07aad..c1e9422c84 100644 --- a/src/arch/x86/pti.rs +++ b/src/arch/x86/pti.rs @@ -4,9 +4,9 @@ use core::ptr; #[cfg(feature = "pti")] use crate::memory::Frame; #[cfg(feature = "pti")] -use crate::paging::ActivePageTable; -#[cfg(feature = "pti")] use crate::paging::entry::EntryFlags; +#[cfg(feature = "pti")] +use crate::paging::ActivePageTable; #[cfg(feature = "pti")] #[thread_local] @@ -26,11 +26,7 @@ unsafe fn switch_stack(old: usize, new: usize) { let new_rsp = new - offset_rsp; - ptr::copy_nonoverlapping( - old_rsp as *const u8, - new_rsp as *mut u8, - offset_rsp - ); + ptr::copy_nonoverlapping(old_rsp as *const u8, new_rsp as *mut u8, offset_rsp); asm!("", out("rsp") new_rsp); } @@ -53,14 +49,20 @@ pub unsafe fn map() { // } // Switch to per-context stack - switch_stack(PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len(), PTI_CONTEXT_STACK); + switch_stack( + PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len(), + PTI_CONTEXT_STACK, + ); } #[cfg(feature = "pti")] #[inline(always)] pub unsafe extern "C" fn unmap() { // Switch to per-CPU stack - switch_stack(PTI_CONTEXT_STACK, PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()); + switch_stack( + PTI_CONTEXT_STACK, + PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len(), + ); // { // let mut active_table = unsafe { ActivePageTable::new() }; diff --git a/src/arch/x86/rmm.rs b/src/arch/x86/rmm.rs index 941ab7ace2..7afb8c6bb9 100644 --- a/src/arch/x86/rmm.rs +++ b/src/arch/x86/rmm.rs @@ -1,26 +1,12 @@ use core::{ cell::SyncUnsafeCell, - cmp, - mem, - slice, + cmp, mem, slice, sync::atomic::{self, AtomicUsize, Ordering}, }; use rmm::{ - KILOBYTE, + Arch, BuddyAllocator, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, MemoryArea, + PageEntry, PageFlags, PageMapper, PhysicalAddress, TableKind, VirtualAddress, KILOBYTE, MEGABYTE, - Arch, - BuddyAllocator, - BumpAllocator, - FrameAllocator, - FrameCount, - FrameUsage, - MemoryArea, - PageEntry, - PageFlags, - PageMapper, - PhysicalAddress, - TableKind, - VirtualAddress, }; use spin::Mutex; @@ -65,11 +51,16 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { unsafe fn inner( areas: &'static [MemoryArea], - kernel_base: usize, kernel_size_aligned: usize, - stack_base: usize, stack_size_aligned: usize, - env_base: usize, env_size_aligned: usize, - acpi_base: usize, acpi_size_aligned: usize, - initfs_base: usize, initfs_size_aligned: usize, + kernel_base: usize, + kernel_size_aligned: usize, + stack_base: usize, + stack_size_aligned: usize, + env_base: usize, + env_size_aligned: usize, + acpi_base: usize, + acpi_size_aligned: usize, + initfs_base: usize, + initfs_size_aligned: usize, ) -> BuddyAllocator { // First, calculate how much memory we have let mut size = 0; @@ -86,17 +77,20 @@ unsafe fn inner( let mut bump_allocator = BumpAllocator::::new(areas, 0); { - let mut mapper = PageMapper::::create( - TableKind::Kernel, - &mut bump_allocator - ).expect("failed to create Mapper"); + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); // Pre-allocate all kernel PD entries so that when the page table is copied, // these entries are synced between processes for i in 512..1024 { - let phys = mapper.allocator_mut().allocate_one().expect("failed to map page table"); + let phys = mapper + .allocator_mut() + .allocate_one() + .expect("failed to map page table"); let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE; - mapper.table().set_entry(i, PageEntry::new(phys.data() | flags)); + mapper + .table() + .set_entry(i, PageEntry::new(phys.data() | flags)); } // Map all physical areas at PHYS_OFFSET @@ -105,11 +99,9 @@ unsafe fn inner( let phys = area.base.add(i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -119,19 +111,15 @@ unsafe fn inner( let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE); let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table let virt = A::phys_to_virt(phys); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } @@ -141,11 +129,9 @@ unsafe fn inner( let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } }; @@ -158,8 +144,8 @@ unsafe fn inner( // Ensure graphical debug region remains paged #[cfg(feature = "graphical_debug")] { - use crate::devices::graphical_debug::FRAMEBUFFER; use super::paging::entry::EntryFlags; + use crate::devices::graphical_debug::FRAMEBUFFER; let (phys, virt, size) = *FRAMEBUFFER.lock(); @@ -167,14 +153,13 @@ unsafe fn inner( for i in 0..pages { let phys = PhysicalAddress::new(phys + i * A::PAGE_SIZE); let virt = VirtualAddress::new(virt + i * A::PAGE_SIZE); - let flags = PageFlags::new().write(true) + let flags = PageFlags::new() + .write(true) // Write combining flag .custom_flag(EntryFlags::HUGE_PAGE.bits(), true); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -194,7 +179,10 @@ unsafe fn inner( // Create the physical memory map let offset = bump_allocator.offset(); - log::info!("Permanently used: {} KB", (offset + (KILOBYTE - 1)) / KILOBYTE); + log::info!( + "Permanently used: {} KB", + (offset + (KILOBYTE - 1)) / KILOBYTE + ); BuddyAllocator::::new(bump_allocator).expect("failed to create BuddyAllocator") } @@ -238,10 +226,12 @@ impl core::fmt::Debug for LockedAllocator { } } -static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new([MemoryArea { - base: PhysicalAddress::new(0), - size: 0, -}; 512]); +static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new( + [MemoryArea { + base: PhysicalAddress::new(0), + size: 0, + }; 512], +); static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); pub fn areas() -> &'static [MemoryArea] { @@ -271,7 +261,12 @@ pub struct KernelMapper { impl KernelMapper { fn lock_inner(current_processor: usize) -> bool { loop { - match LOCK_OWNER.compare_exchange_weak(NO_PROCESSOR, current_processor, Ordering::Acquire, Ordering::Relaxed) { + match LOCK_OWNER.compare_exchange_weak( + NO_PROCESSOR, + current_processor, + Ordering::Acquire, + Ordering::Relaxed, + ) { Ok(_) => break, // already owned by this hardware thread Err(id) if id == current_processor => break, @@ -285,15 +280,20 @@ impl KernelMapper { prev_count > 0 } - pub unsafe fn lock_for_manual_mapper(current_processor: LogicalCpuId, mapper: crate::paging::PageMapper) -> Self { + pub unsafe fn lock_for_manual_mapper( + current_processor: LogicalCpuId, + mapper: crate::paging::PageMapper, + ) -> Self { let ro = Self::lock_inner(current_processor.get() as usize); - Self { - mapper, - ro, - } + Self { mapper, ro } } pub fn lock_manually(current_processor: LogicalCpuId) -> Self { - unsafe { Self::lock_for_manual_mapper(current_processor, PageMapper::current(TableKind::Kernel, FRAME_ALLOCATOR)) } + unsafe { + Self::lock_for_manual_mapper( + current_processor, + PageMapper::current(TableKind::Kernel, FRAME_ALLOCATOR), + ) + } } pub fn lock() -> Self { Self::lock_manually(crate::cpu_id()) @@ -328,12 +328,18 @@ impl Drop for KernelMapper { } pub unsafe fn init( - kernel_base: usize, kernel_size: usize, - stack_base: usize, stack_size: usize, - env_base: usize, env_size: usize, - acpi_base: usize, acpi_size: usize, - areas_base: usize, areas_size: usize, - initfs_base: usize, initfs_size: usize, + kernel_base: usize, + kernel_size: usize, + stack_base: usize, + stack_size: usize, + env_base: usize, + env_size: usize, + acpi_base: usize, + acpi_size: usize, + areas_base: usize, + areas_size: usize, + initfs_base: usize, + initfs_size: usize, ) { type A = RmmA; @@ -341,26 +347,26 @@ pub unsafe fn init( let real_size = 0x100000; let real_end = real_base + real_size; - let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let kernel_end = kernel_base + kernel_size_aligned; - let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let stack_end = stack_base + stack_size_aligned; - let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let env_end = env_base + env_size_aligned; - let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let acpi_end = acpi_base + acpi_size_aligned; - let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let initfs_end = initfs_base + initfs_size_aligned; let areas = &mut *AREAS.get(); let bootloader_areas = slice::from_raw_parts( areas_base as *const BootloaderMemoryEntry, - areas_size / mem::size_of::() + areas_size / mem::size_of::(), ); // Copy memory map from bootloader location, and page align it @@ -393,44 +399,86 @@ pub unsafe fn init( // Ensure real-mode areas are not used if base < real_end && base + size > real_base { - log::warn!("{:X}:{:X} overlaps with real mode {:X}:{:X}", base, size, real_base, real_size); + log::warn!( + "{:X}:{:X} overlaps with real mode {:X}:{:X}", + base, + size, + real_base, + real_size + ); new_base = cmp::max(new_base, real_end); } // Ensure kernel areas are not used if base < kernel_end && base + size > kernel_base { - log::warn!("{:X}:{:X} overlaps with kernel {:X}:{:X}", base, size, kernel_base, kernel_size); + log::warn!( + "{:X}:{:X} overlaps with kernel {:X}:{:X}", + base, + size, + kernel_base, + kernel_size + ); new_base = cmp::max(new_base, kernel_end); } // Ensure stack areas are not used if base < stack_end && base + size > stack_base { - log::warn!("{:X}:{:X} overlaps with stack {:X}:{:X}", base, size, stack_base, stack_size); + log::warn!( + "{:X}:{:X} overlaps with stack {:X}:{:X}", + base, + size, + stack_base, + stack_size + ); new_base = cmp::max(new_base, stack_end); } // Ensure env areas are not used if base < env_end && base + size > env_base { - log::warn!("{:X}:{:X} overlaps with env {:X}:{:X}", base, size, env_base, env_size); + log::warn!( + "{:X}:{:X} overlaps with env {:X}:{:X}", + base, + size, + env_base, + env_size + ); new_base = cmp::max(new_base, env_end); } // Ensure acpi areas are not used if base < acpi_end && base + size > acpi_base { - log::warn!("{:X}:{:X} overlaps with acpi {:X}:{:X}", base, size, acpi_base, acpi_size); + log::warn!( + "{:X}:{:X} overlaps with acpi {:X}:{:X}", + base, + size, + acpi_base, + acpi_size + ); new_base = cmp::max(new_base, acpi_end); } // Ensure initfs areas are not used if base < initfs_end && base + size > initfs_base { - log::warn!("{:X}:{:X} overlaps with initfs {:X}:{:X}", base, size, initfs_base, initfs_size); + log::warn!( + "{:X}:{:X} overlaps with initfs {:X}:{:X}", + base, + size, + initfs_base, + initfs_size + ); new_base = cmp::max(new_base, initfs_end); } if new_base != base { let end = base + size; let new_size = end.checked_sub(new_base).unwrap_or(0); - log::info!("{:X}:{:X} moved to {:X}:{:X}", base, size, new_base, new_size); + log::info!( + "{:X}:{:X} moved to {:X}:{:X}", + base, + size, + new_base, + new_size + ); base = new_base; size = new_size; } @@ -443,7 +491,13 @@ pub unsafe fn init( size = 0; // Skip area } else if base + size > physmap_size { let new_size = physmap_size.checked_sub(base).unwrap_or(0); - log::warn!("{:X}:{:X} outside of physmap, moved to {:X}:{:X}", base, size, base, new_size); + log::warn!( + "{:X}:{:X} outside of physmap, moved to {:X}:{:X}", + base, + size, + base, + new_size + ); size = new_size; } @@ -454,8 +508,18 @@ pub unsafe fn init( let other_end = other_base + other.size; if base < other_end && base + size > other_base { let new_base = cmp::min(base, other_base); - let new_size = cmp::max(base + size, other_end).checked_sub(new_base).unwrap_or(0); - log::warn!("{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", base, size, other_base, other.size, new_base, new_size); + let new_size = cmp::max(base + size, other_end) + .checked_sub(new_base) + .unwrap_or(0); + log::warn!( + "{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}", + base, + size, + other_base, + other.size, + new_base, + new_size + ); areas[other_i].base = PhysicalAddress::new(new_base); areas[other_i].size = new_size; size = 0; // Skip area @@ -475,11 +539,16 @@ pub unsafe fn init( let allocator = inner::( areas, - kernel_base, kernel_size_aligned, - stack_base, stack_size_aligned, - env_base, env_size_aligned, - acpi_base, acpi_size_aligned, - initfs_base, initfs_size_aligned, + kernel_base, + kernel_size_aligned, + stack_base, + stack_size_aligned, + env_base, + env_size_aligned, + acpi_base, + acpi_size_aligned, + initfs_base, + initfs_size_aligned, ); *INNER_ALLOCATOR.lock() = Some(allocator); } diff --git a/src/arch/x86/start.rs b/src/arch/x86/start.rs index c3dff61551..499f29d63a 100644 --- a/src/arch/x86/start.rs +++ b/src/arch/x86/start.rs @@ -2,24 +2,22 @@ /// It is increcibly unsafe, and should be minimal in nature /// It must create the IDT with the correct entries, those entries are /// defined in other files inside of the `arch` module - use core::slice; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering, AtomicU32}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use crate::{allocator, LogicalCpuId}; #[cfg(feature = "acpi")] use crate::acpi; -use crate::arch::pti; -use crate::arch::flags::*; -use crate::device; #[cfg(feature = "graphical_debug")] use crate::devices::graphical_debug; -use crate::gdt; -use crate::idt; -use crate::interrupt; -use crate::log::{self, info}; -use crate::memory; -use crate::paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKind}; +use crate::{ + allocator, + arch::{flags::*, pti}, + device, gdt, idt, interrupt, + log::{self, info}, + memory, + paging::{self, KernelMapper, PhysicalAddress, RmmA, RmmArch, TableKind}, + LogicalCpuId, +}; /// Test of zero values in BSS. static BSS_TEST_ZERO: usize = 0; @@ -68,7 +66,7 @@ pub struct KernelArgs { /// The entry to Rust, all things must be initialized #[no_mangle] -pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { +pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { let bootstrap = { let args = args_ptr.read(); @@ -82,7 +80,10 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { KERNEL_SIZE.store(args.kernel_size as usize, Ordering::SeqCst); // Convert env to slice - let env = slice::from_raw_parts((args.env_base as usize + crate::PHYS_OFFSET) as *const u8, args.env_size as usize); + let env = slice::from_raw_parts( + (args.env_base as usize + crate::PHYS_OFFSET) as *const u8, + args.env_size as usize, + ); // Set up graphical debug #[cfg(feature = "graphical_debug")] @@ -104,12 +105,36 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { }); info!("Redox OS starting..."); - info!("Kernel: {:X}:{:X}", { args.kernel_base }, { args.kernel_base } + { args.kernel_size }); - info!("Stack: {:X}:{:X}", { args.stack_base }, { args.stack_base } + { args.stack_size }); - info!("Env: {:X}:{:X}", { args.env_base }, { args.env_base } + { args.env_size }); - info!("RSDPs: {:X}:{:X}", { args.acpi_rsdps_base }, { args.acpi_rsdps_base } + { args.acpi_rsdps_size }); - info!("Areas: {:X}:{:X}", { args.areas_base }, { args.areas_base } + { args.areas_size }); - info!("Bootstrap: {:X}:{:X}", { args.bootstrap_base }, { args.bootstrap_base } + { args.bootstrap_size }); + info!( + "Kernel: {:X}:{:X}", + { args.kernel_base }, + { args.kernel_base } + { args.kernel_size } + ); + info!( + "Stack: {:X}:{:X}", + { args.stack_base }, + { args.stack_base } + { args.stack_size } + ); + info!( + "Env: {:X}:{:X}", + { args.env_base }, + { args.env_base } + { args.env_size } + ); + info!( + "RSDPs: {:X}:{:X}", + { args.acpi_rsdps_base }, + { args.acpi_rsdps_base } + { args.acpi_rsdps_size } + ); + info!( + "Areas: {:X}:{:X}", + { args.areas_base }, + { args.areas_base } + { args.areas_size } + ); + info!( + "Bootstrap: {:X}:{:X}", + { args.bootstrap_base }, + { args.bootstrap_base } + { args.bootstrap_size } + ); info!("Bootstrap entry point: {:X}", { args.bootstrap_entry }); // Set up GDT before paging @@ -120,18 +145,27 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Initialize RMM crate::arch::rmm::init( - args.kernel_base as usize, args.kernel_size as usize, - args.stack_base as usize, args.stack_size as usize, - args.env_base as usize, args.env_size as usize, - args.acpi_rsdps_base as usize, args.acpi_rsdps_size as usize, - args.areas_base as usize, args.areas_size as usize, - args.bootstrap_base as usize, args.bootstrap_size as usize, + args.kernel_base as usize, + args.kernel_size as usize, + args.stack_base as usize, + args.stack_size as usize, + args.env_base as usize, + args.env_size as usize, + args.acpi_rsdps_base as usize, + args.acpi_rsdps_size as usize, + args.areas_base as usize, + args.areas_size as usize, + args.bootstrap_base as usize, + args.bootstrap_size as usize, ); // Initialize paging paging::init(); // Set up GDT after paging with TLS - gdt::init_paging(args.stack_base as usize + args.stack_size as usize, LogicalCpuId::BSP); + gdt::init_paging( + args.stack_base as usize + args.stack_size as usize, + LogicalCpuId::BSP, + ); // Set up IDT idt::init_paging_bsp(); @@ -163,7 +197,10 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { #[cfg(feature = "acpi")] { acpi::init(if args.acpi_rsdps_base != 0 && args.acpi_rsdps_size > 0 { - Some(((args.acpi_rsdps_base as usize + crate::PHYS_OFFSET) as u64, args.acpi_rsdps_size as u64)) + Some(( + (args.acpi_rsdps_base as usize + crate::PHYS_OFFSET) as u64, + args.acpi_rsdps_size as u64, + )) } else { None }); @@ -183,7 +220,9 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { BSP_READY.store(true, Ordering::SeqCst); crate::Bootstrap { - base: crate::memory::Frame::containing_address(crate::paging::PhysicalAddress::new(args.bootstrap_base as usize)), + base: crate::memory::Frame::containing_address(crate::paging::PhysicalAddress::new( + args.bootstrap_base as usize, + )), page_count: (args.bootstrap_size as usize) / crate::memory::PAGE_SIZE, entry: args.bootstrap_entry as usize, env, @@ -202,7 +241,7 @@ pub struct KernelArgsAp { } /// Entry to rust for an AP -pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { +pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { let cpu_id = { let args = &*args_ptr; let cpu_id = LogicalCpuId::new(args.cpu_id as u32); @@ -240,7 +279,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { cpu_id }; - while ! BSP_READY.load(Ordering::SeqCst) { + while !BSP_READY.load(Ordering::SeqCst) { interrupt::pause(); } diff --git a/src/arch/x86/stop.rs b/src/arch/x86/stop.rs index 246d1ddde2..50ceab3ade 100644 --- a/src/arch/x86/stop.rs +++ b/src/arch/x86/stop.rs @@ -1,14 +1,10 @@ #[cfg(feature = "acpi")] -use crate::{ - context, - scheme::acpi, - time, -}; +use crate::{context, scheme::acpi, time}; use crate::syscall::io::{Io, Pio}; #[no_mangle] -pub unsafe extern fn kreset() -> ! { +pub unsafe extern "C" fn kreset() -> ! { println!("kreset"); // 8042 reset @@ -24,11 +20,14 @@ pub unsafe extern fn kreset() -> ! { pub unsafe fn emergency_reset() -> ! { // Use triple fault to guarantee reset - core::arch::asm!(" + core::arch::asm!( + " cli lidt cs:0 int $3 - ", options(noreturn)); + ", + options(noreturn) + ); } #[cfg(feature = "acpi")] @@ -36,7 +35,7 @@ fn userspace_acpi_shutdown() { log::info!("Notifying any potential ACPI driver"); // Tell whatever driver that handles ACPI, that it should enter the S5 state (i.e. // shutdown). - if ! acpi::register_kstop() { + if !acpi::register_kstop() { // There was no context to switch to. log::info!("No ACPI driver was alive to handle shutdown."); return; @@ -63,7 +62,7 @@ fn userspace_acpi_shutdown() { } #[no_mangle] -pub unsafe extern fn kstop() -> ! { +pub unsafe extern "C" fn kstop() -> ! { log::info!("Running kstop()"); #[cfg(feature = "acpi")] diff --git a/src/arch/x86_64/alternative.rs b/src/arch/x86_64/alternative.rs index 78cda9b6f5..1f94fcd095 100644 --- a/src/arch/x86_64/alternative.rs +++ b/src/arch/x86_64/alternative.rs @@ -5,9 +5,11 @@ use core::mem::size_of; use spin::Once; use x86::controlregs::{Cr4, Xcr0}; -use crate::context::memory::PageSpan; -use crate::cpuid::{has_ext_feat, cpuid_always, feature_info}; -use crate::paging::{KernelMapper, Page, PageFlags, PAGE_SIZE, VirtualAddress}; +use crate::{ + context::memory::PageSpan, + cpuid::{cpuid_always, feature_info, has_ext_feat}, + paging::{KernelMapper, Page, PageFlags, VirtualAddress, PAGE_SIZE}, +}; #[cfg(all(cpu_feature_never = "xsave", not(cpu_feature_never = "xsaveopt")))] compile_error!("cannot force-disable xsave without force-disabling xsaveopt"); @@ -31,7 +33,10 @@ pub unsafe fn early_init(bsp: bool) { let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset; assert_eq!(relocs_size % size_of::(), 0); - let relocs = core::slice::from_raw_parts(relocs_offset as *const AltReloc, relocs_size / size_of::()); + let relocs = core::slice::from_raw_parts( + relocs_offset as *const AltReloc, + relocs_size / size_of::(), + ); let mut enable = KcpuFeatures::empty(); @@ -62,12 +67,16 @@ pub unsafe fn early_init(bsp: bool) { #[cfg(not(cpu_feature_never = "xsave"))] if feature_info().has_xsave() { - use raw_cpuid::{ExtendedRegisterType, ExtendedRegisterStateLocation}; + use raw_cpuid::{ExtendedRegisterStateLocation, ExtendedRegisterType}; - x86::controlregs::cr4_write(x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE); + x86::controlregs::cr4_write( + x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE, + ); let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE; - let ext_state_info = cpuid_always().get_extended_state_info().expect("must be present if XSAVE is supported"); + let ext_state_info = cpuid_always() + .get_extended_state_info() + .expect("must be present if XSAVE is supported"); enable |= KcpuFeatures::XSAVE; enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt()); @@ -77,10 +86,13 @@ pub unsafe fn early_init(bsp: bool) { xcr0 |= Xcr0::XCR0_AVX_STATE; x86::controlregs::xcr0_write(xcr0); - let state = ext_state_info.iter().find(|state| { - state.register() == ExtendedRegisterType::Avx - && state.location() == ExtendedRegisterStateLocation::Xcr0 - }).expect("CPUID said AVX was supported but there's no state info"); + let state = ext_state_info + .iter() + .find(|state| { + state.register() == ExtendedRegisterType::Avx + && state.location() == ExtendedRegisterStateLocation::Xcr0 + }) + .expect("CPUID said AVX was supported but there's no state info"); if state.size() as usize != 16 * core::mem::size_of::() { log::warn!("Unusual AVX state size {}", state.size()); @@ -123,20 +135,37 @@ pub unsafe fn early_init(bsp: bool) { unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) { let mut mapper = KernelMapper::lock(); for reloc in relocs.iter().copied() { - let name = core::str::from_utf8(core::slice::from_raw_parts(reloc.name_start, reloc.name_len)).expect("invalid feature name"); + let name = core::str::from_utf8(core::slice::from_raw_parts( + reloc.name_start, + reloc.name_len, + )) + .expect("invalid feature name"); let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len); let dst_pages = PageSpan::between( Page::containing_address(VirtualAddress::new(reloc.code_start as usize)), - Page::containing_address(VirtualAddress::new((reloc.code_start as usize + reloc.origcode_len).next_multiple_of(PAGE_SIZE))), + Page::containing_address(VirtualAddress::new( + (reloc.code_start as usize + reloc.origcode_len).next_multiple_of(PAGE_SIZE), + )), ); for page in dst_pages.pages() { - mapper.remap(page.start_address(), PageFlags::new().write(true).execute(true).global(true)).unwrap().flush(); + mapper + .remap( + page.start_address(), + PageFlags::new().write(true).execute(true).global(true), + ) + .unwrap() + .flush(); } let code = core::slice::from_raw_parts_mut(reloc.code_start, reloc.padded_len); - log::trace!("feature {} current {:x?} altcode {:x?}", name, code, altcode); + log::trace!( + "feature {} current {:x?} altcode {:x?}", + name, + code, + altcode + ); let feature_is_enabled = match name { "smap" => enable.contains(KcpuFeatures::SMAP), @@ -170,7 +199,9 @@ unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) { &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], &[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], - &[0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00], + &[ + 0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, + ], ]; for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) { @@ -179,7 +210,13 @@ unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) { log::trace!("feature {} new {:x?} altcode {:x?}", name, code, altcode); for page in dst_pages.pages() { - mapper.remap(page.start_address(), PageFlags::new().write(false).execute(true).global(true)).unwrap().flush(); + mapper + .remap( + page.start_address(), + PageFlags::new().write(false).execute(true).global(true), + ) + .unwrap() + .flush(); } } } @@ -208,7 +245,7 @@ mod xsave { pub ymm_upper_offset: Option, pub xsave_size: u32, } - pub(in super) static XSAVE_INFO: Once = Once::new(); + pub(super) static XSAVE_INFO: Once = Once::new(); pub fn info() -> Option<&'static XsaveInfo> { XSAVE_INFO.get() diff --git a/src/arch/x86_64/consts.rs b/src/arch/x86_64/consts.rs index 760f883c7d..17ee32fc86 100644 --- a/src/arch/x86_64/consts.rs +++ b/src/arch/x86_64/consts.rs @@ -15,18 +15,18 @@ pub const PML4_MASK: usize = 0x0000_ff80_0000_0000; /// Offset of kernel pub const KERNEL_MAX_SIZE: usize = 1_usize << 31; pub const KERNEL_OFFSET: usize = KERNEL_MAX_SIZE.wrapping_neg(); -pub const KERNEL_PML4: usize = (KERNEL_OFFSET & PML4_MASK)/PML4_SIZE; +pub const KERNEL_PML4: usize = (KERNEL_OFFSET & PML4_MASK) / PML4_SIZE; /// Offset to kernel heap pub const KERNEL_HEAP_OFFSET: usize = KERNEL_OFFSET - PML4_SIZE; -pub const KERNEL_HEAP_PML4: usize = (KERNEL_HEAP_OFFSET & PML4_MASK)/PML4_SIZE; +pub const KERNEL_HEAP_PML4: usize = (KERNEL_HEAP_OFFSET & PML4_MASK) / PML4_SIZE; /// Size of kernel heap pub const KERNEL_HEAP_SIZE: usize = 1 * 1024 * 1024; // 1 MB /// Offset of physmap // This needs to match RMM's PHYS_OFFSET pub const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000; -pub const PHYS_PML4: usize = (PHYS_OFFSET & PML4_MASK)/PML4_SIZE; +pub const PHYS_PML4: usize = (PHYS_OFFSET & PML4_MASK) / PML4_SIZE; /// End offset of the user image, i.e. kernel start pub const USER_END_OFFSET: usize = 256 * PML4_SIZE; diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 23ccb88f37..51857a2f19 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -17,7 +17,9 @@ pub fn cpuid_always() -> CpuId { } pub fn feature_info() -> FeatureInfo { - cpuid_always().get_feature_info().expect("x86_64 requires CPUID leaf=0x01 to be present") + cpuid_always() + .get_feature_info() + .expect("x86_64 requires CPUID leaf=0x01 to be present") } pub fn has_ext_feat(feat: impl FnOnce(ExtendedFeatures) -> bool) -> bool { diff --git a/src/arch/x86_64/debug.rs b/src/arch/x86_64/debug.rs index 0906c89c62..4013e94f0c 100644 --- a/src/arch/x86_64/debug.rs +++ b/src/arch/x86_64/debug.rs @@ -3,24 +3,24 @@ use core::fmt; use spin::Mutex; use spin::MutexGuard; -use crate::log::{LOG, Log}; -#[cfg(feature = "qemu_debug")] -use syscall::io::Io; -#[cfg(any(feature = "qemu_debug", feature = "serial_debug"))] -use crate::syscall::io::Pio; -#[cfg(feature = "lpss_debug")] -use crate::syscall::io::Mmio; #[cfg(any(feature = "lpss_debug", feature = "serial_debug"))] use crate::devices::uart_16550::SerialPort; - -#[cfg(feature = "graphical_debug")] -use crate::devices::graphical_debug::{DEBUG_DISPLAY, DebugDisplay}; +use crate::log::{Log, LOG}; #[cfg(feature = "lpss_debug")] -use super::device::serial::LPSS; +use crate::syscall::io::Mmio; +#[cfg(any(feature = "qemu_debug", feature = "serial_debug"))] +use crate::syscall::io::Pio; +#[cfg(feature = "qemu_debug")] +use syscall::io::Io; + #[cfg(feature = "serial_debug")] use super::device::serial::COM1; +#[cfg(feature = "lpss_debug")] +use super::device::serial::LPSS; #[cfg(feature = "system76_ec_debug")] -use super::device::system76_ec::{SYSTEM76_EC, System76Ec}; +use super::device::system76_ec::{System76Ec, SYSTEM76_EC}; +#[cfg(feature = "graphical_debug")] +use crate::devices::graphical_debug::{DebugDisplay, DEBUG_DISPLAY}; #[cfg(feature = "qemu_debug")] pub static QEMU: Mutex> = Mutex::new(Pio::::new(0x402)); diff --git a/src/arch/x86_64/device/cpu.rs b/src/arch/x86_64/device/cpu.rs index 5a5c55b8f6..a24405d3c0 100644 --- a/src/arch/x86_64/device/cpu.rs +++ b/src/arch/x86_64/device/cpu.rs @@ -28,101 +28,257 @@ pub fn cpu_info(w: &mut W) -> Result { write!(w, "Features:")?; if let Some(info) = cpuid.get_feature_info() { - if info.has_fpu() { write!(w, " fpu")? }; - if info.has_vme() { write!(w, " vme")? }; - if info.has_de() { write!(w, " de")? }; - if info.has_pse() { write!(w, " pse")? }; - if info.has_tsc() { write!(w, " tsc")? }; - if info.has_msr() { write!(w, " msr")? }; - if info.has_pae() { write!(w, " pae")? }; - if info.has_mce() { write!(w, " mce")? }; + if info.has_fpu() { + write!(w, " fpu")? + }; + if info.has_vme() { + write!(w, " vme")? + }; + if info.has_de() { + write!(w, " de")? + }; + if info.has_pse() { + write!(w, " pse")? + }; + if info.has_tsc() { + write!(w, " tsc")? + }; + if info.has_msr() { + write!(w, " msr")? + }; + if info.has_pae() { + write!(w, " pae")? + }; + if info.has_mce() { + write!(w, " mce")? + }; - if info.has_cmpxchg8b() { write!(w, " cx8")? }; - if info.has_apic() { write!(w, " apic")? }; - if info.has_sysenter_sysexit() { write!(w, " sep")? }; - if info.has_mtrr() { write!(w, " mtrr")? }; - if info.has_pge() { write!(w, " pge")? }; - if info.has_mca() { write!(w, " mca")? }; - if info.has_cmov() { write!(w, " cmov")? }; - if info.has_pat() { write!(w, " pat")? }; + if info.has_cmpxchg8b() { + write!(w, " cx8")? + }; + if info.has_apic() { + write!(w, " apic")? + }; + if info.has_sysenter_sysexit() { + write!(w, " sep")? + }; + if info.has_mtrr() { + write!(w, " mtrr")? + }; + if info.has_pge() { + write!(w, " pge")? + }; + if info.has_mca() { + write!(w, " mca")? + }; + if info.has_cmov() { + write!(w, " cmov")? + }; + if info.has_pat() { + write!(w, " pat")? + }; - if info.has_pse36() { write!(w, " pse36")? }; - if info.has_psn() { write!(w, " psn")? }; - if info.has_clflush() { write!(w, " clflush")? }; - if info.has_ds() { write!(w, " ds")? }; - if info.has_acpi() { write!(w, " acpi")? }; - if info.has_mmx() { write!(w, " mmx")? }; - if info.has_fxsave_fxstor() { write!(w, " fxsr")? }; - if info.has_sse() { write!(w, " sse")? }; + if info.has_pse36() { + write!(w, " pse36")? + }; + if info.has_psn() { + write!(w, " psn")? + }; + if info.has_clflush() { + write!(w, " clflush")? + }; + if info.has_ds() { + write!(w, " ds")? + }; + if info.has_acpi() { + write!(w, " acpi")? + }; + if info.has_mmx() { + write!(w, " mmx")? + }; + if info.has_fxsave_fxstor() { + write!(w, " fxsr")? + }; + if info.has_sse() { + write!(w, " sse")? + }; - if info.has_sse2() { write!(w, " sse2")? }; - if info.has_ss() { write!(w, " ss")? }; - if info.has_htt() { write!(w, " ht")? }; - if info.has_tm() { write!(w, " tm")? }; - if info.has_pbe() { write!(w, " pbe")? }; + if info.has_sse2() { + write!(w, " sse2")? + }; + if info.has_ss() { + write!(w, " ss")? + }; + if info.has_htt() { + write!(w, " ht")? + }; + if info.has_tm() { + write!(w, " tm")? + }; + if info.has_pbe() { + write!(w, " pbe")? + }; - if info.has_sse3() { write!(w, " sse3")? }; - if info.has_pclmulqdq() { write!(w, " pclmulqdq")? }; - if info.has_ds_area() { write!(w, " dtes64")? }; - if info.has_monitor_mwait() { write!(w, " monitor")? }; - if info.has_cpl() { write!(w, " ds_cpl")? }; - if info.has_vmx() { write!(w, " vmx")? }; - if info.has_smx() { write!(w, " smx")? }; - if info.has_eist() { write!(w, " est")? }; + if info.has_sse3() { + write!(w, " sse3")? + }; + if info.has_pclmulqdq() { + write!(w, " pclmulqdq")? + }; + if info.has_ds_area() { + write!(w, " dtes64")? + }; + if info.has_monitor_mwait() { + write!(w, " monitor")? + }; + if info.has_cpl() { + write!(w, " ds_cpl")? + }; + if info.has_vmx() { + write!(w, " vmx")? + }; + if info.has_smx() { + write!(w, " smx")? + }; + if info.has_eist() { + write!(w, " est")? + }; - if info.has_tm2() { write!(w, " tm2")? }; - if info.has_ssse3() { write!(w, " ssse3")? }; - if info.has_cnxtid() { write!(w, " cnxtid")? }; - if info.has_fma() { write!(w, " fma")? }; - if info.has_cmpxchg16b() { write!(w, " cx16")? }; - if info.has_pdcm() { write!(w, " pdcm")? }; - if info.has_pcid() { write!(w, " pcid")? }; - if info.has_dca() { write!(w, " dca")? }; + if info.has_tm2() { + write!(w, " tm2")? + }; + if info.has_ssse3() { + write!(w, " ssse3")? + }; + if info.has_cnxtid() { + write!(w, " cnxtid")? + }; + if info.has_fma() { + write!(w, " fma")? + }; + if info.has_cmpxchg16b() { + write!(w, " cx16")? + }; + if info.has_pdcm() { + write!(w, " pdcm")? + }; + if info.has_pcid() { + write!(w, " pcid")? + }; + if info.has_dca() { + write!(w, " dca")? + }; - if info.has_sse41() { write!(w, " sse4_1")? }; - if info.has_sse42() { write!(w, " sse4_2")? }; - if info.has_x2apic() { write!(w, " x2apic")? }; - if info.has_movbe() { write!(w, " movbe")? }; - if info.has_popcnt() { write!(w, " popcnt")? }; - if info.has_tsc_deadline() { write!(w, " tsc_deadline_timer")? }; - if info.has_aesni() { write!(w, " aes")? }; - if info.has_xsave() { write!(w, " xsave")? }; + if info.has_sse41() { + write!(w, " sse4_1")? + }; + if info.has_sse42() { + write!(w, " sse4_2")? + }; + if info.has_x2apic() { + write!(w, " x2apic")? + }; + if info.has_movbe() { + write!(w, " movbe")? + }; + if info.has_popcnt() { + write!(w, " popcnt")? + }; + if info.has_tsc_deadline() { + write!(w, " tsc_deadline_timer")? + }; + if info.has_aesni() { + write!(w, " aes")? + }; + if info.has_xsave() { + write!(w, " xsave")? + }; - if info.has_oxsave() { write!(w, " xsaveopt")? }; - if info.has_avx() { write!(w, " avx")? }; - if info.has_f16c() { write!(w, " f16c")? }; - if info.has_rdrand() { write!(w, " rdrand")? }; + if info.has_oxsave() { + write!(w, " xsaveopt")? + }; + if info.has_avx() { + write!(w, " avx")? + }; + if info.has_f16c() { + write!(w, " f16c")? + }; + if info.has_rdrand() { + write!(w, " rdrand")? + }; } if let Some(info) = cpuid.get_extended_processor_and_feature_identifiers() { - if info.has_64bit_mode() { write!(w, " lm")? }; - if info.has_rdtscp() { write!(w, " rdtscp")? }; - if info.has_1gib_pages() { write!(w, " pdpe1gb")? }; - if info.has_execute_disable() { write!(w, " nx")? }; - if info.has_syscall_sysret() { write!(w, " syscall")? }; - if info.has_prefetchw() { write!(w, " prefetchw")? }; - if info.has_lzcnt() { write!(w, " lzcnt")? }; - if info.has_lahf_sahf() { write!(w, " lahf_lm")? }; + if info.has_64bit_mode() { + write!(w, " lm")? + }; + if info.has_rdtscp() { + write!(w, " rdtscp")? + }; + if info.has_1gib_pages() { + write!(w, " pdpe1gb")? + }; + if info.has_execute_disable() { + write!(w, " nx")? + }; + if info.has_syscall_sysret() { + write!(w, " syscall")? + }; + if info.has_prefetchw() { + write!(w, " prefetchw")? + }; + if info.has_lzcnt() { + write!(w, " lzcnt")? + }; + if info.has_lahf_sahf() { + write!(w, " lahf_lm")? + }; } if let Some(info) = cpuid.get_advanced_power_mgmt_info() { - if info.has_invariant_tsc() { write!(w, " constant_tsc")? }; + if info.has_invariant_tsc() { + write!(w, " constant_tsc")? + }; } if let Some(info) = cpuid.get_extended_feature_info() { - if info.has_fsgsbase() { write!(w, " fsgsbase")? }; - if info.has_tsc_adjust_msr() { write!(w, " tsc_adjust")? }; - if info.has_bmi1() { write!(w, " bmi1")? }; - if info.has_hle() { write!(w, " hle")? }; - if info.has_avx2() { write!(w, " avx2")? }; - if info.has_smep() { write!(w, " smep")? }; - if info.has_bmi2() { write!(w, " bmi2")? }; - if info.has_rep_movsb_stosb() { write!(w, " erms")? }; - if info.has_invpcid() { write!(w, " invpcid")? }; - if info.has_rtm() { write!(w, " rtm")? }; + if info.has_fsgsbase() { + write!(w, " fsgsbase")? + }; + if info.has_tsc_adjust_msr() { + write!(w, " tsc_adjust")? + }; + if info.has_bmi1() { + write!(w, " bmi1")? + }; + if info.has_hle() { + write!(w, " hle")? + }; + if info.has_avx2() { + write!(w, " avx2")? + }; + if info.has_smep() { + write!(w, " smep")? + }; + if info.has_bmi2() { + write!(w, " bmi2")? + }; + if info.has_rep_movsb_stosb() { + write!(w, " erms")? + }; + if info.has_invpcid() { + write!(w, " invpcid")? + }; + if info.has_rtm() { + write!(w, " rtm")? + }; //if info.has_qm() { write!(w, " qm")? }; - if info.has_fpu_cs_ds_deprecated() { write!(w, " fpu_seg")? }; - if info.has_mpx() { write!(w, " mpx")? }; + if info.has_fpu_cs_ds_deprecated() { + write!(w, " fpu_seg")? + }; + if info.has_mpx() { + write!(w, " mpx")? + }; } writeln!(w)?; diff --git a/src/arch/x86_64/device/hpet.rs b/src/arch/x86_64/device/hpet.rs index e15968035e..b52fe87a16 100644 --- a/src/arch/x86_64/device/hpet.rs +++ b/src/arch/x86_64/device/hpet.rs @@ -1,5 +1,5 @@ -use crate::acpi::hpet::Hpet; use super::pit; +use crate::acpi::hpet::Hpet; const LEG_RT_CNF: u64 = 2; const ENABLE_CNF: u64 = 1; @@ -27,7 +27,8 @@ pub unsafe fn init(hpet: &mut Hpet) -> bool { { let mut config_word = hpet.base_address.read_u64(GENERAL_CONFIG_OFFSET); config_word &= !(LEG_RT_CNF | ENABLE_CNF); - hpet.base_address.write_u64(GENERAL_CONFIG_OFFSET, config_word); + hpet.base_address + .write_u64(GENERAL_CONFIG_OFFSET, config_word); } let capability = hpet.base_address.read_u64(CAPABILITY_OFFSET); @@ -48,9 +49,11 @@ pub unsafe fn init(hpet: &mut Hpet) -> bool { let counter = hpet.base_address.read_u64(MAIN_COUNTER_OFFSET); let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF; - hpet.base_address.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word); + hpet.base_address + .write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word); // set accumulator value - hpet.base_address.write_u64(T0_COMPARATOR_OFFSET, counter + divisor); + hpet.base_address + .write_u64(T0_COMPARATOR_OFFSET, counter + divisor); // set interval hpet.base_address.write_u64(T0_COMPARATOR_OFFSET, divisor); @@ -58,7 +61,8 @@ pub unsafe fn init(hpet: &mut Hpet) -> bool { { let mut config_word: u64 = hpet.base_address.read_u64(GENERAL_CONFIG_OFFSET); config_word |= LEG_RT_CNF | ENABLE_CNF; - hpet.base_address.write_u64(GENERAL_CONFIG_OFFSET, config_word); + hpet.base_address + .write_u64(GENERAL_CONFIG_OFFSET, config_word); } println!("HPET After Init"); @@ -76,7 +80,10 @@ pub unsafe fn debug(hpet: &mut Hpet) { println!(" clock period: {}", (capability >> 32) as u32); println!(" ID: {:#x}", (capability >> 16) as u16); println!(" LEG_RT_CAP: {}", capability & (1 << 15) == (1 << 15)); - println!(" COUNT_SIZE_CAP: {}", capability & (1 << 13) == (1 << 13)); + println!( + " COUNT_SIZE_CAP: {}", + capability & (1 << 13) == (1 << 13) + ); println!(" timers: {}", (capability >> 8) as u8 & 0x1F); println!(" revision: {}", capability as u8); } @@ -92,7 +99,10 @@ pub unsafe fn debug(hpet: &mut Hpet) { let t0_capabilities = hpet.base_address.read_u64(T0_CONFIG_CAPABILITY_OFFSET); println!(" T0 caps: {:#x}", t0_capabilities); - println!(" interrupt routing: {:#x}", (t0_capabilities >> 32) as u32); + println!( + " interrupt routing: {:#x}", + (t0_capabilities >> 32) as u32 + ); println!(" flags: {:#x}", t0_capabilities as u16); let t0_comparator = hpet.base_address.read_u64(T0_COMPARATOR_OFFSET); diff --git a/src/arch/x86_64/device/ioapic.rs b/src/arch/x86_64/device/ioapic.rs index 9fdf416f90..61f12c1172 100644 --- a/src/arch/x86_64/device/ioapic.rs +++ b/src/arch/x86_64/device/ioapic.rs @@ -4,15 +4,15 @@ use alloc::vec::Vec; use spin::Mutex; #[cfg(feature = "acpi")] -use crate::acpi::madt::{self, Madt, MadtEntry, MadtIoApic, MadtIntSrcOverride}; +use crate::acpi::madt::{self, Madt, MadtEntry, MadtIntSrcOverride, MadtIoApic}; -use crate::arch::interrupt::irq; -use crate::memory::Frame; -use crate::paging::{KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch}; -use crate::paging::entry::EntryFlags; +use crate::{ + arch::interrupt::irq, + memory::Frame, + paging::{entry::EntryFlags, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch}, +}; -use super::super::cpuid::cpuid; -use super::pic; +use super::{super::cpuid::cpuid, pic}; pub struct IoApicRegs { pointer: *const u32, @@ -131,12 +131,12 @@ pub enum DestinationMode { #[repr(u8)] #[derive(Clone, Copy, Debug)] pub enum DeliveryMode { - Fixed = 0b000, - LowestPriority = 0b001, - Smi = 0b010, - Nmi = 0b100, - Init = 0b101, - ExtInt = 0b111, + Fixed = 0b000, + LowestPriority = 0b001, + Smi = 0b010, + Nmi = 0b100, + Init = 0b101, + ExtInt = 0b111, } #[derive(Clone, Copy, Debug)] @@ -176,7 +176,9 @@ impl fmt::Debug for IoApic { let mut guard = self.0.lock(); let count = guard.max_redirection_table_entries(); - f.debug_list().entries((0..count).map(|i| guard.read_ioredtbl(i))).finish() + f.debug_list() + .entries((0..count).map(|i| guard.read_ioredtbl(i))) + .finish() } } @@ -219,14 +221,10 @@ static mut IOAPICS: Option> = None; static mut SRC_OVERRIDES: Option> = None; pub fn ioapics() -> &'static [IoApic] { - unsafe { - IOAPICS.as_ref().map_or(&[], |vector| &vector[..]) - } + unsafe { IOAPICS.as_ref().map_or(&[], |vector| &vector[..]) } } pub fn src_overrides() -> &'static [Override] { - unsafe { - SRC_OVERRIDES.as_ref().map_or(&[], |vector| &vector[..]) - } + unsafe { SRC_OVERRIDES.as_ref().map_or(&[], |vector| &vector[..]) } } #[cfg(feature = "acpi")] @@ -241,14 +239,24 @@ pub unsafe fn handle_ioapic(mapper: &mut KernelMapper, madt_ioapic: &'static Mad mapper .get_mut() .expect("expected KernelMapper not to be locked re-entrant while mapping I/O APIC memory") - .map_phys(page.start_address(), frame.start_address(), PageFlags::new().write(true).custom_flag(EntryFlags::NO_CACHE.bits(), true)) + .map_phys( + page.start_address(), + frame.start_address(), + PageFlags::new() + .write(true) + .custom_flag(EntryFlags::NO_CACHE.bits(), true), + ) .expect("failed to map I/O APIC") .flush(); let ioapic_registers = page.start_address().data() as *const u32; let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base); - assert_eq!(ioapic.regs.lock().id(), madt_ioapic.id, "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC"); + assert_eq!( + ioapic.regs.lock().id(), + madt_ioapic.id, + "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC" + ); IOAPICS.get_or_insert_with(Vec::new).push(ioapic); } @@ -286,7 +294,11 @@ pub unsafe fn handle_src_override(src_override: &'static MadtIntSrcOverride) { } pub unsafe fn init(active_table: &mut KernelMapper) { - let bsp_apic_id = cpuid().unwrap().get_feature_info().unwrap().initial_local_apic_id(); // TODO: remove unwraps + let bsp_apic_id = cpuid() + .unwrap() + .get_feature_info() + .unwrap() + .initial_local_apic_id(); // TODO: remove unwraps // search the madt for all IOAPICs. #[cfg(feature = "acpi")] @@ -310,7 +322,11 @@ pub unsafe fn init(active_table: &mut KernelMapper) { } } } - println!("I/O APICs: {:?}, overrides: {:?}", ioapics(), src_overrides()); + println!( + "I/O APICs: {:?}, overrides: {:?}", + ioapics(), + src_overrides() + ); // map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it // wouldn't have been disabled due to this I/O APIC) @@ -318,11 +334,21 @@ pub unsafe fn init(active_table: &mut KernelMapper) { let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) { Some(over) => (over.gsi, over.trigger_mode, over.polarity), None => { - if src_overrides().iter().any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq) && !src_overrides().iter().any(|over| over.bus_irq == legacy_irq) { + if src_overrides() + .iter() + .any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq) + && !src_overrides() + .iter() + .any(|over| over.bus_irq == legacy_irq) + { // there's an IRQ conflict, making this legacy IRQ inaccessible. continue; } - (legacy_irq.into(), TriggerMode::ConformsToSpecs, Polarity::ConformsToSpecs) + ( + legacy_irq.into(), + TriggerMode::ConformsToSpecs, + Polarity::ConformsToSpecs, + ) } }; let apic = match find_ioapic(gsi) { @@ -354,7 +380,11 @@ pub unsafe fn init(active_table: &mut KernelMapper) { }; apic.map(redir_tbl_index, map_info); } - println!("I/O APICs: {:?}, overrides: {:?}", ioapics(), src_overrides()); + println!( + "I/O APICs: {:?}, overrides: {:?}", + ioapics(), + src_overrides() + ); irq::set_irq_method(irq::IrqMethod::Apic); // tell the firmware that we're using APIC rather than the default 8259 PIC. @@ -385,7 +415,9 @@ fn resolve(irq: u8) -> u32 { get_override(irq).map_or(u32::from(irq), |over| over.gsi) } fn find_ioapic(gsi: u32) -> Option<&'static IoApic> { - ioapics().iter().find(|apic| gsi >= apic.gsi_start && gsi < apic.gsi_start + u32::from(apic.count)) + ioapics() + .iter() + .find(|apic| gsi >= apic.gsi_start && gsi < apic.gsi_start + u32::from(apic.count)) } pub unsafe fn mask(irq: u8) { diff --git a/src/arch/x86_64/device/local_apic.rs b/src/arch/x86_64/device/local_apic.rs index 0c15b1744e..b44a0b563d 100644 --- a/src/arch/x86_64/device/local_apic.rs +++ b/src/arch/x86_64/device/local_apic.rs @@ -1,14 +1,16 @@ -use core::sync::atomic::{self, AtomicU64}; -use core::ptr::{read_volatile, write_volatile}; +use core::{ + ptr::{read_volatile, write_volatile}, + sync::atomic::{self, AtomicU64}, +}; use x86::msr::*; -use crate::paging::{KernelMapper, PhysicalAddress, PageFlags, RmmA, RmmArch}; +use crate::paging::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch}; use super::super::cpuid::cpuid; pub static mut LOCAL_APIC: LocalApic = LocalApic { address: 0, - x2: false + x2: false, }; pub unsafe fn init(active_table: &mut KernelMapper) { @@ -22,7 +24,7 @@ pub unsafe fn init_ap() { /// Local APIC pub struct LocalApic { pub address: usize, - pub x2: bool + pub x2: bool, } #[derive(Debug)] @@ -42,19 +44,21 @@ pub fn bsp_apic_id() -> Option { impl LocalApic { unsafe fn init(&mut self, mapper: &mut KernelMapper) { - let mapper = mapper.get_mut().expect("expected KernelMapper not to be locked re-entrant while initializing LAPIC"); + let mapper = mapper + .get_mut() + .expect("expected KernelMapper not to be locked re-entrant while initializing LAPIC"); let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000); let virtaddr = RmmA::phys_to_virt(physaddr); self.address = virtaddr.data(); self.x2 = cpuid().map_or(false, |cpuid| { - cpuid.get_feature_info().map_or(false, |feature_info| { - feature_info.has_x2apic() - }) + cpuid + .get_feature_info() + .map_or(false, |feature_info| feature_info.has_x2apic()) }); - if ! self.x2 { + if !self.x2 { log::info!("Detected xAPIC at {:#x}", physaddr.data()); if let Some((_entry, _, flush)) = mapper.unmap_phys(virtaddr, true) { // Unmap xAPIC page if already mapped @@ -112,15 +116,15 @@ impl LocalApic { if self.x2 { unsafe { rdmsr(IA32_X2APIC_ICR) } } else { - unsafe { - (self.read(0x310) as u64) << 32 | self.read(0x300) as u64 - } + unsafe { (self.read(0x310) as u64) << 32 | self.read(0x300) as u64 } } } pub fn set_icr(&mut self, value: u64) { if self.x2 { - unsafe { wrmsr(IA32_X2APIC_ICR, value); } + unsafe { + wrmsr(IA32_X2APIC_ICR, value); + } } else { unsafe { const PENDING: u32 = 1 << 12; diff --git a/src/arch/x86_64/device/mod.rs b/src/arch/x86_64/device/mod.rs index eafa245c5c..60dfc43440 100644 --- a/src/arch/x86_64/device/mod.rs +++ b/src/arch/x86_64/device/mod.rs @@ -1,12 +1,12 @@ pub mod cpu; +#[cfg(feature = "acpi")] +pub mod hpet; pub mod ioapic; pub mod local_apic; pub mod pic; pub mod pit; pub mod rtc; pub mod serial; -#[cfg(feature = "acpi")] -pub mod hpet; #[cfg(feature = "system76_ec_debug")] pub mod system76_ec; @@ -16,7 +16,7 @@ pub unsafe fn init() { pic::init(); local_apic::init(&mut KernelMapper::lock()); } -pub unsafe fn init_after_acpi() { +pub unsafe fn init_after_acpi() { // this will disable the IOAPIC if needed. //ioapic::init(mapper); } diff --git a/src/arch/x86_64/device/pic.rs b/src/arch/x86_64/device/pic.rs index 12e68a61e9..487229892e 100644 --- a/src/arch/x86_64/device/pic.rs +++ b/src/arch/x86_64/device/pic.rs @@ -1,5 +1,7 @@ -use crate::syscall::io::{Io, Pio}; -use crate::arch::interrupt::irq; +use crate::{ + arch::interrupt::irq, + syscall::io::{Io, Pio}, +}; pub static mut MASTER: Pic = Pic::new(0x20); pub static mut SLAVE: Pic = Pic::new(0xA0); diff --git a/src/arch/x86_64/device/rtc.rs b/src/arch/x86_64/device/rtc.rs index d68cdbb8a0..0b7bba9e7a 100644 --- a/src/arch/x86_64/device/rtc.rs +++ b/src/arch/x86_64/device/rtc.rs @@ -1,5 +1,7 @@ -use crate::syscall::io::{Io, Pio}; -use crate::time; +use crate::{ + syscall::io::{Io, Pio}, + time, +}; pub fn init() { let mut rtc = Rtc::new(); diff --git a/src/arch/x86_64/device/serial.rs b/src/arch/x86_64/device/serial.rs index 280125bc12..83a480e92d 100644 --- a/src/arch/x86_64/device/serial.rs +++ b/src/arch/x86_64/device/serial.rs @@ -1,7 +1,6 @@ -use crate::devices::uart_16550::SerialPort; #[cfg(feature = "lpss_debug")] use crate::syscall::io::Mmio; -use crate::syscall::io::Pio; +use crate::{devices::uart_16550::SerialPort, syscall::io::Pio}; use spin::Mutex; pub static COM1: Mutex>> = Mutex::new(SerialPort::>::new(0x3F8)); @@ -22,19 +21,24 @@ pub unsafe fn init() { let address = crate::PHYS_OFFSET + 0xFE032000; { - use crate::paging::{ActivePageTable, Page, VirtualAddress, entry::EntryFlags}; - use crate::memory::{Frame, PhysicalAddress}; + use crate::{ + memory::{Frame, PhysicalAddress}, + paging::{entry::EntryFlags, ActivePageTable, Page, VirtualAddress}, + }; let mut active_table = ActivePageTable::new(); let page = Page::containing_address(VirtualAddress::new(address)); - let frame = Frame::containing_address(PhysicalAddress::new(address - crate::PHYS_OFFSET)); - let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE); + let frame = + Frame::containing_address(PhysicalAddress::new(address - crate::PHYS_OFFSET)); + let result = active_table.map_to( + page, + frame, + EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE, + ); result.flush(&mut active_table); } - let lpss = SerialPort::>::new( - crate::PHYS_OFFSET + 0xFE032000 - ); + let lpss = SerialPort::>::new(crate::PHYS_OFFSET + 0xFE032000); lpss.init(); *LPSS.lock() = Some(lpss); diff --git a/src/arch/x86_64/device/system76_ec.rs b/src/arch/x86_64/device/system76_ec.rs index 49f3a29134..dca4d5efe3 100644 --- a/src/arch/x86_64/device/system76_ec.rs +++ b/src/arch/x86_64/device/system76_ec.rs @@ -13,9 +13,7 @@ pub struct System76Ec { impl System76Ec { pub fn new() -> Option { - let mut system76_ec = Self { - base: 0x0E00, - }; + let mut system76_ec = Self { base: 0x0E00 }; if system76_ec.probe() { Some(system76_ec) } else { diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 80e26c15dc..106de1b316 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,17 +1,19 @@ //! Global descriptor table -use core::convert::TryInto; -use core::mem; +use core::{convert::TryInto, mem}; -use crate::LogicalCpuId; -use crate::paging::{RmmA, RmmArch}; -use crate::percpu::PercpuBlock; +use crate::{ + paging::{RmmA, RmmArch}, + percpu::PercpuBlock, + LogicalCpuId, +}; -use x86::bits64::task::TaskStateSegment; -use x86::Ring; -use x86::dtables::{self, DescriptorTablePointer}; -use x86::segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector}; -use x86::task; +use x86::{ + bits64::task::TaskStateSegment, + dtables::{self, DescriptorTablePointer}, + segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector}, + task, Ring, +}; pub const GDT_NULL: usize = 0; pub const GDT_KERNEL_CODE: usize = 1; @@ -44,9 +46,19 @@ static mut INIT_GDT: [GdtEntry; 3] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_LONG_MODE, + ), // Kernel data - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_LONG_MODE, + ), ]; // Later copied into the actual GDT with various fields set. @@ -54,15 +66,40 @@ const BASE_GDT: [GdtEntry; 8] = [ // Null GdtEntry::new(0, 0, 0, 0), // Kernel code - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_LONG_MODE, + ), // Kernel data - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_LONG_MODE, + ), // Dummy 32-bit user code - apparently necessary for SYSRET. We restrict it to ring 0 anyway. - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_PROTECTED_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_PROTECTED_MODE, + ), // User data - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE, + GDT_F_LONG_MODE, + ), // User (64-bit) code - GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, GDT_F_LONG_MODE), + GdtEntry::new( + 0, + 0, + GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE, + GDT_F_LONG_MODE, + ), // TSS GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0), // TSS must be 16 bytes long, twice the normal size @@ -104,8 +141,9 @@ pub unsafe fn pcr() -> *mut ProcessorControlRegion { #[cfg(feature = "pti")] pub unsafe fn set_tss_stack(stack: usize) { - use super::pti::{PTI_CPU_STACK, PTI_CONTEXT_STACK}; - core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64); + use super::pti::{PTI_CONTEXT_STACK, PTI_CPU_STACK}; + core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]) + .write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64); PTI_CONTEXT_STACK = stack; } @@ -145,7 +183,8 @@ unsafe fn load_segments() { #[cold] pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { let pcr_frame = crate::memory::allocate_frames(1).expect("failed to allocate PCR"); - let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); + let pcr = + &mut *(RmmA::phys_to_virt(pcr_frame.start_address()).data() as *mut ProcessorControlRegion); pcr.self_ref = pcr as *mut ProcessorControlRegion as usize; @@ -157,10 +196,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { .expect("main GDT way too large"); let base = pcr.gdt.as_ptr() as *const SegmentDescriptor; - let gdtr: DescriptorTablePointer = DescriptorTablePointer { - limit, - base, - }; + let gdtr: DescriptorTablePointer = DescriptorTablePointer { limit, base }; { pcr.tss.iomap_base = 0xFFFF; @@ -172,7 +208,9 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { pcr.gdt[GDT_TSS].set_offset(tss_lo); pcr.gdt[GDT_TSS].set_limit(mem::size_of::() as u32); - (&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry).cast::().write(tss_hi); + (&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry) + .cast::() + .write(tss_hi); } // Load the new GDT, which is correctly located in thread local storage. @@ -214,7 +252,7 @@ pub struct GdtEntry { pub offsetm: u8, pub access: u8, pub flags_limith: u8, - pub offseth: u8 + pub offseth: u8, } impl GdtEntry { @@ -225,7 +263,7 @@ impl GdtEntry { offsetm: (offset >> 16) as u8, access, flags_limith: flags & 0xF0 | ((limit >> 16) as u8) & 0x0F, - offseth: (offset >> 24) as u8 + offseth: (offset >> 24) as u8, } } diff --git a/src/arch/x86_64/idt.rs b/src/arch/x86_64/idt.rs index a5ea9e045a..a7dd5aee04 100644 --- a/src/arch/x86_64/idt.rs +++ b/src/arch/x86_64/idt.rs @@ -1,22 +1,31 @@ -use core::num::NonZeroU8; -use core::sync::atomic::{AtomicU64, Ordering}; -use core::mem; +use core::{ + mem, + num::NonZeroU8, + sync::atomic::{AtomicU64, Ordering}, +}; use alloc::boxed::Box; use hashbrown::HashMap; -use x86::segmentation::Descriptor as X86IdtEntry; -use x86::dtables::{self, DescriptorTablePointer}; +use x86::{ + dtables::{self, DescriptorTablePointer}, + segmentation::Descriptor as X86IdtEntry, +}; -use crate::{interrupt::*, LogicalCpuId}; -use crate::interrupt::irq::{__generic_interrupts_end, __generic_interrupts_start}; -use crate::ipi::IpiKind; +use crate::{ + interrupt::{ + irq::{__generic_interrupts_end, __generic_interrupts_start}, + *, + }, + ipi::IpiKind, + LogicalCpuId, +}; use spin::RwLock; pub static mut INIT_IDTR: DescriptorTablePointer = DescriptorTablePointer { limit: 0, - base: 0 as *const X86IdtEntry + base: 0 as *const X86IdtEntry, }; pub type IdtEntries = [IdtEntry; 256]; @@ -47,7 +56,8 @@ impl Idt { let byte_index = index / 64; let bit = index % 64; - { &self.reservations[usize::from(byte_index)] }.fetch_or(u64::from(reserved) << bit, Ordering::AcqRel); + { &self.reservations[usize::from(byte_index)] } + .fetch_or(u64::from(reserved) << bit, Ordering::AcqRel); } #[inline] pub fn is_reserved_mut(&mut self, index: u8) -> bool { @@ -62,7 +72,8 @@ impl Idt { let byte_index = index / 64; let bit = index % 64; - *{ &mut self.reservations[usize::from(byte_index)] }.get_mut() |= u64::from(reserved) << bit; + *{ &mut self.reservations[usize::from(byte_index)] }.get_mut() |= + u64::from(reserved) << bit; } } @@ -76,7 +87,18 @@ pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool { let byte_index = index / 64; let bit = index % 64; - { &IDTS.read().as_ref().unwrap().get(&cpu_id).unwrap().reservations[usize::from(byte_index)] }.load(Ordering::Acquire) & (1 << bit) != 0 + { + &IDTS + .read() + .as_ref() + .unwrap() + .get(&cpu_id) + .unwrap() + .reservations[usize::from(byte_index)] + } + .load(Ordering::Acquire) + & (1 << bit) + != 0 } #[inline] @@ -84,13 +106,22 @@ pub fn set_reserved(cpu_id: LogicalCpuId, index: u8, reserved: bool) { let byte_index = index / 64; let bit = index % 64; - { &IDTS.read().as_ref().unwrap().get(&cpu_id).unwrap().reservations[usize::from(byte_index)] }.fetch_or(u64::from(reserved) << bit, Ordering::AcqRel); + { + &IDTS + .read() + .as_ref() + .unwrap() + .get(&cpu_id) + .unwrap() + .reservations[usize::from(byte_index)] + } + .fetch_or(u64::from(reserved) << bit, Ordering::AcqRel); } pub fn allocate_interrupt() -> Option { let cpu_id = crate::cpu_id(); for number in 50..=254 { - if ! is_reserved(cpu_id, number) { + if !is_reserved(cpu_id, number) { set_reserved(cpu_id, number, true); return Some(unsafe { NonZeroU8::new_unchecked(number) }); } @@ -107,7 +138,12 @@ pub unsafe fn init() { } const fn new_idt_reservations() -> [AtomicU64; 4] { - [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)] + [ + AtomicU64::new(0), + AtomicU64::new(0), + AtomicU64::new(0), + AtomicU64::new(0), + ] } /// Initialize the IDT for a @@ -118,7 +154,9 @@ pub unsafe fn init_paging_post_heap(cpu_id: LogicalCpuId) { if cpu_id == LogicalCpuId::BSP { idts_btree.insert(cpu_id, &mut INIT_BSP_IDT); } else { - let idt = idts_btree.entry(cpu_id).or_insert_with(|| Box::leak(Box::new(Idt::new()))); + let idt = idts_btree + .entry(cpu_id) + .or_insert_with(|| Box::leak(Box::new(Idt::new()))); init_generic(cpu_id, idt); } } @@ -199,7 +237,10 @@ pub unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) { current_idt[30].set_func(exception::security); // 31 reserved - assert_eq!(__generic_interrupts_end as usize - __generic_interrupts_start as usize, 224 * 8); + assert_eq!( + __generic_interrupts_end as usize - __generic_interrupts_start as usize, + 224 * 8 + ); for i in 0..224 { current_idt[i + 32].set_func(mem::transmute(__generic_interrupts_start as usize + i * 8)); @@ -282,7 +323,7 @@ pub struct IdtEntry { attribute: u8, offsetm: u16, offseth: u32, - _zero2: u32 + _zero2: u32, } impl IdtEntry { @@ -294,7 +335,7 @@ impl IdtEntry { attribute: 0, offsetm: 0, offseth: 0, - _zero2: 0 + _zero2: 0, } } @@ -303,7 +344,11 @@ impl IdtEntry { } pub fn set_ist(&mut self, ist: u8) { - assert_eq!(ist & 0x07, ist, "interrupt stack table must be within 0..=7"); + assert_eq!( + ist & 0x07, + ist, + "interrupt stack table must be within 0..=7" + ); self.zero &= 0xF8; self.zero |= ist; } @@ -316,7 +361,7 @@ impl IdtEntry { } // A function to set the offset more easily - pub fn set_func(&mut self, func: unsafe extern fn()) { + pub fn set_func(&mut self, func: unsafe extern "C" fn()) { self.set_flags(IdtFlags::PRESENT | IdtFlags::RING_0 | IdtFlags::INTERRUPT); self.set_offset((crate::gdt::GDT_KERNEL_CODE as u16) << 3, func as usize); } diff --git a/src/arch/x86_64/interrupt/exception.rs b/src/arch/x86_64/interrupt/exception.rs index 6a73d59079..7162b3277b 100644 --- a/src/arch/x86_64/interrupt/exception.rs +++ b/src/arch/x86_64/interrupt/exception.rs @@ -1,17 +1,11 @@ use x86::irq::PageFaultError; -use crate::memory::GenericPfFlags; -use crate::ptrace; use crate::{ - interrupt::stack_trace, - paging::VirtualAddress, - syscall::flag::*, - - interrupt_stack, - interrupt_error, + interrupt::stack_trace, interrupt_error, interrupt_stack, memory::GenericPfFlags, + paging::VirtualAddress, ptrace, syscall::flag::*, }; -extern { +extern "C" { fn ksignal(signal: usize); } @@ -145,11 +139,26 @@ interrupt_error!(page, |stack, code| { let arch_flags = PageFaultError::from_bits_truncate(code as u32); let mut generic_flags = GenericPfFlags::empty(); - generic_flags.set(GenericPfFlags::PRESENT, arch_flags.contains(PageFaultError::P)); - generic_flags.set(GenericPfFlags::INVOLVED_WRITE, arch_flags.contains(PageFaultError::WR)); - generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, arch_flags.contains(PageFaultError::US)); - generic_flags.set(GenericPfFlags::INVL, arch_flags.contains(PageFaultError::RSVD)); - generic_flags.set(GenericPfFlags::INSTR_NOT_DATA, arch_flags.contains(PageFaultError::ID)); + generic_flags.set( + GenericPfFlags::PRESENT, + arch_flags.contains(PageFaultError::P), + ); + generic_flags.set( + GenericPfFlags::INVOLVED_WRITE, + arch_flags.contains(PageFaultError::WR), + ); + generic_flags.set( + GenericPfFlags::USER_NOT_SUPERVISOR, + arch_flags.contains(PageFaultError::US), + ); + generic_flags.set( + GenericPfFlags::INVL, + arch_flags.contains(PageFaultError::RSVD), + ); + generic_flags.set( + GenericPfFlags::INSTR_NOT_DATA, + arch_flags.contains(PageFaultError::ID), + ); if crate::memory::page_fault_handler(stack, generic_flags, cr2).is_err() { println!("Page fault: {:>016X} {:#?}", cr2.data(), arch_flags); diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index b653395b3a..b551c5c614 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -1,7 +1,6 @@ use core::mem; -use crate::memory::ArchIntCtx; -use crate::syscall::IntRegisters; +use crate::{memory::ArchIntCtx, syscall::IntRegisters}; use super::super::flags::*; @@ -65,7 +64,6 @@ pub struct IretRegisters { // In x86 Protected Mode, i.e. 32-bit kernels, the following two registers are conditionally // pushed if the privilege ring changes. In x86 Long Mode however, i.e. 64-bit kernels, they // are unconditionally pushed, mostly due to stack alignment requirements. - pub rsp: usize, pub ss: usize, } @@ -83,7 +81,10 @@ impl IretRegisters { let fsbase = x86::msr::rdmsr(x86::msr::IA32_FS_BASE); let gsbase = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE); let kgsbase = x86::msr::rdmsr(x86::msr::IA32_GS_BASE); - println!("FSBASE {:016x}\nGSBASE {:016x}\nKGSBASE {:016x}", fsbase, gsbase, kgsbase); + println!( + "FSBASE {:016x}\nGSBASE {:016x}\nKGSBASE {:016x}", + fsbase, gsbase, kgsbase + ); } } } @@ -190,7 +191,8 @@ impl InterruptStack { #[macro_export] macro_rules! push_scratch { - () => { " + () => { + " // Push scratch registers push rcx push rdx @@ -200,11 +202,13 @@ macro_rules! push_scratch { push r9 push r10 push r11 - " }; + " + }; } #[macro_export] macro_rules! pop_scratch { - () => { " + () => { + " // Pop scratch registers pop r11 pop r10 @@ -215,12 +219,14 @@ macro_rules! pop_scratch { pop rdx pop rcx pop rax - " }; + " + }; } #[macro_export] macro_rules! push_preserved { - () => { " + () => { + " // Push preserved registers push rbx push rbp @@ -228,11 +234,13 @@ macro_rules! push_preserved { push r13 push r14 push r15 - " }; + " + }; } #[macro_export] macro_rules! pop_preserved { - () => { " + () => { + " // Pop preserved registers pop r15 pop r14 @@ -240,27 +248,32 @@ macro_rules! pop_preserved { pop r12 pop rbp pop rbx - " }; + " + }; } macro_rules! swapgs_iff_ring3_fast { // TODO: Spectre V1: LFENCE? - () => { " + () => { + " // Check whether the last two bits RSP+8 (code segment) are equal to zero. test QWORD PTR [rsp + 8], 0x3 // Skip the SWAPGS instruction if CS & 0b11 == 0b00. jz 1f swapgs 1: - " }; + " + }; } macro_rules! swapgs_iff_ring3_fast_errorcode { // TODO: Spectre V1: LFENCE? - () => { " + () => { + " test QWORD PTR [rsp + 16], 0x3 jz 1f swapgs 1: - " }; + " + }; } macro_rules! conditional_swapgs_paranoid { @@ -328,17 +341,21 @@ macro_rules! conditional_swapgs_paranoid { ) } } macro_rules! conditional_swapgs_back_paranoid { - () => { " + () => { + " test bl, bl jnz 1f swapgs 1: - " } + " + }; } macro_rules! nop { - () => { " + () => { + " // Unused: {IA32_GS_BASE} {PCR_GDT_OFFSET} - " } + " + }; } #[macro_export] diff --git a/src/arch/x86_64/interrupt/ipi.rs b/src/arch/x86_64/interrupt/ipi.rs index fefbd8c02a..6a37411d10 100644 --- a/src/arch/x86_64/interrupt/ipi.rs +++ b/src/arch/x86_64/interrupt/ipi.rs @@ -1,7 +1,6 @@ use x86::tlb; -use crate::context; -use crate::device::local_apic::LOCAL_APIC; +use crate::{context, device::local_apic::LOCAL_APIC}; interrupt!(wakeup, || { LOCAL_APIC.eoi(); diff --git a/src/arch/x86_64/interrupt/irq.rs b/src/arch/x86_64/interrupt/irq.rs index 36191ee762..9e8d041cc4 100644 --- a/src/arch/x86_64/interrupt/irq.rs +++ b/src/arch/x86_64/interrupt/irq.rs @@ -2,14 +2,21 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::vec::Vec; -use crate::{interrupt, interrupt_stack}; -use crate::context::timeout; -use crate::device::{local_apic, ioapic, pic, pit}; -use crate::device::serial::{COM1, COM2}; -use crate::ipi::{ipi, IpiKind, IpiTarget}; -use crate::scheme::debug::{debug_input, debug_notify}; -use crate::scheme::serio::serio_input; -use crate::{context, time}; +use crate::{ + context, + context::timeout, + device::{ + ioapic, local_apic, pic, pit, + serial::{COM1, COM2}, + }, + interrupt, interrupt_stack, + ipi::{ipi, IpiKind, IpiTarget}, + scheme::{ + debug::{debug_input, debug_notify}, + serio::serio_input, + }, + time, +}; #[repr(u8)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -33,9 +40,13 @@ pub fn spurious_count() -> usize { pub fn spurious_irq_resource() -> syscall::Result> { match irq_method() { IrqMethod::Apic => Ok(Vec::from(&b"(not implemented for APIC yet)"[..])), - IrqMethod::Pic => { - Ok(format!("{}\tIRQ7\n{}\tIRQ15\n{}\ttotal\n", spurious_count_irq7(), spurious_count_irq15(), spurious_count()).into_bytes()) - } + IrqMethod::Pic => Ok(format!( + "{}\tIRQ7\n{}\tIRQ15\n{}\ttotal\n", + spurious_count_irq7(), + spurious_count_irq15(), + spurious_count() + ) + .into_bytes()), } } @@ -55,7 +66,7 @@ fn irq_method() -> IrqMethod { } } -extern { +extern "C" { // triggers irq scheme fn irq_trigger(irq: u8); } @@ -64,7 +75,11 @@ extern { /// scheme user unmasks it ("acknowledges" it). unsafe fn trigger(irq: u8) { match irq_method() { - IrqMethod::Pic => if irq < 16 { pic_mask(irq) }, + IrqMethod::Pic => { + if irq < 16 { + pic_mask(irq) + } + } IrqMethod::Apic => ioapic_mask(irq), } irq_trigger(irq); @@ -74,7 +89,11 @@ unsafe fn trigger(irq: u8) { /// processed the IRQ. pub unsafe fn acknowledge(irq: usize) { match irq_method() { - IrqMethod::Pic => if irq < 16 { pic_unmask(irq) }, + IrqMethod::Pic => { + if irq < 16 { + pic_unmask(irq) + } + } IrqMethod::Apic => ioapic_unmask(irq), } } @@ -82,7 +101,11 @@ pub unsafe fn acknowledge(irq: usize) { /// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one. pub unsafe fn eoi(irq: u8) { match irq_method() { - IrqMethod::Pic => if irq < 16 { pic_eoi(irq) }, + IrqMethod::Pic => { + if irq < 16 { + pic_eoi(irq) + } + } IrqMethod::Apic => lapic_eoi(), } } @@ -242,7 +265,7 @@ interrupt!(ata2, || { if irq_method() == IrqMethod::Pic && pic::SLAVE.isr() & (1 << 7) == 0 { SPURIOUS_COUNT_IRQ15.fetch_add(1, Ordering::Relaxed); pic::MASTER.ack(); - return + return; } trigger(15); eoi(15); @@ -259,7 +282,10 @@ interrupt!(aux_timer, || { }); interrupt!(lapic_error, || { - println!("Local apic internal error: ESR={:#0x}", local_apic::LOCAL_APIC.esr()); + println!( + "Local apic internal error: ESR={:#0x}", + local_apic::LOCAL_APIC.esr() + ); lapic_eoi(); }); diff --git a/src/arch/x86_64/interrupt/mod.rs b/src/arch/x86_64/interrupt/mod.rs index ebd013f23d..7b212b07b4 100644 --- a/src/arch/x86_64/interrupt/mod.rs +++ b/src/arch/x86_64/interrupt/mod.rs @@ -9,11 +9,12 @@ pub mod irq; pub mod syscall; pub mod trace; -pub use self::handler::InterruptStack; -pub use self::trace::stack_trace; +pub use self::{handler::InterruptStack, trace::stack_trace}; -pub use super::idt::{available_irqs_iter, is_reserved, set_reserved}; -pub use super::device::local_apic::bsp_apic_id; +pub use super::{ + device::local_apic::bsp_apic_id, + idt::{available_irqs_iter, is_reserved, set_reserved}, +}; /// Clear interrupts #[inline(always)] @@ -53,5 +54,7 @@ pub unsafe fn halt() { /// Safe because it is similar to a NOP, and has no memory effects #[inline(always)] pub fn pause() { - unsafe { core::arch::asm!("pause", options(nomem, nostack)); } + unsafe { + core::arch::asm!("pause", options(nomem, nostack)); + } } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index 5c2c1fb46e..618185a54d 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -1,12 +1,14 @@ use crate::{ arch::{gdt, interrupt::InterruptStack}, - context, - ptrace, - syscall, - syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_PRE_SYSCALL, PTRACE_STOP_POST_SYSCALL}, + context, ptrace, syscall, + syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL}, }; use core::mem::offset_of; -use x86::{bits64::{rflags::RFlags, task::TaskStateSegment}, msr, segmentation::SegmentSelector}; +use x86::{ + bits64::{rflags::RFlags, task::TaskStateSegment}, + msr, + segmentation::SegmentSelector, +}; pub unsafe fn init() { // IA32_STAR[31:0] are reserved. @@ -46,7 +48,12 @@ pub unsafe fn init() { // user and kernel mode), VM8086 (not used at all), and VIF/VIP (system-level status flags?). let mask_critical = RFlags::FLAGS_DF | RFlags::FLAGS_IF | RFlags::FLAGS_TF | RFlags::FLAGS_AC; - let mask_other = RFlags::FLAGS_CF | RFlags::FLAGS_PF | RFlags::FLAGS_AF | RFlags::FLAGS_ZF | RFlags::FLAGS_SF | RFlags::FLAGS_OF; + let mask_other = RFlags::FLAGS_CF + | RFlags::FLAGS_PF + | RFlags::FLAGS_AF + | RFlags::FLAGS_ZF + | RFlags::FLAGS_SF + | RFlags::FLAGS_OF; msr::wrmsr(msr::IA32_FMASK, (mask_critical | mask_other).bits()); let efer = msr::rdmsr(msr::IA32_EFER); @@ -68,7 +75,7 @@ macro_rules! with_interrupt_stack { } ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None); - }} + }}; } #[no_mangle] @@ -76,7 +83,15 @@ pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) let _guard = ptrace::set_process_regs(stack); with_interrupt_stack!(|stack| { let scratch = &stack.scratch; - syscall::syscall(scratch.rax, scratch.rdi, scratch.rsi, scratch.rdx, scratch.r10, scratch.r8, stack) + syscall::syscall( + scratch.rax, + scratch.rdi, + scratch.rsi, + scratch.rdx, + scratch.r10, + scratch.r8, + stack, + ) }); } @@ -193,13 +208,24 @@ interrupt_stack!(syscall, |stack| { let context = contexts.current(); if let Some(current) = context { let current = current.read(); - println!("Warning: Context {} used deprecated `int 0x80` construct", current.name); + println!( + "Warning: Context {} used deprecated `int 0x80` construct", + current.name + ); } else { println!("Warning: Unknown context used deprecated `int 0x80` construct"); } } let scratch = &stack.scratch; - syscall::syscall(scratch.rax, stack.preserved.rbx, scratch.rcx, scratch.rdx, scratch.rsi, scratch.rdi, stack) + syscall::syscall( + scratch.rax, + stack.preserved.rbx, + scratch.rcx, + scratch.rdx, + scratch.rsi, + scratch.rdi, + stack, + ) }) }); diff --git a/src/arch/x86_64/interrupt/trace.rs b/src/arch/x86_64/interrupt/trace.rs index 938f6aa4d3..ad99389cf6 100644 --- a/src/arch/x86_64/interrupt/trace.rs +++ b/src/arch/x86_64/interrupt/trace.rs @@ -3,7 +3,10 @@ use core::{mem, str}; use goblin::elf::sym; use rustc_demangle::demangle; -use crate::{paging::{KernelMapper, VirtualAddress}, USER_END_OFFSET}; +use crate::{ + paging::{KernelMapper, VirtualAddress}, + USER_END_OFFSET, +}; /// Get a stack trace //TODO: Check for stack being mapped before dereferencing @@ -21,7 +24,11 @@ pub unsafe fn stack_trace() { if let Some(rip_rbp) = rbp.checked_add(mem::size_of::()) { let rbp_virt = VirtualAddress::new(rbp); let rip_rbp_virt = VirtualAddress::new(rip_rbp); - if rbp_virt.data() >= USER_END_OFFSET && rip_rbp_virt.data() >= USER_END_OFFSET && mapper.translate(rbp_virt).is_some() && mapper.translate(rip_rbp_virt).is_some() { + if rbp_virt.data() >= USER_END_OFFSET + && rip_rbp_virt.data() >= USER_END_OFFSET + && mapper.translate(rbp_virt).is_some() + && mapper.translate(rip_rbp_virt).is_some() + { let rip = (rip_rbp as *const usize).read(); if rip == 0 { println!(" {:>016X}: EMPTY RETURN", rbp); @@ -45,11 +52,12 @@ pub unsafe fn stack_trace() { //TODO: Do not create Elf object for every symbol lookup #[inline(never)] pub unsafe fn symbol_trace(addr: usize) { - use core::slice; - use core::sync::atomic::Ordering; + use core::{slice, sync::atomic::Ordering}; - use crate::elf::Elf; - use crate::start::{KERNEL_BASE, KERNEL_SIZE}; + use crate::{ + elf::Elf, + start::{KERNEL_BASE, KERNEL_SIZE}, + }; let kernel_ptr = (KERNEL_BASE.load(Ordering::SeqCst) + crate::PHYS_OFFSET) as *const u8; let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst)); @@ -65,10 +73,14 @@ pub unsafe fn symbol_trace(addr: usize) { if let Some(symbols) = elf.symbols() { for sym in symbols { if sym::st_type(sym.st_info) == sym::STT_FUNC - && addr >= sym.st_value as usize - && addr < (sym.st_value + sym.st_size) as usize + && addr >= sym.st_value as usize + && addr < (sym.st_value + sym.st_size) as usize { - println!(" {:>016X}+{:>04X}", sym.st_value, addr - sym.st_value as usize); + println!( + " {:>016X}+{:>04X}", + sym.st_value, + addr - sym.st_value as usize + ); if let Some(strtab) = strtab_opt { let start = strtab.sh_offset as usize + sym.st_name as usize; @@ -82,7 +94,7 @@ pub unsafe fn symbol_trace(addr: usize) { } if end > start { - let sym_slice = &elf.data[start .. end - 1]; + let sym_slice = &elf.data[start..end - 1]; if let Ok(sym_name) = str::from_utf8(sym_slice) { println!(" {:#}", demangle(sym_name)); } diff --git a/src/arch/x86_64/macros.rs b/src/arch/x86_64/macros.rs index 16303b5ce6..936e1a205c 100644 --- a/src/arch/x86_64/macros.rs +++ b/src/arch/x86_64/macros.rs @@ -95,4 +95,3 @@ macro_rules! alternative_auto( ", ) } ); - diff --git a/src/arch/x86_64/misc.rs b/src/arch/x86_64/misc.rs index 181efaf357..b193719b15 100644 --- a/src/arch/x86_64/misc.rs +++ b/src/arch/x86_64/misc.rs @@ -1,7 +1,9 @@ use x86::controlregs::Cr4; -use crate::LogicalCpuId; -use crate::cpuid::{cpuid_always, has_ext_feat}; +use crate::{ + cpuid::{cpuid_always, has_ext_feat}, + LogicalCpuId, +}; pub unsafe fn init(cpu_id: LogicalCpuId) { if has_ext_feat(|feat| feat.has_umip()) { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index a900337bb0..f749e5a953 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -68,9 +68,10 @@ pub mod flags { pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 { // TODO: spectre_v1 - core::arch::asm!(alternative!( - feature: "smap", - then: [" + core::arch::asm!( + alternative!( + feature: "smap", + then: [" xor eax, eax mov rcx, rdx stac @@ -78,19 +79,24 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - clac ret "], - default: [" + default: [" xor eax, eax mov rcx, rdx rep movsb ret "] - ), options(noreturn)); + ), + options(noreturn) + ); } pub use arch_copy_to_user as arch_copy_from_user; // TODO: This doesn't need to be arch-specific, right? pub unsafe fn bootstrap_mem(bootstrap: &Bootstrap) -> &'static [u8] { - core::slice::from_raw_parts(CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, bootstrap.page_count * PAGE_SIZE) + core::slice::from_raw_parts( + CurrentRmmArch::phys_to_virt(bootstrap.base.start_address()).data() as *const u8, + bootstrap.page_count * PAGE_SIZE, + ) } pub use alternative::kfx_size; diff --git a/src/arch/x86_64/paging/mapper.rs b/src/arch/x86_64/paging/mapper.rs index 9f7659b4e6..1543c6ff80 100644 --- a/src/arch/x86_64/paging/mapper.rs +++ b/src/arch/x86_64/paging/mapper.rs @@ -4,16 +4,22 @@ use super::RmmA; pub use rmm::{Flusher, PageFlush, PageFlushAll}; -pub struct InactiveFlusher { _inner: () } +pub struct InactiveFlusher { + _inner: (), +} impl InactiveFlusher { // TODO: cpu id - pub fn new() -> Self { Self { _inner: () } } + pub fn new() -> Self { + Self { _inner: () } + } } impl Flusher for InactiveFlusher { fn consume(&mut self, flush: PageFlush) { // TODO: Push to TLB "mailbox" or tell it to reload CR3 if there are too many entries. - unsafe { flush.ignore(); } + unsafe { + flush.ignore(); + } } } impl Drop for InactiveFlusher { diff --git a/src/arch/x86_64/paging/mod.rs b/src/arch/x86_64/paging/mod.rs index c1d3f1430b..fc6f44ca24 100644 --- a/src/arch/x86_64/paging/mod.rs +++ b/src/arch/x86_64/paging/mod.rs @@ -5,15 +5,8 @@ use core::fmt::Debug; use x86::msr; -pub use rmm::{ - Arch as RmmArch, - Flusher, - PageFlags, - PhysicalAddress, - TableKind, - VirtualAddress, -}; pub use super::CurrentRmmArch as RmmA; +pub use rmm::{Arch as RmmArch, Flusher, PageFlags, PhysicalAddress, TableKind, VirtualAddress}; pub type PageMapper = rmm::PageMapper; pub use crate::rmm::KernelMapper; @@ -92,7 +85,10 @@ impl Page { } pub fn range_inclusive(start: Page, r#final: Page) -> PageIter { - PageIter { start, end: r#final.next() } + PageIter { + start, + end: r#final.next(), + } } pub fn range_exclusive(start: Page, end: Page) -> PageIter { PageIter { start, end } @@ -112,7 +108,11 @@ impl Page { } impl Debug for Page { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "[page at {:p}]", self.start_address().data() as *const u8) + write!( + f, + "[page at {:p}]", + self.start_address().data() as *const u8 + ) } } diff --git a/src/arch/x86_64/pti.rs b/src/arch/x86_64/pti.rs index 0f5cc07aad..c1e9422c84 100644 --- a/src/arch/x86_64/pti.rs +++ b/src/arch/x86_64/pti.rs @@ -4,9 +4,9 @@ use core::ptr; #[cfg(feature = "pti")] use crate::memory::Frame; #[cfg(feature = "pti")] -use crate::paging::ActivePageTable; -#[cfg(feature = "pti")] use crate::paging::entry::EntryFlags; +#[cfg(feature = "pti")] +use crate::paging::ActivePageTable; #[cfg(feature = "pti")] #[thread_local] @@ -26,11 +26,7 @@ unsafe fn switch_stack(old: usize, new: usize) { let new_rsp = new - offset_rsp; - ptr::copy_nonoverlapping( - old_rsp as *const u8, - new_rsp as *mut u8, - offset_rsp - ); + ptr::copy_nonoverlapping(old_rsp as *const u8, new_rsp as *mut u8, offset_rsp); asm!("", out("rsp") new_rsp); } @@ -53,14 +49,20 @@ pub unsafe fn map() { // } // Switch to per-context stack - switch_stack(PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len(), PTI_CONTEXT_STACK); + switch_stack( + PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len(), + PTI_CONTEXT_STACK, + ); } #[cfg(feature = "pti")] #[inline(always)] pub unsafe extern "C" fn unmap() { // Switch to per-CPU stack - switch_stack(PTI_CONTEXT_STACK, PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()); + switch_stack( + PTI_CONTEXT_STACK, + PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len(), + ); // { // let mut active_table = unsafe { ActivePageTable::new() }; diff --git a/src/arch/x86_64/rmm.rs b/src/arch/x86_64/rmm.rs index 68173a3c20..cc7255e06a 100644 --- a/src/arch/x86_64/rmm.rs +++ b/src/arch/x86_64/rmm.rs @@ -1,24 +1,11 @@ use core::{ - cmp, - mem, - slice, - sync::atomic::{self, AtomicUsize, Ordering}, cell::SyncUnsafeCell, + cell::SyncUnsafeCell, + cmp, mem, slice, + sync::atomic::{self, AtomicUsize, Ordering}, }; use rmm::{ - KILOBYTE, - MEGABYTE, - Arch, - BuddyAllocator, - BumpAllocator, - FrameAllocator, - FrameCount, - FrameUsage, - MemoryArea, - PageFlags, - PageMapper, - PhysicalAddress, - TableKind, - VirtualAddress, + Arch, BuddyAllocator, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, MemoryArea, + PageFlags, PageMapper, PhysicalAddress, TableKind, VirtualAddress, KILOBYTE, MEGABYTE, }; use spin::Mutex; @@ -63,11 +50,16 @@ unsafe fn page_flags(virt: VirtualAddress) -> PageFlags { unsafe fn inner( areas: &'static [MemoryArea], - kernel_base: usize, kernel_size_aligned: usize, - stack_base: usize, stack_size_aligned: usize, - env_base: usize, env_size_aligned: usize, - acpi_base: usize, acpi_size_aligned: usize, - initfs_base: usize, initfs_size_aligned: usize, + kernel_base: usize, + kernel_size_aligned: usize, + stack_base: usize, + stack_size_aligned: usize, + env_base: usize, + env_size_aligned: usize, + acpi_base: usize, + acpi_size_aligned: usize, + initfs_base: usize, + initfs_size_aligned: usize, ) -> BuddyAllocator { // First, calculate how much memory we have let mut size = 0; @@ -84,10 +76,8 @@ unsafe fn inner( let mut bump_allocator = BumpAllocator::::new(areas, 0); { - let mut mapper = PageMapper::::create( - TableKind::Kernel, - &mut bump_allocator - ).expect("failed to create Mapper"); + let mut mapper = PageMapper::::create(TableKind::Kernel, &mut bump_allocator) + .expect("failed to create Mapper"); // Map all physical areas at PHYS_OFFSET for area in areas.iter() { @@ -95,11 +85,9 @@ unsafe fn inner( let phys = area.base.add(i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -109,19 +97,15 @@ unsafe fn inner( let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE); let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table let virt = A::phys_to_virt(phys); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } @@ -131,11 +115,9 @@ unsafe fn inner( let phys = PhysicalAddress::new(base + i * A::PAGE_SIZE); let virt = A::phys_to_virt(phys); let flags = page_flags::(virt); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } }; @@ -148,8 +130,8 @@ unsafe fn inner( // Ensure graphical debug region remains paged #[cfg(feature = "graphical_debug")] { - use crate::devices::graphical_debug::FRAMEBUFFER; use super::paging::entry::EntryFlags; + use crate::devices::graphical_debug::FRAMEBUFFER; let (phys, virt, size) = *FRAMEBUFFER.lock(); @@ -157,14 +139,13 @@ unsafe fn inner( for i in 0..pages { let phys = PhysicalAddress::new(phys + i * A::PAGE_SIZE); let virt = VirtualAddress::new(virt + i * A::PAGE_SIZE); - let flags = PageFlags::new().write(true) + let flags = PageFlags::new() + .write(true) // Write combining flag .custom_flag(EntryFlags::HUGE_PAGE.bits(), true); - let flush = mapper.map_phys( - virt, - phys, - flags - ).expect("failed to map frame"); + let flush = mapper + .map_phys(virt, phys, flags) + .expect("failed to map frame"); flush.ignore(); // Not the active table } } @@ -184,7 +165,10 @@ unsafe fn inner( // Create the physical memory map let offset = bump_allocator.offset(); - log::info!("Permanently used: {} KB", (offset + (KILOBYTE - 1)) / KILOBYTE); + log::info!( + "Permanently used: {} KB", + (offset + (KILOBYTE - 1)) / KILOBYTE + ); BuddyAllocator::::new(bump_allocator).expect("failed to create BuddyAllocator") } @@ -228,10 +212,12 @@ impl core::fmt::Debug for LockedAllocator { } } -static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new([MemoryArea { - base: PhysicalAddress::new(0), - size: 0, -}; 512]); +static AREAS: SyncUnsafeCell<[MemoryArea; 512]> = SyncUnsafeCell::new( + [MemoryArea { + base: PhysicalAddress::new(0), + size: 0, + }; 512], +); static AREA_COUNT: SyncUnsafeCell = SyncUnsafeCell::new(0); pub fn areas() -> &'static [MemoryArea] { @@ -263,7 +249,12 @@ pub struct KernelMapper { impl KernelMapper { fn lock_inner(current_processor: usize) -> bool { loop { - match LOCK_OWNER.compare_exchange_weak(NO_PROCESSOR, current_processor, Ordering::Acquire, Ordering::Relaxed) { + match LOCK_OWNER.compare_exchange_weak( + NO_PROCESSOR, + current_processor, + Ordering::Acquire, + Ordering::Relaxed, + ) { Ok(_) => break, // already owned by this hardware thread Err(id) if id == current_processor => break, @@ -277,15 +268,20 @@ impl KernelMapper { prev_count > 0 } - pub unsafe fn lock_for_manual_mapper(current_processor: LogicalCpuId, mapper: crate::paging::PageMapper) -> Self { + pub unsafe fn lock_for_manual_mapper( + current_processor: LogicalCpuId, + mapper: crate::paging::PageMapper, + ) -> Self { let ro = Self::lock_inner(current_processor.get() as usize); - Self { - mapper, - ro, - } + Self { mapper, ro } } pub fn lock_manually(current_processor: LogicalCpuId) -> Self { - unsafe { Self::lock_for_manual_mapper(current_processor, PageMapper::current(TableKind::Kernel, FRAME_ALLOCATOR)) } + unsafe { + Self::lock_for_manual_mapper( + current_processor, + PageMapper::current(TableKind::Kernel, FRAME_ALLOCATOR), + ) + } } pub fn lock() -> Self { Self::lock_manually(crate::cpu_id()) @@ -320,12 +316,18 @@ impl Drop for KernelMapper { } pub unsafe fn init( - kernel_base: usize, kernel_size: usize, - stack_base: usize, stack_size: usize, - env_base: usize, env_size: usize, - acpi_base: usize, acpi_size: usize, - areas_base: usize, areas_size: usize, - initfs_base: usize, initfs_size: usize, + kernel_base: usize, + kernel_size: usize, + stack_base: usize, + stack_size: usize, + env_base: usize, + env_size: usize, + acpi_base: usize, + acpi_size: usize, + areas_base: usize, + areas_size: usize, + initfs_base: usize, + initfs_size: usize, ) { type A = RmmA; @@ -333,24 +335,24 @@ pub unsafe fn init( let real_size = 0x100000; let real_end = real_base + real_size; - let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let kernel_end = kernel_base + kernel_size_aligned; - let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let stack_size_aligned = ((stack_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let stack_end = stack_base + stack_size_aligned; - let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let env_size_aligned = ((env_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let env_end = env_base + env_size_aligned; - let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let acpi_size_aligned = ((acpi_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let acpi_end = acpi_base + acpi_size_aligned; - let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE; + let initfs_size_aligned = ((initfs_size + (A::PAGE_SIZE - 1)) / A::PAGE_SIZE) * A::PAGE_SIZE; let initfs_end = initfs_base + initfs_size_aligned; let bootloader_areas = slice::from_raw_parts( areas_base as *const BootloaderMemoryEntry, - areas_size / mem::size_of::() + areas_size / mem::size_of::(), ); // Copy memory map from bootloader location, and page align it @@ -383,44 +385,86 @@ pub unsafe fn init( // Ensure real-mode areas are not used if base < real_end && base + size > real_base { - log::warn!("{:X}:{:X} overlaps with real mode {:X}:{:X}", base, size, real_base, real_size); + log::warn!( + "{:X}:{:X} overlaps with real mode {:X}:{:X}", + base, + size, + real_base, + real_size + ); new_base = cmp::max(new_base, real_end); } // Ensure kernel areas are not used if base < kernel_end && base + size > kernel_base { - log::warn!("{:X}:{:X} overlaps with kernel {:X}:{:X}", base, size, kernel_base, kernel_size); + log::warn!( + "{:X}:{:X} overlaps with kernel {:X}:{:X}", + base, + size, + kernel_base, + kernel_size + ); new_base = cmp::max(new_base, kernel_end); } // Ensure stack areas are not used if base < stack_end && base + size > stack_base { - log::warn!("{:X}:{:X} overlaps with stack {:X}:{:X}", base, size, stack_base, stack_size); + log::warn!( + "{:X}:{:X} overlaps with stack {:X}:{:X}", + base, + size, + stack_base, + stack_size + ); new_base = cmp::max(new_base, stack_end); } // Ensure env areas are not used if base < env_end && base + size > env_base { - log::warn!("{:X}:{:X} overlaps with env {:X}:{:X}", base, size, env_base, env_size); + log::warn!( + "{:X}:{:X} overlaps with env {:X}:{:X}", + base, + size, + env_base, + env_size + ); new_base = cmp::max(new_base, env_end); } // Ensure acpi areas are not used if base < acpi_end && base + size > acpi_base { - log::warn!("{:X}:{:X} overlaps with acpi {:X}:{:X}", base, size, acpi_base, acpi_size); + log::warn!( + "{:X}:{:X} overlaps with acpi {:X}:{:X}", + base, + size, + acpi_base, + acpi_size + ); new_base = cmp::max(new_base, acpi_end); } // Ensure initfs areas are not used if base < initfs_end && base + size > initfs_base { - log::warn!("{:X}:{:X} overlaps with initfs {:X}:{:X}", base, size, initfs_base, initfs_size); + log::warn!( + "{:X}:{:X} overlaps with initfs {:X}:{:X}", + base, + size, + initfs_base, + initfs_size + ); new_base = cmp::max(new_base, initfs_end); } if new_base != base { let end = base + size; let new_size = end.checked_sub(new_base).unwrap_or(0); - log::info!("{:X}:{:X} moved to {:X}:{:X}", base, size, new_base, new_size); + log::info!( + "{:X}:{:X} moved to {:X}:{:X}", + base, + size, + new_base, + new_size + ); base = new_base; size = new_size; } @@ -439,11 +483,16 @@ pub unsafe fn init( let allocator = inner::( areas(), - kernel_base, kernel_size_aligned, - stack_base, stack_size_aligned, - env_base, env_size_aligned, - acpi_base, acpi_size_aligned, - initfs_base, initfs_size_aligned, + kernel_base, + kernel_size_aligned, + stack_base, + stack_size_aligned, + env_base, + env_size_aligned, + acpi_base, + acpi_size_aligned, + initfs_base, + initfs_size_aligned, ); *INNER_ALLOCATOR.lock() = Some(allocator); } diff --git a/src/arch/x86_64/start.rs b/src/arch/x86_64/start.rs index b7ac9fd919..7c2b8fa54c 100644 --- a/src/arch/x86_64/start.rs +++ b/src/arch/x86_64/start.rs @@ -2,24 +2,22 @@ /// It is increcibly unsafe, and should be minimal in nature /// It must create the IDT with the correct entries, those entries are /// defined in other files inside of the `arch` module - use core::slice; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering, AtomicU32}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use crate::{allocator, memory, LogicalCpuId}; #[cfg(feature = "acpi")] use crate::acpi; -use crate::arch::pti; -use crate::arch::flags::*; -use crate::device; #[cfg(feature = "graphical_debug")] use crate::devices::graphical_debug; -use crate::gdt; -use crate::idt; -use crate::interrupt; -use crate::misc; -use crate::log::{self, info}; -use crate::paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind}; +use crate::{ + allocator, + arch::{flags::*, pti}, + device, gdt, idt, interrupt, + log::{self, info}, + memory, misc, + paging::{self, PhysicalAddress, RmmA, RmmArch, TableKind}, + LogicalCpuId, +}; /// Test of zero values in BSS. static BSS_TEST_ZERO: usize = 0; @@ -68,7 +66,7 @@ pub struct KernelArgs { /// The entry to Rust, all things must be initialized #[no_mangle] -pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { +pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { let bootstrap = { let args = args_ptr.read(); @@ -82,7 +80,10 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { KERNEL_SIZE.store(args.kernel_size as usize, Ordering::SeqCst); // Convert env to slice - let env = slice::from_raw_parts((args.env_base as usize + crate::PHYS_OFFSET) as *const u8, args.env_size as usize); + let env = slice::from_raw_parts( + (args.env_base as usize + crate::PHYS_OFFSET) as *const u8, + args.env_size as usize, + ); // Set up graphical debug #[cfg(feature = "graphical_debug")] @@ -104,12 +105,36 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { }); info!("Redox OS starting..."); - info!("Kernel: {:X}:{:X}", { args.kernel_base }, { args.kernel_base } + { args.kernel_size }); - info!("Stack: {:X}:{:X}", { args.stack_base }, { args.stack_base } + { args.stack_size }); - info!("Env: {:X}:{:X}", { args.env_base }, { args.env_base } + { args.env_size }); - info!("RSDPs: {:X}:{:X}", { args.acpi_rsdps_base }, { args.acpi_rsdps_base } + { args.acpi_rsdps_size }); - info!("Areas: {:X}:{:X}", { args.areas_base }, { args.areas_base } + { args.areas_size }); - info!("Bootstrap: {:X}:{:X}", { args.bootstrap_base }, { args.bootstrap_base } + { args.bootstrap_size }); + info!( + "Kernel: {:X}:{:X}", + { args.kernel_base }, + { args.kernel_base } + { args.kernel_size } + ); + info!( + "Stack: {:X}:{:X}", + { args.stack_base }, + { args.stack_base } + { args.stack_size } + ); + info!( + "Env: {:X}:{:X}", + { args.env_base }, + { args.env_base } + { args.env_size } + ); + info!( + "RSDPs: {:X}:{:X}", + { args.acpi_rsdps_base }, + { args.acpi_rsdps_base } + { args.acpi_rsdps_size } + ); + info!( + "Areas: {:X}:{:X}", + { args.areas_base }, + { args.areas_base } + { args.areas_size } + ); + info!( + "Bootstrap: {:X}:{:X}", + { args.bootstrap_base }, + { args.bootstrap_base } + { args.bootstrap_size } + ); info!("Bootstrap entry point: {:X}", { args.bootstrap_entry }); // Set up GDT before paging @@ -120,19 +145,28 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { // Initialize RMM crate::arch::rmm::init( - args.kernel_base as usize, args.kernel_size as usize, - args.stack_base as usize, args.stack_size as usize, - args.env_base as usize, args.env_size as usize, - args.acpi_rsdps_base as usize, args.acpi_rsdps_size as usize, - args.areas_base as usize, args.areas_size as usize, - args.bootstrap_base as usize, args.bootstrap_size as usize, + args.kernel_base as usize, + args.kernel_size as usize, + args.stack_base as usize, + args.stack_size as usize, + args.env_base as usize, + args.env_size as usize, + args.acpi_rsdps_base as usize, + args.acpi_rsdps_size as usize, + args.areas_base as usize, + args.areas_size as usize, + args.bootstrap_base as usize, + args.bootstrap_size as usize, ); // Initialize PAT paging::init(); // Set up GDT after paging with TLS - gdt::init_paging(args.stack_base as usize + args.stack_size as usize, LogicalCpuId::BSP); + gdt::init_paging( + args.stack_base as usize + args.stack_size as usize, + LogicalCpuId::BSP, + ); // Set up IDT idt::init_paging_bsp(); @@ -172,7 +206,10 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { #[cfg(feature = "acpi")] { acpi::init(if args.acpi_rsdps_base != 0 && args.acpi_rsdps_size > 0 { - Some(((args.acpi_rsdps_base as usize + crate::PHYS_OFFSET) as u64, args.acpi_rsdps_size as u64)) + Some(( + (args.acpi_rsdps_base as usize + crate::PHYS_OFFSET) as u64, + args.acpi_rsdps_size as u64, + )) } else { None }); @@ -192,7 +229,9 @@ pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! { BSP_READY.store(true, Ordering::SeqCst); crate::Bootstrap { - base: crate::memory::Frame::containing_address(crate::paging::PhysicalAddress::new(args.bootstrap_base as usize)), + base: crate::memory::Frame::containing_address(crate::paging::PhysicalAddress::new( + args.bootstrap_base as usize, + )), page_count: (args.bootstrap_size as usize) / crate::memory::PAGE_SIZE, entry: args.bootstrap_entry as usize, env, @@ -213,7 +252,7 @@ pub struct KernelArgsAp { } /// Entry to rust for an AP -pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { +pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { let cpu_id = { let args = &*args_ptr; let cpu_id = LogicalCpuId::new(args.cpu_id as u32); @@ -259,7 +298,7 @@ pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! { cpu_id }; - while ! BSP_READY.load(Ordering::SeqCst) { + while !BSP_READY.load(Ordering::SeqCst) { interrupt::pause(); } diff --git a/src/arch/x86_64/stop.rs b/src/arch/x86_64/stop.rs index 246d1ddde2..50ceab3ade 100644 --- a/src/arch/x86_64/stop.rs +++ b/src/arch/x86_64/stop.rs @@ -1,14 +1,10 @@ #[cfg(feature = "acpi")] -use crate::{ - context, - scheme::acpi, - time, -}; +use crate::{context, scheme::acpi, time}; use crate::syscall::io::{Io, Pio}; #[no_mangle] -pub unsafe extern fn kreset() -> ! { +pub unsafe extern "C" fn kreset() -> ! { println!("kreset"); // 8042 reset @@ -24,11 +20,14 @@ pub unsafe extern fn kreset() -> ! { pub unsafe fn emergency_reset() -> ! { // Use triple fault to guarantee reset - core::arch::asm!(" + core::arch::asm!( + " cli lidt cs:0 int $3 - ", options(noreturn)); + ", + options(noreturn) + ); } #[cfg(feature = "acpi")] @@ -36,7 +35,7 @@ fn userspace_acpi_shutdown() { log::info!("Notifying any potential ACPI driver"); // Tell whatever driver that handles ACPI, that it should enter the S5 state (i.e. // shutdown). - if ! acpi::register_kstop() { + if !acpi::register_kstop() { // There was no context to switch to. log::info!("No ACPI driver was alive to handle shutdown."); return; @@ -63,7 +62,7 @@ fn userspace_acpi_shutdown() { } #[no_mangle] -pub unsafe extern fn kstop() -> ! { +pub unsafe extern "C" fn kstop() -> ! { log::info!("Running kstop()"); #[cfg(feature = "acpi")] diff --git a/src/common/aligned_box.rs b/src/common/aligned_box.rs index 10a5fd091d..447225b73e 100644 --- a/src/common/aligned_box.rs +++ b/src/common/aligned_box.rs @@ -1,8 +1,6 @@ -use core::alloc::Layout; -use core::alloc::GlobalAlloc; +use core::alloc::{GlobalAlloc, Layout}; -use crate::common::unique::Unique; -use crate::memory::Enomem; +use crate::{common::unique::Unique, memory::Enomem}; // Necessary because GlobalAlloc::dealloc requires the layout to be the same, and therefore Box // cannot be used for increased alignment directly. @@ -24,7 +22,11 @@ impl AlignedBox { } const fn layout_upgrade_align(layout: Layout, align: usize) -> Layout { const fn max(a: usize, b: usize) -> usize { - if a > b { a } else { b } + if a > b { + a + } else { + b + } } let Ok(x) = Layout::from_size_align(layout.size(), max(align, layout.align())) else { panic!("failed to calculate layout"); @@ -39,7 +41,8 @@ impl AlignedBox { T: ValidForZero, { Ok(unsafe { - let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::new::(), ALIGN)); + let ptr = + crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::new::(), ALIGN)); if ptr.is_null() { return Err(Enomem); } @@ -56,7 +59,10 @@ impl AlignedBox<[T], ALIGN> { T: ValidForZero, { Ok(unsafe { - let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align(Layout::array::(len).unwrap(), ALIGN)); + let ptr = crate::ALLOCATOR.alloc_zeroed(layout_upgrade_align( + Layout::array::(len).unwrap(), + ALIGN, + )); if ptr.is_null() { return Err(Enomem); } @@ -69,7 +75,13 @@ impl AlignedBox<[T], ALIGN> { impl core::fmt::Debug for AlignedBox { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "[aligned box at {:p}, size {} alignment {}]", self.inner.as_ptr(), self.layout().size(), self.layout().align()) + write!( + f, + "[aligned box at {:p}, size {} alignment {}]", + self.inner.as_ptr(), + self.layout().size(), + self.layout().align() + ) } } impl Drop for AlignedBox { @@ -95,14 +107,16 @@ impl core::ops::DerefMut for AlignedBox } impl Clone for AlignedBox { fn clone(&self) -> Self { - let mut new = Self::try_zeroed().unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout())); + let mut new = + Self::try_zeroed().unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout())); T::clone_from(&mut new, self); new } } impl Clone for AlignedBox<[T], ALIGN> { fn clone(&self) -> Self { - let mut new = Self::try_zeroed_slice(self.len()).unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout())); + let mut new = Self::try_zeroed_slice(self.len()) + .unwrap_or_else(|_| alloc::alloc::handle_alloc_error(self.layout())); for i in 0..self.len() { new[i].clone_from(&self[i]); } diff --git a/src/common/int_like.rs b/src/common/int_like.rs index 5596e9c4fa..a6b5a9f1e5 100644 --- a/src/common/int_like.rs +++ b/src/common/int_like.rs @@ -69,7 +69,7 @@ macro_rules! int_like { #[inline] pub const fn new(x: $new_type_name) -> Self { $new_atomic_type_name { - container: $backing_atomic_type::new(x.get()) + container: $backing_atomic_type::new(x.get()), } } #[allow(dead_code)] @@ -84,23 +84,47 @@ macro_rules! int_like { } #[allow(dead_code)] #[inline] - pub fn swap(&self, val: $new_type_name, order: ::core::sync::atomic::Ordering) -> $new_type_name { + pub fn swap( + &self, + val: $new_type_name, + order: ::core::sync::atomic::Ordering, + ) -> $new_type_name { $new_type_name::from(self.container.swap(val.into(), order)) } #[allow(dead_code)] #[inline] - pub fn compare_exchange(&self, current: $new_type_name, new: $new_type_name, success: ::core::sync::atomic::Ordering, failure: ::core::sync::atomic::Ordering) -> ::core::result::Result<$new_type_name, $new_type_name> { - match self.container.compare_exchange(current.into(), new.into(), success, failure) { + pub fn compare_exchange( + &self, + current: $new_type_name, + new: $new_type_name, + success: ::core::sync::atomic::Ordering, + failure: ::core::sync::atomic::Ordering, + ) -> ::core::result::Result<$new_type_name, $new_type_name> { + match self + .container + .compare_exchange(current.into(), new.into(), success, failure) + { Ok(result) => Ok($new_type_name::from(result)), - Err(result) => Err($new_type_name::from(result)) + Err(result) => Err($new_type_name::from(result)), } } #[allow(dead_code)] #[inline] - pub fn compare_exchange_weak(&self, current: $new_type_name, new: $new_type_name, success: ::core::sync::atomic::Ordering, failure: ::core::sync::atomic::Ordering) -> ::core::result::Result<$new_type_name, $new_type_name> { - match self.container.compare_exchange_weak(current.into(), new.into(), success, failure) { + pub fn compare_exchange_weak( + &self, + current: $new_type_name, + new: $new_type_name, + success: ::core::sync::atomic::Ordering, + failure: ::core::sync::atomic::Ordering, + ) -> ::core::result::Result<$new_type_name, $new_type_name> { + match self.container.compare_exchange_weak( + current.into(), + new.into(), + success, + failure, + ) { Ok(result) => Ok($new_type_name::from(result)), - Err(result) => Err($new_type_name::from(result)) + Err(result) => Err($new_type_name::from(result)), } } } @@ -110,23 +134,20 @@ macro_rules! int_like { Self::new($new_type_name::new(0)) } } - } + }; } #[test] fn test() { - use core::mem::size_of; use ::core::sync::atomic::AtomicUsize; + use core::mem::size_of; // Generate type `usize_like`. int_like!(UsizeLike, usize); assert_eq!(size_of::(), size_of::()); - // Generate types `usize_like` and `AtomicUsize`. int_like!(UsizeLike2, AtomicUsizeLike, usize, AtomicUsize); assert_eq!(size_of::(), size_of::()); assert_eq!(size_of::(), size_of::()); } - - diff --git a/src/common/mod.rs b/src/common/mod.rs index 7a2a38289e..9f2a70f77c 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -26,5 +26,3 @@ macro_rules! dbg { ($($crate::dbg!($val)),+,) }; } - - diff --git a/src/context/arch/aarch64.rs b/src/context/arch/aarch64.rs index 9f7fe6f70c..3521c9b632 100644 --- a/src/context/arch/aarch64.rs +++ b/src/context/arch/aarch64.rs @@ -1,16 +1,20 @@ use alloc::sync::Arc; -use core::arch::asm; -use core::mem; -use core::ptr; -use core::sync::atomic::{AtomicBool, Ordering}; -use core::mem::offset_of; +use core::{ + arch::asm, + mem, + mem::offset_of, + ptr, + sync::atomic::{AtomicBool, Ordering}, +}; use spin::Once; -use crate::{push_scratch, pop_scratch}; -use crate::interrupt::handler::ScratchRegisters; -use crate::device::cpu::registers::{control_regs, tlb}; -use crate::paging::{RmmA, RmmArch, TableKind}; -use crate::syscall::FloatRegisters; +use crate::{ + device::cpu::registers::{control_regs, tlb}, + interrupt::handler::ScratchRegisters, + paging::{RmmA, RmmArch, TableKind}, + pop_scratch, push_scratch, + syscall::FloatRegisters, +}; /// This must be used by the kernel to ensure that context switches are done atomically /// Compare and exchange this to true when beginning a context switch on any CPU @@ -25,24 +29,24 @@ pub const KFX_ALIGN: usize = 16; pub struct Context { elr_el1: usize, sp_el0: usize, - pub(crate) tpidr_el0: usize, /* Pointer to TLS region for this Context */ + pub(crate) tpidr_el0: usize, /* Pointer to TLS region for this Context */ pub(crate) tpidrro_el0: usize, /* Pointer to TLS (read-only) region for this Context */ spsr_el1: usize, esr_el1: usize, fx_loadable: bool, - sp: usize, /* Stack Pointer (x31) */ - lr: usize, /* Link Register (x30) */ - fp: usize, /* Frame pointer Register (x29) */ - x28: usize, /* Callee saved Register */ - x27: usize, /* Callee saved Register */ - x26: usize, /* Callee saved Register */ - x25: usize, /* Callee saved Register */ - x24: usize, /* Callee saved Register */ - x23: usize, /* Callee saved Register */ - x22: usize, /* Callee saved Register */ - x21: usize, /* Callee saved Register */ - x20: usize, /* Callee saved Register */ - x19: usize, /* Callee saved Register */ + sp: usize, /* Stack Pointer (x31) */ + lr: usize, /* Link Register (x30) */ + fp: usize, /* Frame pointer Register (x29) */ + x28: usize, /* Callee saved Register */ + x27: usize, /* Callee saved Register */ + x26: usize, /* Callee saved Register */ + x25: usize, /* Callee saved Register */ + x24: usize, /* Callee saved Register */ + x23: usize, /* Callee saved Register */ + x22: usize, /* Callee saved Register */ + x21: usize, /* Callee saved Register */ + x20: usize, /* Callee saved Register */ + x19: usize, /* Callee saved Register */ } impl Context { @@ -92,7 +96,7 @@ impl Context { self.tpidrro_el0 } - pub unsafe fn signal_stack(&mut self, handler: extern fn(usize), sig: u8) { + pub unsafe fn signal_stack(&mut self, handler: extern "C" fn(usize), sig: u8) { let lr = self.lr.clone(); self.push_pair((sig as usize, lr)); self.push_pair((0 as usize, handler as usize)); @@ -148,9 +152,7 @@ impl super::Context { panic!("TODO: make get_fx_regs always work"); } - unsafe { - ptr::read(self.kfx.as_ptr() as *const FloatRegisters) - } + unsafe { ptr::read(self.kfx.as_ptr() as *const FloatRegisters) } } pub fn set_fx_regs(&mut self, mut new: FloatRegisters) { @@ -231,16 +233,22 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { // Since Arc is essentially just wraps a pointer, in this case a regular pointer (as // opposed to dyn or slice fat pointers), and NonNull optimization exists, map_or will // hopefully be optimized down to checking prev and next pointers, as next cannot be null. - Some(ref next_space) => if prev.addr_space.as_ref().map_or(true, |prev_space| !Arc::ptr_eq(&prev_space, &next_space)) { - // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A - // recently called yield and is now here about to switch back. Meanwhile, B is - // currently creating a new mapping in their shared address space, for example a - // message on a channel. - // - // Unless we acquire this lock, it may be possible that the TLB will not contain new - // entries. While this can be caught and corrected in a page fault handler, this is not - // true when entries are removed from a page table! - next_space.read().table.utable.make_current(); + Some(ref next_space) => { + if prev + .addr_space + .as_ref() + .map_or(true, |prev_space| !Arc::ptr_eq(&prev_space, &next_space)) + { + // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A + // recently called yield and is now here about to switch back. Meanwhile, B is + // currently creating a new mapping in their shared address space, for example a + // message on a channel. + // + // Unless we acquire this lock, it may be possible that the TLB will not contain new + // entries. While this can be caught and corrected in a page fault handler, this is not + // true when entries are removed from a page table! + next_space.read().table.utable.make_current(); + } } None => { RmmA::set_table(TableKind::User, empty_cr3()); @@ -357,13 +365,13 @@ unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) { pub struct SignalHandlerStack { scratch: ScratchRegisters, padding: usize, - handler: extern fn(usize), + handler: extern "C" fn(usize), sig: usize, lr: usize, } #[naked] -unsafe extern fn signal_handler_wrapper() { +unsafe extern "C" fn signal_handler_wrapper() { #[inline(never)] unsafe extern "C" fn inner(stack: &SignalHandlerStack) { (stack.handler)(stack.sig); diff --git a/src/context/arch/x86.rs b/src/context/arch/x86.rs index d5fd31a479..ff2f5bfba7 100644 --- a/src/context/arch/x86.rs +++ b/src/context/arch/x86.rs @@ -1,13 +1,14 @@ -use core::mem; -use core::sync::atomic::AtomicBool; +use core::{mem, sync::atomic::AtomicBool}; use alloc::sync::Arc; -use crate::{push_scratch, pop_scratch}; -use crate::gdt::{pcr, GDT_USER_FS, GDT_USER_GS}; -use crate::interrupt::handler::ScratchRegisters; -use crate::paging::{RmmA, RmmArch, TableKind}; -use crate::syscall::FloatRegisters; +use crate::{ + gdt::{pcr, GDT_USER_FS, GDT_USER_GS}, + interrupt::handler::ScratchRegisters, + paging::{RmmA, RmmArch, TableKind}, + pop_scratch, push_scratch, + syscall::FloatRegisters, +}; use core::mem::offset_of; use spin::Once; @@ -67,7 +68,7 @@ impl Context { self.esp = address; } - pub unsafe fn signal_stack(&mut self, handler: extern fn(usize), sig: u8) { + pub unsafe fn signal_stack(&mut self, handler: extern "C" fn(usize), sig: u8) { self.push_stack(sig as usize); self.push_stack(handler as usize); self.push_stack(signal_handler_wrapper as usize); @@ -148,16 +149,22 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { // Since Arc is essentially just wraps a pointer, in this case a regular pointer (as // opposed to dyn or slice fat pointers), and NonNull optimization exists, map_or will // hopefully be optimized down to checking prev and next pointers, as next cannot be null. - Some(ref next_space) => if prev.addr_space.as_ref().map_or(true, |prev_space| !Arc::ptr_eq(&prev_space, &next_space)) { - // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A - // recently called yield and is now here about to switch back. Meanwhile, B is - // currently creating a new mapping in their shared address space, for example a - // message on a channel. - // - // Unless we acquire this lock, it may be possible that the TLB will not contain new - // entries. While this can be caught and corrected in a page fault handler, this is not - // true when entries are removed from a page table! - next_space.read().table.utable.make_current(); + Some(ref next_space) => { + if prev + .addr_space + .as_ref() + .map_or(true, |prev_space| !Arc::ptr_eq(&prev_space, &next_space)) + { + // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A + // recently called yield and is now here about to switch back. Meanwhile, B is + // currently creating a new mapping in their shared address space, for example a + // message on a channel. + // + // Unless we acquire this lock, it may be possible that the TLB will not contain new + // entries. While this can be caught and corrected in a page fault handler, this is not + // true when entries are removed from a page table! + next_space.read().table.utable.make_current(); + } } None => { RmmA::set_table(TableKind::User, empty_cr3()); @@ -238,7 +245,7 @@ unsafe extern "cdecl" fn switch_to_inner() { #[repr(packed)] pub struct SignalHandlerStack { scratch: ScratchRegisters, - handler: extern fn(usize), + handler: extern "C" fn(usize), sig: usize, eip: usize, } diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 01e0e6b13f..6eaa3ca074 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -1,13 +1,17 @@ -use core::mem; -use core::ptr::{addr_of, addr_of_mut}; -use core::sync::atomic::AtomicBool; +use core::{ + mem, + ptr::{addr_of, addr_of_mut}, + sync::atomic::AtomicBool, +}; use alloc::sync::Arc; -use crate::{push_scratch, pop_scratch}; -use crate::interrupt::handler::ScratchRegisters; -use crate::paging::{RmmA, RmmArch, TableKind}; -use crate::syscall::FloatRegisters; +use crate::{ + interrupt::handler::ScratchRegisters, + paging::{RmmA, RmmArch, TableKind}, + pop_scratch, push_scratch, + syscall::FloatRegisters, +}; use core::mem::offset_of; use spin::Once; @@ -78,7 +82,7 @@ impl Context { self.rsp = address; } - pub unsafe fn signal_stack(&mut self, handler: extern fn(usize), sig: u8) { + pub unsafe fn signal_stack(&mut self, handler: extern "C" fn(usize), sig: u8) { self.push_stack(sig as usize); self.push_stack(handler as usize); self.push_stack(signal_handler_wrapper as usize); @@ -217,16 +221,22 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { // Since Arc essentially just wraps a pointer, in this case a regular pointer (as opposed // to dyn or slice fat pointers), and NonNull optimization exists, map_or will hopefully be // optimized down to checking prev and next pointers, as next cannot be null. - Some(ref next_space) => if prev.addr_space.as_ref().map_or(true, |prev_space| !Arc::ptr_eq(prev_space, next_space)) { - // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A - // recently called yield and is now here about to switch back. Meanwhile, B is - // currently creating a new mapping in their shared address space, for example a - // message on a channel. - // - // Unless we acquire this lock, it may be possible that the TLB will not contain new - // entries. While this can be caught and corrected in a page fault handler, this is not - // true when entries are removed from a page table! - next_space.read().table.utable.make_current(); + Some(ref next_space) => { + if prev + .addr_space + .as_ref() + .map_or(true, |prev_space| !Arc::ptr_eq(prev_space, next_space)) + { + // Suppose we have two sibling threads A and B. A runs on CPU 0 and B on CPU 1. A + // recently called yield and is now here about to switch back. Meanwhile, B is + // currently creating a new mapping in their shared address space, for example a + // message on a channel. + // + // Unless we acquire this lock, it may be possible that the TLB will not contain new + // entries. While this can be caught and corrected in a page fault handler, this is not + // true when entries are removed from a page table! + next_space.read().table.utable.make_current(); + } } None => { RmmA::set_table(TableKind::User, empty_cr3()); @@ -308,13 +318,13 @@ unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Conte #[repr(packed)] pub struct SignalHandlerStack { scratch: ScratchRegisters, - handler: extern fn(usize), + handler: extern "C" fn(usize), sig: usize, rip: usize, } #[naked] -unsafe extern fn signal_handler_wrapper() { +unsafe extern "C" fn signal_handler_wrapper() { #[inline(never)] unsafe extern "C" fn inner(stack: &SignalHandlerStack) { (stack.handler)(stack.sig); diff --git a/src/context/context.rs b/src/context/context.rs index b98eb68f2d..6955d51759 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -1,31 +1,24 @@ -use core::{ - cmp::Ordering, - mem, -}; -use alloc::{ - boxed::Box, - collections::VecDeque, - sync::Arc, - vec::Vec, borrow::Cow, -}; +use alloc::{borrow::Cow, boxed::Box, collections::VecDeque, sync::Arc, vec::Vec}; +use core::{cmp::Ordering, mem}; use spin::RwLock; -use crate::{LogicalCpuId, LogicalCpuSet}; -use crate::arch::{interrupt::InterruptStack, paging::PAGE_SIZE}; -use crate::common::aligned_box::AlignedBox; -use crate::common::unique::Unique; -use crate::context::{self, arch}; -use crate::context::file::FileDescriptor; -use crate::context::memory::AddrSpace; -use crate::ipi::{ipi, IpiKind, IpiTarget}; -use crate::paging::{RmmA, RmmArch}; -use crate::memory::{RaiiFrame, Frame}; -use crate::scheme::{SchemeNamespace, FileHandle, CallerCtx}; -use crate::sync::WaitMap; +use crate::{ + arch::{interrupt::InterruptStack, paging::PAGE_SIZE}, + common::{aligned_box::AlignedBox, unique::Unique}, + context::{self, arch, file::FileDescriptor, memory::AddrSpace}, + ipi::{ipi, IpiKind, IpiTarget}, + memory::{Frame, RaiiFrame}, + paging::{RmmA, RmmArch}, + scheme::{CallerCtx, FileHandle, SchemeNamespace}, + sync::WaitMap, + LogicalCpuId, LogicalCpuSet, +}; -use crate::syscall::data::SigAction; -use crate::syscall::error::{Result, Error, EAGAIN, ESRCH}; -use crate::syscall::flag::{SIG_DFL, SigActionFlags}; +use crate::syscall::{ + data::SigAction, + error::{Error, Result, EAGAIN, ESRCH}, + flag::{SigActionFlags, SIG_DFL}, +}; /// Unique identifier for a context (i.e. `pid`). use ::core::sync::atomic::AtomicUsize; @@ -40,14 +33,15 @@ pub enum Status { Runnable, // TODO: Rename to SoftBlocked and move status_reason to this variant. - /// Not currently runnable, typically due to some blocking syscall, but it can be trivially /// unblocked by e.g. signals. Blocked, /// Not currently runnable, and cannot be runnable until manually unblocked, depending on what /// reason. - HardBlocked { reason: HardBlockedReason }, + HardBlocked { + reason: HardBlockedReason, + }, Stopped(usize), Exited(usize), @@ -188,11 +182,16 @@ pub struct Context { /// The architecture specific context pub arch: arch::Context, /// Kernel FX - used to store SIMD and FPU registers on context switch - pub kfx: AlignedBox<[u8], {arch::KFX_ALIGN}>, + pub kfx: AlignedBox<[u8], { arch::KFX_ALIGN }>, /// Kernel stack pub kstack: Option>, /// Kernel signal backup: Registers, Kernel FX, Kernel Stack, Signal number - pub ksig: Option<(arch::Context, AlignedBox<[u8], {arch::KFX_ALIGN}>, Option>, u8)>, + pub ksig: Option<( + arch::Context, + AlignedBox<[u8], { arch::KFX_ALIGN }>, + Option>, + u8, + )>, /// Restore ksig context on next switch pub ksig_restore: bool, /// Address space containing a page table lock, and grants. Normally this will have a value, @@ -257,7 +256,7 @@ impl Context { pending: VecDeque::new(), wake: None, arch: arch::Context::new(), - kfx: AlignedBox::<[u8], {arch::KFX_ALIGN}>::try_zeroed_slice(crate::arch::kfx_size())?, + kfx: AlignedBox::<[u8], { arch::KFX_ALIGN }>::try_zeroed_slice(crate::arch::kfx_size())?, kstack: None, ksig: None, ksig_restore: false, @@ -299,10 +298,10 @@ impl Context { pub fn unblock(&mut self) -> bool { if self.unblock_no_ipi() { if let Some(cpu_id) = self.cpu_id { - if cpu_id != crate::cpu_id() { + if cpu_id != crate::cpu_id() { // Send IPI if not on current CPU ipi(IpiKind::Wakeup, IpiTarget::Other); - } + } } true @@ -396,25 +395,37 @@ impl Context { pub fn addr_space(&self) -> Result<&Arc>> { self.addr_space.as_ref().ok_or(Error::new(ESRCH)) } - pub fn set_addr_space(&mut self, addr_space: Arc>) -> Option>> { + pub fn set_addr_space( + &mut self, + addr_space: Arc>, + ) -> Option>> { if self.id == super::context_id() { - unsafe { addr_space.read().table.utable.make_current(); } + unsafe { + addr_space.read().table.utable.make_current(); + } } self.addr_space.replace(addr_space) } pub fn empty_actions() -> Arc>> { - Arc::new(RwLock::new(vec![( - SigAction { - sa_handler: unsafe { mem::transmute(SIG_DFL) }, - sa_mask: [0; 2], - sa_flags: SigActionFlags::empty(), - }, - 0 - ); 128])) + Arc::new(RwLock::new(vec![ + ( + SigAction { + sa_handler: unsafe { mem::transmute(SIG_DFL) }, + sa_mask: [0; 2], + sa_flags: SigActionFlags::empty(), + }, + 0 + ); + 128 + ])) } pub fn caller_ctx(&self) -> CallerCtx { - CallerCtx { pid: self.id.into(), uid: self.euid, gid: self.egid } + CallerCtx { + pid: self.id.into(), + uid: self.euid, + gid: self.egid, + } } } @@ -427,21 +438,51 @@ pub struct BorrowedHtBuf { impl BorrowedHtBuf { pub fn head() -> Result { Ok(Self { - inner: Some(context::current()?.write().syscall_head.take().ok_or(Error::new(EAGAIN))?), + inner: Some( + context::current()? + .write() + .syscall_head + .take() + .ok_or(Error::new(EAGAIN))?, + ), head_and_not_tail: true, }) } pub fn tail() -> Result { Ok(Self { - inner: Some(context::current()?.write().syscall_tail.take().ok_or(Error::new(EAGAIN))?), + inner: Some( + context::current()? + .write() + .syscall_tail + .take() + .ok_or(Error::new(EAGAIN))?, + ), head_and_not_tail: false, }) } pub fn buf(&self) -> &[u8; PAGE_SIZE] { - unsafe { &*(RmmA::phys_to_virt(self.inner.as_ref().expect("must succeed").get().start_address()).data() as *const [u8; PAGE_SIZE]) } + unsafe { + &*(RmmA::phys_to_virt( + self.inner + .as_ref() + .expect("must succeed") + .get() + .start_address(), + ) + .data() as *const [u8; PAGE_SIZE]) + } } pub fn buf_mut(&mut self) -> &mut [u8; PAGE_SIZE] { - unsafe { &mut *(RmmA::phys_to_virt(self.inner.as_mut().expect("must succeed").get().start_address()).data() as *mut [u8; PAGE_SIZE]) } + unsafe { + &mut *(RmmA::phys_to_virt( + self.inner + .as_mut() + .expect("must succeed") + .get() + .start_address(), + ) + .data() as *mut [u8; PAGE_SIZE]) + } } pub fn frame(&self) -> Frame { self.inner.as_ref().expect("must succeed").get() @@ -477,7 +518,12 @@ impl Drop for BorrowedHtBuf { }; match context.write() { mut context => { - (if self.head_and_not_tail { &mut context.syscall_head } else { &mut context.syscall_tail }).get_or_insert(inner); + (if self.head_and_not_tail { + &mut context.syscall_head + } else { + &mut context.syscall_tail + }) + .get_or_insert(inner); } } } diff --git a/src/context/file.rs b/src/context/file.rs index fd882e92ee..4e49dd81a1 100644 --- a/src/context/file.rs +++ b/src/context/file.rs @@ -1,10 +1,12 @@ //! File structs +use crate::{ + event, + scheme::{self, SchemeId, SchemeNamespace}, + syscall::error::{Error, Result, EBADF}, +}; use alloc::sync::Arc; -use crate::event; use spin::RwLock; -use crate::scheme::{self, SchemeNamespace, SchemeId}; -use crate::syscall::error::{Result, Error, EBADF}; /// A file description #[derive(Clone, Copy, Debug)] @@ -36,7 +38,8 @@ impl FileDescription { event::unregister_file(self.scheme, self.number); let scheme = scheme::schemes() - .get(self.scheme).ok_or(Error::new(EBADF))? + .get(self.scheme) + .ok_or(Error::new(EBADF))? .clone(); scheme.close(self.number) diff --git a/src/context/list.rs b/src/context/list.rs index 3c83b40e1c..99c94eb86a 100644 --- a/src/context/list.rs +++ b/src/context/list.rs @@ -1,17 +1,16 @@ -use alloc::sync::Arc; -use alloc::collections::BTreeMap; +use alloc::{collections::BTreeMap, sync::Arc}; use core::{iter, mem}; use spin::RwLock; -use crate::syscall::error::{Result, Error, EAGAIN}; use super::context::{Context, ContextId}; +use crate::syscall::error::{Error, Result, EAGAIN}; /// Context list type pub struct ContextList { // Using a BTreeMap for it's range method map: BTreeMap>>, - next_id: usize + next_id: usize, } impl ContextList { @@ -19,7 +18,7 @@ impl ContextList { pub const fn new() -> Self { ContextList { map: BTreeMap::new(), - next_id: 1 + next_id: 1, } } @@ -29,12 +28,18 @@ impl ContextList { } /// Get an iterator of all parents - pub fn ancestors(&'_ self, id: ContextId) -> impl Iterator>)> + '_ { - iter::successors(self.get(id).map(|context| (id, context)), move |(_id, context)| { - let context = context.read(); - let id = context.ppid; - self.get(id).map(|context| (id, context)) - }) + pub fn ancestors( + &'_ self, + id: ContextId, + ) -> impl Iterator>)> + '_ { + iter::successors( + self.get(id).map(|context| (id, context)), + move |(_id, context)| { + let context = context.read(); + let id = context.ppid; + self.get(id).map(|context| (id, context)) + }, + ) } /// Get the current context. @@ -46,14 +51,23 @@ impl ContextList { self.map.iter() } - pub fn range(&self, range: impl core::ops::RangeBounds) -> ::alloc::collections::btree_map::Range<'_, ContextId, Arc>> { + pub fn range( + &self, + range: impl core::ops::RangeBounds, + ) -> ::alloc::collections::btree_map::Range<'_, ContextId, Arc>> { self.map.range(range) } pub(crate) fn insert_context_raw(&mut self, id: ContextId) -> Result<&Arc>> { - assert!(self.map.insert(id, Arc::new(RwLock::new(Context::new(id)?))).is_none()); + assert!(self + .map + .insert(id, Arc::new(RwLock::new(Context::new(id)?))) + .is_none()); - Ok(self.map.get(&id).expect("Failed to insert new context. ID is out of bounds.")) + Ok(self + .map + .get(&id) + .expect("Failed to insert new context. ID is out of bounds.")) } /// Create a new context. @@ -84,7 +98,7 @@ impl ContextList { } /// Spawn a context from a function. - pub fn spawn(&mut self, func: extern fn()) -> Result<&Arc>> { + pub fn spawn(&mut self, func: extern "C" fn()) -> Result<&Arc>> { let context_lock = self.new_context()?; { let mut context = context_lock.write(); diff --git a/src/context/memory.rs b/src/context/memory.rs index 77749d5690..93035af01b 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -1,26 +1,24 @@ -use alloc::collections::BTreeMap; -use alloc::{sync::Arc, vec::Vec}; +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; +use core::{cmp, fmt::Debug, num::NonZeroUsize, sync::atomic::Ordering}; use hashbrown::HashMap; -use syscall::{GrantFlags, MunmapFlags}; -use core::cmp; -use core::fmt::Debug; -use core::num::NonZeroUsize; -use core::sync::atomic::Ordering; -use spin::{RwLock, RwLockWriteGuard, RwLockUpgradableGuard}; -use syscall::{ - flag::MapFlags, - error::*, -}; use rmm::{Arch as _, PageFlush}; +use spin::{RwLock, RwLockUpgradableGuard, RwLockWriteGuard}; +use syscall::{error::*, flag::MapFlags, GrantFlags, MunmapFlags}; -use crate::arch::paging::PAGE_SIZE; -use crate::memory::{Enomem, Frame, get_page_info, PageInfo, deallocate_frames, RefKind, AddRefError, RefCount, the_zeroed_frame, init_frame}; -use crate::paging::mapper::{Flusher, InactiveFlusher, PageFlushAll}; -use crate::paging::{Page, PageFlags, PageMapper, RmmA, TableKind, VirtualAddress}; -use crate::scheme::{self, KernelSchemes}; +use crate::{ + arch::paging::PAGE_SIZE, + memory::{ + deallocate_frames, get_page_info, init_frame, the_zeroed_frame, AddRefError, Enomem, Frame, + PageInfo, RefCount, RefKind, + }, + paging::{ + mapper::{Flusher, InactiveFlusher, PageFlushAll}, + Page, PageFlags, PageMapper, RmmA, TableKind, VirtualAddress, + }, + scheme::{self, KernelSchemes}, +}; -use super::context::HardBlockedReason; -use super::file::FileDescription; +use super::{context::HardBlockedReason, file::FileDescription}; pub const MMAP_MIN_DEFAULT: usize = PAGE_SIZE; @@ -29,12 +27,16 @@ pub fn page_flags(flags: MapFlags) -> PageFlags { .user(true) .execute(flags.contains(MapFlags::PROT_EXEC)) .write(flags.contains(MapFlags::PROT_WRITE)) - //TODO: PROT_READ + //TODO: PROT_READ } pub fn map_flags(page_flags: PageFlags) -> MapFlags { let mut flags = MapFlags::PROT_READ; - if page_flags.has_write() { flags |= MapFlags::PROT_WRITE; } - if page_flags.has_execute() { flags |= MapFlags::PROT_EXEC; } + if page_flags.has_write() { + flags |= MapFlags::PROT_WRITE; + } + if page_flags.has_execute() { + flags |= MapFlags::PROT_EXEC; + } flags } @@ -45,7 +47,11 @@ pub struct UnmapResult { } impl UnmapResult { pub fn unmap(mut self) -> Result<()> { - let Some(GrantFileRef { base_offset, description }) = self.file_desc.take() else { + let Some(GrantFileRef { + base_offset, + description, + }) = self.file_desc.take() + else { return Ok(()); }; @@ -54,7 +60,9 @@ impl UnmapResult { }; let funmap_result = crate::scheme::schemes() - .get(scheme_id).cloned().ok_or(Error::new(ENODEV)) + .get(scheme_id) + .cloned() + .ok_or(Error::new(ENODEV)) .and_then(|scheme| scheme.kfunmap(number, base_offset, self.size, self.flags)); if let Ok(fd) = Arc::try_unwrap(description) { @@ -100,10 +108,20 @@ impl AddrSpace { for (grant_base, grant_info) in self.grants.iter() { let new_grant = match grant_info.provider { // No, your temporary UserScheme mappings will not be kept across forks. - Provider::External { is_pinned_userscheme_borrow: true, .. } | Provider::AllocatedShared { is_pinned_userscheme_borrow: true, .. } => continue, + Provider::External { + is_pinned_userscheme_borrow: true, + .. + } + | Provider::AllocatedShared { + is_pinned_userscheme_borrow: true, + .. + } => continue, // No, physically contiguous driver memory won't either. - Provider::Allocated { phys_contiguous: true, .. } => continue, + Provider::Allocated { + phys_contiguous: true, + .. + } => continue, Provider::PhysBorrowed { base } => Grant::physmap( base.clone(), @@ -112,7 +130,10 @@ impl AddrSpace { new_mapper, (), )?, - Provider::Allocated { ref cow_file_ref, phys_contiguous: false } => Grant::copy_mappings( + Provider::Allocated { + ref cow_file_ref, + phys_contiguous: false, + } => Grant::copy_mappings( grant_base, grant_base, grant_info.page_count, @@ -121,10 +142,14 @@ impl AddrSpace { new_mapper, &mut this_flusher, (), - CopyMappingsMode::Owned { cow_file_ref: cow_file_ref.clone() }, + CopyMappingsMode::Owned { + cow_file_ref: cow_file_ref.clone(), + }, )?, // TODO: Merge Allocated and AllocatedShared, and make CopyMappingsMode a field? - Provider::AllocatedShared { is_pinned_userscheme_borrow: false } => Grant::copy_mappings( + Provider::AllocatedShared { + is_pinned_userscheme_borrow: false, + } => Grant::copy_mappings( grant_base, grant_base, grant_info.page_count, @@ -138,7 +163,11 @@ impl AddrSpace { // MAP_SHARED grants are retained by reference, across address space clones (across // forks on monolithic kernels). - Provider::External { ref address_space, src_base, .. } => Grant::borrow_grant( + Provider::External { + ref address_space, + src_base, + .. + } => Grant::borrow_grant( Arc::clone(&address_space), src_base, grant_base, @@ -176,31 +205,48 @@ impl AddrSpace { let mapper = &mut self.table.utable; // TODO: Remove allocation (might require BTreeMap::set_key or interior mutability). - let regions = self.grants.conflicts(requested_span).map(|(base, info)| if info.is_pinned() { - Err(Error::new(EBUSY)) - } else { - Ok(PageSpan::new(base, info.page_count)) - }).collect::>(); + let regions = self + .grants + .conflicts(requested_span) + .map(|(base, info)| { + if info.is_pinned() { + Err(Error::new(EBUSY)) + } else { + Ok(PageSpan::new(base, info.page_count)) + } + }) + .collect::>(); for grant_span_res in regions { let grant_span = grant_span_res?; - let grant = self.grants.remove(grant_span.base).expect("grant cannot magically disappear while we hold the lock!"); + let grant = self + .grants + .remove(grant_span.base) + .expect("grant cannot magically disappear while we hold the lock!"); //log::info!("Mprotecting {:#?} to {:#?} in {:#?}", grant, flags, grant_span); let intersection = grant_span.intersection(requested_span); - let (before, mut grant, after) = grant.extract(intersection).expect("failed to extract grant"); + let (before, mut grant, after) = grant + .extract(intersection) + .expect("failed to extract grant"); //log::info!("Sliced into\n\n{:#?}\n\n{:#?}\n\n{:#?}", before, grant, after); - if let Some(before) = before { self.grants.insert(before); } - if let Some(after) = after { self.grants.insert(after); } + if let Some(before) = before { + self.grants.insert(before); + } + if let Some(after) = after { + self.grants.insert(after); + } if !grant.info.can_have_flags(flags) { self.grants.insert(grant); return Err(Error::new(EACCES)); } - let new_flags = grant.info.flags() + let new_flags = grant + .info + .flags() // TODO: Require a capability in order to map executable memory? .execute(flags.contains(MapFlags::PROT_EXEC)) .write(flags.contains(MapFlags::PROT_WRITE)); @@ -216,25 +262,39 @@ impl AddrSpace { Ok(()) } #[must_use = "needs to notify files"] - pub fn munmap(&mut self, mut requested_span: PageSpan, unpin: bool) -> Result> { + pub fn munmap( + &mut self, + mut requested_span: PageSpan, + unpin: bool, + ) -> Result> { let mut notify_files = Vec::new(); let mut flusher = PageFlushAll::new(); let this = &mut *self; - let next = |grants: &mut UserGrants, span: PageSpan| grants.conflicts(span).map(|(base, info)| if info.is_pinned() && !unpin { - Err(Error::new(EBUSY)) - } else if !info.can_extract(unpin) { - Err(Error::new(EINVAL)) - } else { - Ok(PageSpan::new(base, info.page_count)) - }).next(); + let next = |grants: &mut UserGrants, span: PageSpan| { + grants + .conflicts(span) + .map(|(base, info)| { + if info.is_pinned() && !unpin { + Err(Error::new(EBUSY)) + } else if !info.can_extract(unpin) { + Err(Error::new(EINVAL)) + } else { + Ok(PageSpan::new(base, info.page_count)) + } + }) + .next() + }; while let Some(conflicting_span_res) = next(&mut this.grants, requested_span) { let conflicting_span = conflicting_span_res?; - let mut grant = this.grants.remove(conflicting_span.base).expect("conflicting region didn't exist"); + let mut grant = this + .grants + .remove(conflicting_span.base) + .expect("conflicting region didn't exist"); if unpin { grant.info.unpin(); } @@ -243,10 +303,15 @@ impl AddrSpace { requested_span = { let offset = conflicting_span.base.offset_from(requested_span.base); - PageSpan::new(conflicting_span.end(), requested_span.count - offset - conflicting_span.count) + PageSpan::new( + conflicting_span.end(), + requested_span.count - offset - conflicting_span.count, + ) }; - let (before, grant, after) = grant.extract(intersection).expect("conflicting region shared no common parts"); + let (before, grant, after) = grant + .extract(intersection) + .expect("conflicting region shared no common parts"); // Keep untouched regions if let Some(before) = before { @@ -267,7 +332,17 @@ impl AddrSpace { Ok(notify_files) } - pub fn mmap_anywhere(&mut self, page_count: NonZeroUsize, flags: MapFlags, map: impl FnOnce(Page, PageFlags, &mut PageMapper, &mut dyn Flusher) -> Result) -> Result { + pub fn mmap_anywhere( + &mut self, + page_count: NonZeroUsize, + flags: MapFlags, + map: impl FnOnce( + Page, + PageFlags, + &mut PageMapper, + &mut dyn Flusher, + ) -> Result, + ) -> Result { self.mmap(None, page_count, flags, &mut Vec::new(), map) } pub fn mmap( @@ -276,7 +351,12 @@ impl AddrSpace { page_count: NonZeroUsize, flags: MapFlags, notify_files_out: &mut Vec, - map: impl FnOnce(Page, PageFlags, &mut PageMapper, &mut dyn Flusher) -> Result, + map: impl FnOnce( + Page, + PageFlags, + &mut PageMapper, + &mut dyn Flusher, + ) -> Result, ) -> Result { let selected_span = match requested_base_opt { // TODO: Rename MAP_FIXED+MAP_FIXED_NOREPLACE to MAP_FIXED and @@ -296,10 +376,15 @@ impl AddrSpace { requested_span } else { - self.grants.find_free_near(self.mmap_min, page_count.get(), Some(requested_base)).ok_or(Error::new(ENOMEM))? + self.grants + .find_free_near(self.mmap_min, page_count.get(), Some(requested_base)) + .ok_or(Error::new(ENOMEM))? } } - None => self.grants.find_free(self.mmap_min, page_count.get()).ok_or(Error::new(ENOMEM))?, + None => self + .grants + .find_free(self.mmap_min, page_count.get()) + .ok_or(Error::new(ENOMEM))?, }; // TODO: Threads share address spaces, so not only the inactive flusher should be sending @@ -315,19 +400,37 @@ impl AddrSpace { &mut inactive as &mut dyn Flusher }; - let grant = map(selected_span.base, page_flags(flags), &mut self.table.utable, flusher)?; + let grant = map( + selected_span.base, + page_flags(flags), + &mut self.table.utable, + flusher, + )?; self.grants.insert(grant); Ok(selected_span.base) } - pub fn r#move(dst: &mut AddrSpace, mut src_opt: Option<&mut AddrSpace>, src_span: PageSpan, requested_dst_base: Option, new_page_count: usize, new_flags: MapFlags, notify_files: &mut Vec) -> Result { + pub fn r#move( + dst: &mut AddrSpace, + mut src_opt: Option<&mut AddrSpace>, + src_span: PageSpan, + requested_dst_base: Option, + new_page_count: usize, + new_flags: MapFlags, + notify_files: &mut Vec, + ) -> Result { // TODO let mut src_flusher = PageFlushAll::new(); let mut dst_flusher = PageFlushAll::new(); let dst_base = match requested_dst_base { Some(base) if new_flags.contains(MapFlags::MAP_FIXED_NOREPLACE) => { - if dst.grants.conflicts(PageSpan::new(base, new_page_count)).next().is_some() { + if dst + .grants + .conflicts(PageSpan::new(base, new_page_count)) + .next() + .is_some() + { return Err(Error::new(EEXIST)); } @@ -339,15 +442,28 @@ impl AddrSpace { base } - _ => dst.grants.find_free(dst.mmap_min, core::cmp::max(new_page_count, src_span.count)).ok_or(Error::new(ENOMEM))?.base, + _ => { + dst.grants + .find_free(dst.mmap_min, core::cmp::max(new_page_count, src_span.count)) + .ok_or(Error::new(ENOMEM))? + .base + } }; let src = src_opt.as_deref_mut().unwrap_or(&mut *dst); - if src.grants.conflicts(src_span).any(|(_, g)| !g.can_extract(false)) { + if src + .grants + .conflicts(src_span) + .any(|(_, g)| !g.can_extract(false)) + { return Err(Error::new(EBUSY)); } - if src.grants.conflicts(src_span).any(|(_, g)| !g.can_have_flags(new_flags)) { + if src + .grants + .conflicts(src_span) + .any(|(_, g)| !g.can_have_flags(new_flags)) + { return Err(Error::new(EPERM)); } if PageSpan::new(dst_base, new_page_count).intersects(src_span) { @@ -356,13 +472,23 @@ impl AddrSpace { if new_page_count < src_span.count { let unpin = false; - notify_files.append(&mut src.munmap(PageSpan::new(src_span.base.next_by(new_page_count), src_span.count - new_page_count), unpin)?); + notify_files.append(&mut src.munmap( + PageSpan::new( + src_span.base.next_by(new_page_count), + src_span.count - new_page_count, + ), + unpin, + )?); } let mut remaining_src_span = PageSpan::new(src_span.base, new_page_count); //let next = |mut src_opt: Option<&mut AddrSpace>, dst: &mut AddrSpace, rem| { let opt = src_opt.as_deref_mut().unwrap_or(dst).grants.conflicts(rem).next(); opt.map(|(b, _)| b) }; - let to_remap = src.grants.conflicts(remaining_src_span).map(|(b, _)| b).collect::>(); + let to_remap = src + .grants + .conflicts(remaining_src_span) + .map(|(b, _)| b) + .collect::>(); let mut prev_grant_end = src_span.base; @@ -370,38 +496,76 @@ impl AddrSpace { for grant_base in to_remap { if prev_grant_end < grant_base { let hole_page_count = grant_base.offset_from(prev_grant_end); - let hole_span = PageSpan::new(dst_base.next_by(prev_grant_end.offset_from(src_span.base)), hole_page_count); - dst.grants.insert(Grant::zeroed(hole_span, page_flags(new_flags), &mut dst.table.utable, &mut dst_flusher, false)?); + let hole_span = PageSpan::new( + dst_base.next_by(prev_grant_end.offset_from(src_span.base)), + hole_page_count, + ); + dst.grants.insert(Grant::zeroed( + hole_span, + page_flags(new_flags), + &mut dst.table.utable, + &mut dst_flusher, + false, + )?); } let src = src_opt.as_deref_mut().unwrap_or(&mut *dst); - let grant = src.grants.remove(grant_base).expect("grant cannot disappear"); + let grant = src + .grants + .remove(grant_base) + .expect("grant cannot disappear"); let grant_span = PageSpan::new(grant.base, grant.info.page_count()); - let (before, middle, after) = grant.extract(remaining_src_span.intersection(grant_span)).expect("called intersect(), must succeed"); + let (before, middle, after) = grant + .extract(remaining_src_span.intersection(grant_span)) + .expect("called intersect(), must succeed"); - if let Some(before) = before { src.grants.insert(before); } - if let Some(after) = after { src.grants.insert(after); } + if let Some(before) = before { + src.grants.insert(before); + } + if let Some(after) = after { + src.grants.insert(after); + } let dst_grant_base = dst_base.next_by(middle.base.offset_from(src_span.base)); let middle_span = middle.span(); dst.grants.insert(match src_opt { - Some(ref mut different_src) => { - middle.transfer(dst_grant_base, page_flags(new_flags), &mut different_src.table.utable, Some(&mut dst.table.utable), &mut src_flusher, &mut dst_flusher)? - } - None => { - middle.transfer(dst_grant_base, page_flags(new_flags), &mut dst.table.utable, None, &mut src_flusher, ())? - } + Some(ref mut different_src) => middle.transfer( + dst_grant_base, + page_flags(new_flags), + &mut different_src.table.utable, + Some(&mut dst.table.utable), + &mut src_flusher, + &mut dst_flusher, + )?, + None => middle.transfer( + dst_grant_base, + page_flags(new_flags), + &mut dst.table.utable, + None, + &mut src_flusher, + (), + )?, }); prev_grant_end = middle_span.base.next_by(middle_span.count); let pages_advanced = prev_grant_end.offset_from(remaining_src_span.base); - remaining_src_span = PageSpan::new(prev_grant_end, remaining_src_span.count - pages_advanced); - }; + remaining_src_span = + PageSpan::new(prev_grant_end, remaining_src_span.count - pages_advanced); + } if prev_grant_end < src_span.base.next_by(new_page_count) { - let last_hole_span = PageSpan::new(dst_base.next_by(prev_grant_end.offset_from(src_span.base)), new_page_count - prev_grant_end.offset_from(src_span.base)); - dst.grants.insert(Grant::zeroed(last_hole_span, page_flags(new_flags), &mut dst.table.utable, &mut dst_flusher, false)?); + let last_hole_span = PageSpan::new( + dst_base.next_by(prev_grant_end.offset_from(src_span.base)), + new_page_count - prev_grant_end.offset_from(src_span.base), + ); + dst.grants.insert(Grant::zeroed( + last_hole_span, + page_flags(new_flags), + &mut dst.table.utable, + &mut dst_flusher, + false, + )?); } Ok(dst_base) @@ -434,10 +598,17 @@ impl PageSpan { Self::validate(address, size).filter(|this| !this.is_empty()) } pub fn validate(address: VirtualAddress, size: usize) -> Option { - if address.data() % PAGE_SIZE != 0 || size % PAGE_SIZE != 0 { return None; } - if address.data().saturating_add(size) > crate::USER_END_OFFSET { return None; } + if address.data() % PAGE_SIZE != 0 || size % PAGE_SIZE != 0 { + return None; + } + if address.data().saturating_add(size) > crate::USER_END_OFFSET { + return None; + } - Some(Self::new(Page::containing_address(address), size / PAGE_SIZE)) + Some(Self::new( + Page::containing_address(address), + size / PAGE_SIZE, + )) } pub fn is_empty(&self) -> bool { self.count == 0 @@ -465,25 +636,22 @@ impl PageSpan { /// Returns the span from the start of self until the start of the specified span. pub fn before(self, span: Self) -> Option { assert!(self.base <= span.base); - Some(Self::between( - self.base, - span.base, - )).filter(|reg| !reg.is_empty()) + Some(Self::between(self.base, span.base)).filter(|reg| !reg.is_empty()) } /// Returns the span from the end of the given span until the end of self. pub fn after(self, span: Self) -> Option { assert!(span.end() <= self.end()); - Some(Self::between( - span.end(), - self.end(), - )).filter(|reg| !reg.is_empty()) + Some(Self::between(span.end(), self.end())).filter(|reg| !reg.is_empty()) } /// Returns the span between two pages, `[start, end)`, truncating to zero if end < start. pub fn between(start: Page, end: Page) -> Self { Self::new( start, - end.start_address().data().saturating_sub(start.start_address().data()) / PAGE_SIZE, + end.start_address() + .data() + .saturating_sub(start.start_address().data()) + / PAGE_SIZE, ) } } @@ -495,7 +663,16 @@ impl Default for UserGrants { } impl Debug for PageSpan { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "[{:p}:{:p}, {} pages]", self.base.start_address().data() as *const u8, self.base.start_address().add(self.count * PAGE_SIZE - 1).data() as *const u8, self.count) + write!( + f, + "[{:p}:{:p}, {} pages]", + self.base.start_address().data() as *const u8, + self.base + .start_address() + .add(self.count * PAGE_SIZE - 1) + .data() as *const u8, + self.count + ) } } @@ -503,7 +680,8 @@ impl UserGrants { pub fn new() -> Self { Self { inner: BTreeMap::new(), - holes: core::iter::once((VirtualAddress::new(0), crate::USER_END_OFFSET)).collect::>(), + holes: core::iter::once((VirtualAddress::new(0), crate::USER_END_OFFSET)) + .collect::>(), funmap: HashMap::new(), } } @@ -522,24 +700,29 @@ impl UserGrants { // If there is a grant that contains the base page, start searching at the base of that // grant, rather than the requested base here. - let start_span = start.map(|(base, info)| PageSpan::new(base, info.page_count)).unwrap_or(span); + let start_span = start + .map(|(base, info)| PageSpan::new(base, info.page_count)) + .unwrap_or(span); - self - .inner + self.inner .range(start_span.base..) .take_while(move |(base, info)| PageSpan::new(**base, info.page_count).intersects(span)) .map(|(base, info)| (*base, info)) } // TODO: DEDUPLICATE CODE! - pub fn conflicts_mut(&mut self, span: PageSpan) -> impl Iterator + '_ { + pub fn conflicts_mut( + &mut self, + span: PageSpan, + ) -> impl Iterator + '_ { let start = self.contains(span.base); // If there is a grant that contains the base page, start searching at the base of that // grant, rather than the requested base here. - let start_span = start.map(|(base, info)| PageSpan::new(base, info.page_count)).unwrap_or(span); + let start_span = start + .map(|(base, info)| PageSpan::new(base, info.page_count)) + .unwrap_or(span); - self - .inner + self.inner .range_mut(start_span.base..) .take_while(move |(base, info)| PageSpan::new(**base, info.page_count).intersects(span)) .map(|(base, info)| (*base, info)) @@ -547,7 +730,12 @@ impl UserGrants { /// Return a free region with the specified size // TODO: Alignment (x86_64: 4 KiB, 2 MiB, or 1 GiB). // TODO: Support finding grant close to a requested address? - pub fn find_free_near(&self, min: usize, page_count: usize, _near: Option) -> Option { + pub fn find_free_near( + &self, + min: usize, + page_count: usize, + _near: Option, + ) -> Option { // Get first available hole, but do reserve the page starting from zero as most compiled // languages cannot handle null pointers safely even if they point to valid memory. If an // application absolutely needs to map the 0th page, they will have to do so explicitly via @@ -555,18 +743,24 @@ impl UserGrants { // TODO: Allow explicitly allocating guard pages? Perhaps using mprotect or mmap with // PROT_NONE? - let (hole_start, _hole_size) = self.holes.iter() + let (hole_start, _hole_size) = self + .holes + .iter() .skip_while(|(hole_offset, hole_size)| hole_offset.data() + **hole_size <= min) .find(|(hole_offset, hole_size)| { - let avail_size = if hole_offset.data() <= min && min <= hole_offset.data() + **hole_size { - **hole_size - (min - hole_offset.data()) - } else { - **hole_size - }; + let avail_size = + if hole_offset.data() <= min && min <= hole_offset.data() + **hole_size { + **hole_size - (min - hole_offset.data()) + } else { + **hole_size + }; page_count * PAGE_SIZE <= avail_size })?; // Create new region - Some(PageSpan::new(Page::containing_address(VirtualAddress::new(cmp::max(hole_start.data(), min))), page_count)) + Some(PageSpan::new( + Page::containing_address(VirtualAddress::new(cmp::max(hole_start.data(), min))), + page_count, + )) } pub fn find_free(&self, min: usize, page_count: usize) -> Option { self.find_free_near(min, page_count, None) @@ -593,7 +787,8 @@ impl UserGrants { } if prev_hole_end > end_address.data() { // The grant is splitting this hole in two, so insert the new one at the end. - self.holes.insert(end_address, prev_hole_end - end_address.data()); + self.holes + .insert(end_address, prev_hole_end - end_address.data()); } } @@ -617,7 +812,11 @@ impl UserGrants { // There was a range that began exactly prior to the to-be-freed region, so simply // increment the size such that it occupies the grant too. If in addition there was a grant // directly after the grant, include it too in the size. - if let Some((hole_offset, hole_size)) = holes.range_mut(..start_address).next_back().filter(|(offset, size)| offset.data() + **size == start_address.data()) { + if let Some((hole_offset, hole_size)) = holes + .range_mut(..start_address) + .next_back() + .filter(|(offset, size)| offset.data() + **size == start_address.data()) + { *hole_size = end_address.data() - hole_offset.data() + exactly_after_size.unwrap_or(0); } else { // There was no free region directly before the to-be-freed region, however will @@ -627,16 +826,31 @@ impl UserGrants { } } pub fn insert(&mut self, mut grant: Grant) { - assert!(self.conflicts(PageSpan::new(grant.base, grant.info.page_count)).next().is_none()); + assert!(self + .conflicts(PageSpan::new(grant.base, grant.info.page_count)) + .next() + .is_none()); self.reserve(grant.base, grant.info.page_count); - let before_region = self.inner - .range(..grant.base).next_back() - .filter(|(base, info)| base.next_by(info.page_count) == grant.base && info.can_be_merged_if_adjacent(&grant.info)).map(|(base, info)| (*base, info.page_count)); + let before_region = self + .inner + .range(..grant.base) + .next_back() + .filter(|(base, info)| { + base.next_by(info.page_count) == grant.base + && info.can_be_merged_if_adjacent(&grant.info) + }) + .map(|(base, info)| (*base, info.page_count)); - let after_region = self.inner - .range(grant.span().end()..).next() - .filter(|(base, info)| **base == grant.base.next_by(grant.info.page_count) && info.can_be_merged_if_adjacent(&grant.info)).map(|(base, info)| (*base, info.page_count)); + let after_region = self + .inner + .range(grant.span().end()..) + .next() + .filter(|(base, info)| { + **base == grant.base.next_by(grant.info.page_count) + && info.can_be_merged_if_adjacent(&grant.info) + }) + .map(|(base, info)| (*base, info.page_count)); if let Some((before_base, before_page_count)) = before_region { grant.base = before_base; @@ -660,9 +874,13 @@ impl UserGrants { pub fn iter(&self) -> impl Iterator + '_ { self.inner.iter().map(|(base, info)| (*base, info)) } - pub fn is_empty(&self) -> bool { self.inner.is_empty() } + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } pub fn into_iter(self) -> impl Iterator { - self.inner.into_iter().map(|(base, info)| Grant { base, info }) + self.inner + .into_iter() + .map(|(base, info)| Grant { base, info }) } } @@ -683,7 +901,10 @@ pub enum Provider { /// The pages this grant spans, need not necessarily be initialized right away, and can be /// populated either from zeroed frames, the CoW zeroed frame, or from a scheme fmap call, if /// mapped with MAP_LAZY. All frames must have an available PageInfo. - Allocated { cow_file_ref: Option, phys_contiguous: bool }, + Allocated { + cow_file_ref: Option, + phys_contiguous: bool, + }, /// The grant is owned, but possibly shared. /// @@ -699,14 +920,21 @@ pub enum Provider { PhysBorrowed { base: Frame }, /// The memory is borrowed directly from another address space. - External { address_space: Arc>, src_base: Page, is_pinned_userscheme_borrow: bool }, + External { + address_space: Arc>, + src_base: Page, + is_pinned_userscheme_borrow: bool, + }, /// The memory is MAP_SHARED borrowed from a scheme. /// /// Since the address space is not tracked here, all nonpresent pages must be present before /// the fmap operation completes, unless MAP_LAZY is specified. They are tracked using /// PageInfo, or treated as PhysBorrowed if any frame lacks a PageInfo. - FmapBorrowed { file_ref: GrantFileRef, pin_refcount: usize }, + FmapBorrowed { + file_ref: GrantFileRef, + pin_refcount: usize, + }, } #[derive(Debug)] @@ -725,7 +953,14 @@ impl Grant { // TODO: PageCount newtype, to avoid confusion between bytes and pages? // TODO: is_pinned - pub fn allocated_shared_one_page(frame: Frame, page: Page, flags: PageFlags, mapper: &mut PageMapper, mut flusher: impl Flusher, is_pinned: bool) -> Result { + pub fn allocated_shared_one_page( + frame: Frame, + page: Page, + flags: PageFlags, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + is_pinned: bool, + ) -> Result { let info = get_page_info(frame).expect("needs page info"); // TODO: @@ -738,10 +973,15 @@ impl Grant { // Semantically, the page will be shared between the "context struct" and whatever // else. - info.add_ref(RefKind::Shared).expect("must be possible if previously Zero"); + info.add_ref(RefKind::Shared) + .expect("must be possible if previously Zero"); unsafe { - flusher.consume(mapper.map_phys(page.start_address(), frame.start_address(), flags).ok_or(Error::new(ENOMEM))?); + flusher.consume( + mapper + .map_phys(page.start_address(), frame.start_address(), flags) + .ok_or(Error::new(ENOMEM))?, + ); } Ok(Grant { @@ -750,12 +990,20 @@ impl Grant { page_count: 1, flags, mapped: true, - provider: Provider::AllocatedShared { is_pinned_userscheme_borrow: is_pinned }, - } + provider: Provider::AllocatedShared { + is_pinned_userscheme_borrow: is_pinned, + }, + }, }) } - pub fn physmap(phys: Frame, span: PageSpan, flags: PageFlags, mapper: &mut PageMapper, mut flusher: impl Flusher) -> Result { + pub fn physmap( + phys: Frame, + span: PageSpan, + flags: PageFlags, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + ) -> Result { const MAX_EAGER_PAGES: usize = 4096; for i in 0..span.count { @@ -767,7 +1015,11 @@ impl Grant { for (i, page) in span.pages().enumerate().take(MAX_EAGER_PAGES) { unsafe { - let Some(result) = mapper.map_phys(page.start_address(), phys.next_by(i).start_address(), flags.write(false)) else { + let Some(result) = mapper.map_phys( + page.start_address(), + phys.next_by(i).start_address(), + flags.write(false), + ) else { break; }; flusher.consume(result); @@ -784,16 +1036,26 @@ impl Grant { }, }) } - pub fn zeroed_phys_contiguous(span: PageSpan, flags: PageFlags, mapper: &mut PageMapper, mut flusher: impl Flusher) -> Result { + pub fn zeroed_phys_contiguous( + span: PageSpan, + flags: PageFlags, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + ) -> Result { let base = crate::memory::allocate_frames(span.count).ok_or(Enomem)?; for (i, page) in span.pages().enumerate() { let frame = base.next_by(i); - get_page_info(frame).expect("PageInfo must exist for allocated frame").refcount.store(RefCount::One.to_raw(), Ordering::Relaxed); + get_page_info(frame) + .expect("PageInfo must exist for allocated frame") + .refcount + .store(RefCount::One.to_raw(), Ordering::Relaxed); unsafe { - let result = mapper.map_phys(page.start_address(), frame.start_address(), flags).expect("TODO: page table OOM"); + let result = mapper + .map_phys(page.start_address(), frame.start_address(), flags) + .expect("TODO: page table OOM"); flusher.consume(result); } } @@ -804,11 +1066,20 @@ impl Grant { page_count: span.count, flags, mapped: true, - provider: Provider::Allocated { cow_file_ref: None, phys_contiguous: true }, + provider: Provider::Allocated { + cow_file_ref: None, + phys_contiguous: true, + }, }, }) } - pub fn zeroed(span: PageSpan, flags: PageFlags, mapper: &mut PageMapper, mut flusher: impl Flusher, shared: bool) -> Result { + pub fn zeroed( + span: PageSpan, + flags: PageFlags, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + shared: bool, + ) -> Result { const MAX_EAGER_PAGES: usize = 16; let (the_frame, the_frame_info) = the_zeroed_frame(); @@ -819,9 +1090,15 @@ impl Grant { // Good thing with lazy page fault handlers, is that if we fail due to ENOMEM here, we // can continue and let the process face the OOM killer later. unsafe { - the_frame_info.add_ref(RefKind::Cow).expect("the static zeroed frame cannot be shared!"); + the_frame_info + .add_ref(RefKind::Cow) + .expect("the static zeroed frame cannot be shared!"); - let Some(result) = mapper.map_phys(page.start_address(), the_frame.start_address(), flags.write(false)) else { + let Some(result) = mapper.map_phys( + page.start_address(), + the_frame.start_address(), + flags.write(false), + ) else { break; }; flusher.consume(result); @@ -835,9 +1112,14 @@ impl Grant { flags, mapped: true, provider: if shared { - Provider::AllocatedShared { is_pinned_userscheme_borrow: false } + Provider::AllocatedShared { + is_pinned_userscheme_borrow: false, + } } else { - Provider::Allocated { cow_file_ref: None, phys_contiguous: false } + Provider::Allocated { + cow_file_ref: None, + phys_contiguous: false, + } }, }, }) @@ -845,7 +1127,15 @@ impl Grant { // XXX: borrow_grant is needed because of the borrow checker (iterator invalidation), maybe // borrow_grant/borrow can be abstracted somehow? - pub fn borrow_grant(src_address_space_lock: Arc>, src_base: Page, dst_base: Page, src_info: &GrantInfo, _mapper: &mut PageMapper, _dst_flusher: impl Flusher, _eager: bool) -> Result { + pub fn borrow_grant( + src_address_space_lock: Arc>, + src_base: Page, + dst_base: Page, + src_info: &GrantInfo, + _mapper: &mut PageMapper, + _dst_flusher: impl Flusher, + _eager: bool, + ) -> Result { Ok(Grant { base: dst_base, info: GrantInfo { @@ -856,12 +1146,19 @@ impl Grant { src_base, address_space: src_address_space_lock, is_pinned_userscheme_borrow: false, - } + }, }, }) } - pub fn borrow_fmap(span: PageSpan, new_flags: PageFlags, file_ref: GrantFileRef, src: Option>, mapper: &mut PageMapper, mut flusher: impl Flusher) -> Result { + pub fn borrow_fmap( + span: PageSpan, + new_flags: PageFlags, + file_ref: GrantFileRef, + src: Option>, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + ) -> Result { if let Some(src) = src { let mut guard = src.addr_space_guard; for dst_page in span.pages() { @@ -874,7 +1171,14 @@ impl Grant { Some((phys, _)) => Frame::containing_address(phys), // TODO: ensure the correct context is hardblocked, if necessary None => { - let (frame, _, new_guard) = correct_inner(src.addr_space_lock, guard, src_page, AccessMode::Read, 0).map_err(|_| Error::new(EIO))?; + let (frame, _, new_guard) = correct_inner( + src.addr_space_lock, + guard, + src_page, + AccessMode::Read, + 0, + ) + .map_err(|_| Error::new(EIO))?; guard = new_guard; frame } @@ -904,18 +1208,35 @@ impl Grant { match page_info.add_ref(RefKind::Shared) { Ok(()) => frame, Err(AddRefError::CowToShared) => unsafe { - let new_cow_frame = cow(frame, page_info, RefKind::Shared).map_err(|_| Error::new(ENOMEM))?; - let (_, _, flush) = guard.table.utable.remap_with_full(src_page.start_address(), |_, flags| (new_cow_frame.start_address(), flags)).expect("page did exist"); + let new_cow_frame = cow(frame, page_info, RefKind::Shared) + .map_err(|_| Error::new(ENOMEM))?; + let (_, _, flush) = guard + .table + .utable + .remap_with_full(src_page.start_address(), |_, flags| { + (new_cow_frame.start_address(), flags) + }) + .expect("page did exist"); src.flusher.consume(flush); new_cow_frame - } + }, Err(AddRefError::SharedToCow) => unreachable!(), } - } else { frame }; + } else { + frame + }; unsafe { - flusher.consume(mapper.map_phys(dst_page.start_address(), frame.start_address(), new_flags.write(new_flags.has_write() && !is_cow)).unwrap()); + flusher.consume( + mapper + .map_phys( + dst_page.start_address(), + frame.start_address(), + new_flags.write(new_flags.has_write() && !is_cow), + ) + .unwrap(), + ); } } } @@ -926,8 +1247,11 @@ impl Grant { page_count: span.count, mapped: true, flags: new_flags, - provider: Provider::FmapBorrowed { file_ref, pin_refcount: 0 }, - } + provider: Provider::FmapBorrowed { + file_ref, + pin_refcount: 0, + }, + }, }) } @@ -971,7 +1295,11 @@ impl Grant { return Err(Error::new(EPERM)); } - if let Provider::FmapBorrowed { ref mut pin_refcount, .. } = src_grant.provider { + if let Provider::FmapBorrowed { + ref mut pin_refcount, + .. + } = src_grant.provider + { *pin_refcount += 1; } } @@ -986,8 +1314,16 @@ impl Grant { return Err(Error::new(EINVAL)); } if eager { - for (i, page) in PageSpan::new(src_base, page_count).pages().enumerate().take(MAX_EAGER_PAGES) { - let Some((phys, _)) = src_address_space.table.utable.translate(page.start_address()) else { + for (i, page) in PageSpan::new(src_base, page_count) + .pages() + .enumerate() + .take(MAX_EAGER_PAGES) + { + let Some((phys, _)) = src_address_space + .table + .utable + .translate(page.start_address()) + else { continue; }; @@ -1003,20 +1339,29 @@ impl Grant { }; unsafe { - let flush = dst_mapper.map_phys(dst_base.next_by(i).start_address(), phys, flags.write(flags.has_write() && writable)).ok_or(Error::new(ENOMEM))?; + let flush = dst_mapper + .map_phys( + dst_base.next_by(i).start_address(), + phys, + flags.write(flags.has_write() && writable), + ) + .ok_or(Error::new(ENOMEM))?; dst_flusher.consume(flush); } } } - Ok(Grant { base: dst_base, info: GrantInfo { page_count, flags, mapped: true, - provider: Provider::External { address_space: src_address_space_lock, src_base, is_pinned_userscheme_borrow } + provider: Provider::External { + address_space: src_address_space_lock, + src_base, + is_pinned_userscheme_borrow, + }, }, }) } @@ -1043,7 +1388,9 @@ impl Grant { let src_frame = match rk { RefKind::Cow => { - let Some((_, phys, flush)) = (unsafe { src_mapper.remap_with(src_page.start_address(), |flags| flags.write(false)) }) else { + let Some((_, phys, flush)) = (unsafe { + src_mapper.remap_with(src_page.start_address(), |flags| flags.write(false)) + }) else { // Page is not mapped, let the page fault handler take care of that (initializing // it to zero). // @@ -1061,7 +1408,15 @@ impl Grant { } else { // TODO: Omit the unnecessary subsequent add_ref call. let new_frame = init_frame(RefCount::One).expect("TODO: handle OOM"); - let src_flush = unsafe { src_mapper.map_phys(src_page.start_address(), new_frame.start_address(), flags).expect("TODO: handle OOM") }; + let src_flush = unsafe { + src_mapper + .map_phys( + src_page.start_address(), + new_frame.start_address(), + flags, + ) + .expect("TODO: handle OOM") + }; src_flusher.consume(src_flush); new_frame @@ -1070,7 +1425,8 @@ impl Grant { }; let src_frame = { - let src_page_info = get_page_info(src_frame).expect("allocated page was not present in the global page array"); + let src_page_info = get_page_info(src_frame) + .expect("allocated page was not present in the global page array"); match src_page_info.add_ref(rk) { Ok(()) => src_frame, @@ -1079,20 +1435,28 @@ impl Grant { // TODO: Flusher unsafe { - src_mapper.remap_with_full(src_page.start_address(), |_, f| (new_frame.start_address(), f)); + src_mapper.remap_with_full(src_page.start_address(), |_, f| { + (new_frame.start_address(), f) + }); } new_frame - }, + } // Cannot be shared and CoW simultaneously. Err(AddRefError::SharedToCow) => { // TODO: Copy in place, or use a zeroed page? cow(src_frame, src_page_info, rk).map_err(|_| Enomem)? - }, + } } }; - let Some(map_result) = (unsafe { dst_mapper.map_phys(dst_page, src_frame.start_address(), flags.write(flags.has_write() && allows_writable)) }) else { + let Some(map_result) = (unsafe { + dst_mapper.map_phys( + dst_page, + src_frame.start_address(), + flags.write(flags.has_write() && allows_writable), + ) + }) else { break; }; @@ -1106,14 +1470,27 @@ impl Grant { flags, mapped: true, provider: match mode { - CopyMappingsMode::Owned { cow_file_ref } => Provider::Allocated { cow_file_ref, phys_contiguous: false }, - CopyMappingsMode::Borrowed => Provider::AllocatedShared { is_pinned_userscheme_borrow: false }, + CopyMappingsMode::Owned { cow_file_ref } => Provider::Allocated { + cow_file_ref, + phys_contiguous: false, + }, + CopyMappingsMode::Borrowed => Provider::AllocatedShared { + is_pinned_userscheme_borrow: false, + }, }, - } + }, }) } /// Move a grant between two address spaces. - pub fn transfer(mut self, dst_base: Page, flags: PageFlags, src_mapper: &mut PageMapper, mut dst_mapper: Option<&mut PageMapper>, mut src_flusher: impl Flusher, mut dst_flusher: impl Flusher) -> Result { + pub fn transfer( + mut self, + dst_base: Page, + flags: PageFlags, + src_mapper: &mut PageMapper, + mut dst_mapper: Option<&mut PageMapper>, + mut src_flusher: impl Flusher, + mut dst_flusher: impl Flusher, + ) -> Result { assert!(!self.info.is_pinned()); for src_page in self.span().pages() { @@ -1122,7 +1499,9 @@ impl Grant { let unmap_parents = true; // TODO: Validate flags? - let Some((phys, _flags, flush)) = (unsafe { src_mapper.unmap_phys(src_page.start_address(), unmap_parents) }) else { + let Some((phys, _flags, flush)) = + (unsafe { src_mapper.unmap_phys(src_page.start_address(), unmap_parents) }) + else { continue; }; src_flusher.consume(flush); @@ -1130,7 +1509,11 @@ impl Grant { let dst_mapper = dst_mapper.as_deref_mut().unwrap_or(&mut *src_mapper); // TODO: Preallocate to handle OOM? - let flush = unsafe { dst_mapper.map_phys(dst_page.start_address(), phys, flags).expect("TODO: OOM") }; + let flush = unsafe { + dst_mapper + .map_phys(dst_page.start_address(), phys, flags) + .expect("TODO: OOM") + }; dst_flusher.consume(flush); } @@ -1138,7 +1521,12 @@ impl Grant { Ok(self) } - pub fn remap(&mut self, mapper: &mut PageMapper, mut flusher: impl Flusher, flags: PageFlags) { + pub fn remap( + &mut self, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + flags: PageFlags, + ) { assert!(self.info.mapped); for page in self.span().pages() { @@ -1155,22 +1543,47 @@ impl Grant { self.info.flags = flags; } #[must_use = "will not unmap itself"] - pub fn unmap(mut self, mapper: &mut PageMapper, mut flusher: impl Flusher) -> UnmapResult { + pub fn unmap( + mut self, + mapper: &mut PageMapper, + mut flusher: impl Flusher, + ) -> UnmapResult { assert!(self.info.mapped); assert!(!self.info.is_pinned()); - if let Provider::External { ref address_space, src_base, .. } = self.info.provider { + if let Provider::External { + ref address_space, + src_base, + .. + } = self.info.provider + { let mut guard = address_space.write(); - for (_, grant) in guard.grants.conflicts_mut(PageSpan::new(src_base, self.info.page_count)) { + for (_, grant) in guard + .grants + .conflicts_mut(PageSpan::new(src_base, self.info.page_count)) + { match grant.provider { - Provider::FmapBorrowed { ref mut pin_refcount, .. } => *pin_refcount = pin_refcount.checked_sub(1).expect("fmap pinning code is wrong"), + Provider::FmapBorrowed { + ref mut pin_refcount, + .. + } => { + *pin_refcount = pin_refcount + .checked_sub(1) + .expect("fmap pinning code is wrong") + } _ => continue, } } } - let is_phys_contiguous = matches!(self.info.provider, Provider::Allocated { phys_contiguous: true, .. }); + let is_phys_contiguous = matches!( + self.info.provider, + Provider::Allocated { + phys_contiguous: true, + .. + } + ); let (require_info, is_fmap_shared) = match self.info.provider { Provider::Allocated { .. } => (Some(true), Some(false)), @@ -1185,27 +1598,42 @@ impl Grant { let base = Frame::containing_address(phys); for i in 0..self.info.page_count { - assert_eq!(get_page_info(base.next_by(i)).unwrap().refcount.swap(0, Ordering::Relaxed), RefCount::One.to_raw()); + assert_eq!( + get_page_info(base.next_by(i)) + .unwrap() + .refcount + .swap(0, Ordering::Relaxed), + RefCount::One.to_raw() + ); } deallocate_frames(Frame::containing_address(phys), self.info.page_count); } else { for page in self.span().pages() { // Lazy mappings do not need to be unmapped. - let Some((phys, _, flush)) = (unsafe { mapper.unmap_phys(page.start_address(), true) }) else { + let Some((phys, _, flush)) = + (unsafe { mapper.unmap_phys(page.start_address(), true) }) + else { continue; }; let frame = Frame::containing_address(phys); if let Some(info) = get_page_info(frame) { - assert_ne!(require_info, Some(false), "PhysBorrowed frame was allocator-owned"); + assert_ne!( + require_info, + Some(false), + "PhysBorrowed frame was allocator-owned" + ); if info.remove_ref() == RefCount::Zero { deallocate_frames(frame, 1); }; } else { - assert_ne!(require_info, Some(true), "allocated frame did not have an associated PageInfo"); + assert_ne!( + require_info, + Some(true), + "allocated frame did not have an associated PageInfo" + ); } - flusher.consume(flush); } } @@ -1213,10 +1641,18 @@ impl Grant { self.info.mapped = false; // Dummy value, won't be read. - let provider = core::mem::replace(&mut self.info.provider, Provider::AllocatedShared { is_pinned_userscheme_borrow: false }); + let provider = core::mem::replace( + &mut self.info.provider, + Provider::AllocatedShared { + is_pinned_userscheme_borrow: false, + }, + ); let mut munmap_flags = MunmapFlags::empty(); - munmap_flags.set(MunmapFlags::NEEDS_SYNC, is_fmap_shared.unwrap_or(false) && self.info.flags.has_write()); + munmap_flags.set( + MunmapFlags::NEEDS_SYNC, + is_fmap_shared.unwrap_or(false) && self.info.flags.has_write(), + ); UnmapResult { size: self.info.page_count * PAGE_SIZE, @@ -1257,16 +1693,32 @@ impl Grant { mapped: self.info.mapped, page_count: span.count, provider: match self.info.provider { - Provider::External { ref address_space, src_base, .. } => Provider::External { + Provider::External { + ref address_space, + src_base, + .. + } => Provider::External { address_space: Arc::clone(address_space), src_base, is_pinned_userscheme_borrow: false, }, - Provider::Allocated { ref cow_file_ref, .. } => Provider::Allocated { cow_file_ref: cow_file_ref.clone(), phys_contiguous: false }, - Provider::AllocatedShared { .. } => Provider::AllocatedShared { is_pinned_userscheme_borrow: false }, - Provider::PhysBorrowed { base } => Provider::PhysBorrowed { base: base.clone() }, - Provider::FmapBorrowed { ref file_ref, .. } => Provider::FmapBorrowed { file_ref: file_ref.clone(), pin_refcount: 0 }, - } + Provider::Allocated { + ref cow_file_ref, .. + } => Provider::Allocated { + cow_file_ref: cow_file_ref.clone(), + phys_contiguous: false, + }, + Provider::AllocatedShared { .. } => Provider::AllocatedShared { + is_pinned_userscheme_borrow: false, + }, + Provider::PhysBorrowed { base } => { + Provider::PhysBorrowed { base: base.clone() } + } + Provider::FmapBorrowed { ref file_ref, .. } => Provider::FmapBorrowed { + file_ref: file_ref.clone(), + pin_refcount: 0, + }, + }, }, }); @@ -1274,11 +1726,20 @@ impl Grant { match self.info.provider { Provider::PhysBorrowed { ref mut base } => *base = base.next_by(middle_page_offset), - Provider::FmapBorrowed { ref mut file_ref, .. } | Provider::Allocated { cow_file_ref: Some(ref mut file_ref), .. } => file_ref.base_offset += middle_page_offset * PAGE_SIZE, - Provider::Allocated { cow_file_ref: None, .. } | Provider::AllocatedShared { .. } | Provider::External { .. } => (), + Provider::FmapBorrowed { + ref mut file_ref, .. + } + | Provider::Allocated { + cow_file_ref: Some(ref mut file_ref), + .. + } => file_ref.base_offset += middle_page_offset * PAGE_SIZE, + Provider::Allocated { + cow_file_ref: None, .. + } + | Provider::AllocatedShared { .. } + | Provider::External { .. } => (), } - let after_grant = after_span.map(|span| Grant { base: span.base, info: GrantInfo { @@ -1286,19 +1747,38 @@ impl Grant { mapped: self.info.mapped, page_count: span.count, provider: match self.info.provider { - Provider::Allocated { cow_file_ref: None, .. } => Provider::Allocated { cow_file_ref: None, phys_contiguous: false }, - Provider::AllocatedShared { .. } => Provider::AllocatedShared { is_pinned_userscheme_borrow: false }, - Provider::Allocated { cow_file_ref: Some(ref file_ref), .. } => Provider::Allocated { cow_file_ref: Some(GrantFileRef { - base_offset: file_ref.base_offset + this_span.count * PAGE_SIZE, - description: Arc::clone(&file_ref.description), - }), phys_contiguous: false, }, - Provider::External { ref address_space, src_base, .. } => Provider::External { + Provider::Allocated { + cow_file_ref: None, .. + } => Provider::Allocated { + cow_file_ref: None, + phys_contiguous: false, + }, + Provider::AllocatedShared { .. } => Provider::AllocatedShared { + is_pinned_userscheme_borrow: false, + }, + Provider::Allocated { + cow_file_ref: Some(ref file_ref), + .. + } => Provider::Allocated { + cow_file_ref: Some(GrantFileRef { + base_offset: file_ref.base_offset + this_span.count * PAGE_SIZE, + description: Arc::clone(&file_ref.description), + }), + phys_contiguous: false, + }, + Provider::External { + ref address_space, + src_base, + .. + } => Provider::External { address_space: Arc::clone(address_space), src_base, is_pinned_userscheme_borrow: false, }, - Provider::PhysBorrowed { base } => Provider::PhysBorrowed { base: base.next_by(this_span.count) }, + Provider::PhysBorrowed { base } => Provider::PhysBorrowed { + base: base.next_by(this_span.count), + }, Provider::FmapBorrowed { ref file_ref, .. } => Provider::FmapBorrowed { file_ref: GrantFileRef { base_offset: file_ref.base_offset + this_span.count * PAGE_SIZE, @@ -1306,7 +1786,7 @@ impl Grant { }, pin_refcount: 0, }, - } + }, }, }); @@ -1318,17 +1798,40 @@ impl Grant { } impl GrantInfo { pub fn is_pinned(&self) -> bool { - matches!(self.provider, - Provider::External { is_pinned_userscheme_borrow: true, .. } - | Provider::AllocatedShared { is_pinned_userscheme_borrow: true, .. } - | Provider::FmapBorrowed { pin_refcount: 1.., .. } + matches!( + self.provider, + Provider::External { + is_pinned_userscheme_borrow: true, + .. + } | Provider::AllocatedShared { + is_pinned_userscheme_borrow: true, + .. + } | Provider::FmapBorrowed { + pin_refcount: 1.., + .. + } ) } pub fn can_extract(&self, unpin: bool) -> bool { - !(self.is_pinned() && !unpin) | matches!(self.provider, Provider::Allocated { phys_contiguous: true, .. }) + !(self.is_pinned() && !unpin) + | matches!( + self.provider, + Provider::Allocated { + phys_contiguous: true, + .. + } + ) } pub fn unpin(&mut self) { - if let Provider::External { ref mut is_pinned_userscheme_borrow, .. } | Provider::AllocatedShared { ref mut is_pinned_userscheme_borrow, .. } = self.provider { + if let Provider::External { + ref mut is_pinned_userscheme_borrow, + .. + } + | Provider::AllocatedShared { + ref mut is_pinned_userscheme_borrow, + .. + } = self.provider + { *is_pinned_userscheme_borrow = false; } } @@ -1341,7 +1844,8 @@ impl GrantInfo { } pub fn can_have_flags(&self, flags: MapFlags) -> bool { // TODO: read (some architectures support execute-only pages) - let is_downgrade = (self.flags.has_write() || !flags.contains(MapFlags::PROT_WRITE)) && (self.flags.has_execute() || !flags.contains(MapFlags::PROT_EXEC)); + let is_downgrade = (self.flags.has_write() || !flags.contains(MapFlags::PROT_WRITE)) + && (self.flags.has_execute() || !flags.contains(MapFlags::PROT_EXEC)); match self.provider { Provider::Allocated { .. } => true, @@ -1355,10 +1859,18 @@ impl GrantInfo { } match (&self.provider, &with.provider) { - (Provider::Allocated { cow_file_ref: None, phys_contiguous: false }, Provider::Allocated { cow_file_ref: None, phys_contiguous: false }) => true, + ( + Provider::Allocated { + cow_file_ref: None, + phys_contiguous: false, + }, + Provider::Allocated { + cow_file_ref: None, + phys_contiguous: false, + }, + ) => true, //(Provider::PhysBorrowed { base: ref lhs }, Provider::PhysBorrowed { base: ref rhs }) => lhs.next_by(self.page_count) == rhs.clone(), //(Provider::External { address_space: ref lhs_space, src_base: ref lhs_base, cow: lhs_cow, .. }, Provider::External { address_space: ref rhs_space, src_base: ref rhs_base, cow: rhs_cow, .. }) => Arc::ptr_eq(lhs_space, rhs_space) && lhs_cow == rhs_cow && lhs_base.next_by(self.page_count) == rhs_base.clone(), - _ => false, } } @@ -1373,16 +1885,24 @@ impl GrantInfo { // TODO: Set GRANT_LAZY match self.provider { - Provider::External { is_pinned_userscheme_borrow, .. } => { + Provider::External { + is_pinned_userscheme_borrow, + .. + } => { flags.set(GrantFlags::GRANT_PINNED, is_pinned_userscheme_borrow); flags |= GrantFlags::GRANT_SHARED; } - Provider::Allocated { ref cow_file_ref, phys_contiguous } => { + Provider::Allocated { + ref cow_file_ref, + phys_contiguous, + } => { // !GRANT_SHARED is equivalent to "GRANT_PRIVATE" flags.set(GrantFlags::GRANT_SCHEME, cow_file_ref.is_some()); flags.set(GrantFlags::GRANT_PHYS_CONTIGUOUS, phys_contiguous); } - Provider::AllocatedShared { is_pinned_userscheme_borrow } => { + Provider::AllocatedShared { + is_pinned_userscheme_borrow, + } => { flags |= GrantFlags::GRANT_SHARED; flags.set(GrantFlags::GRANT_PINNED, is_pinned_userscheme_borrow); } @@ -1397,7 +1917,12 @@ impl GrantInfo { flags } pub fn file_ref(&self) -> Option<&GrantFileRef> { - if let Provider::FmapBorrowed { ref file_ref, .. } | Provider::Allocated { cow_file_ref: Some(ref file_ref), .. } = self.provider { + if let Provider::FmapBorrowed { ref file_ref, .. } + | Provider::Allocated { + cow_file_ref: Some(ref file_ref), + .. + } = self.provider + { Some(file_ref) } else { None @@ -1409,7 +1934,11 @@ impl Drop for GrantInfo { #[track_caller] fn drop(&mut self) { // XXX: This will not show the address... - assert!(!self.mapped, "Grant dropped while still mapped: {:#x?}", self); + assert!( + !self.mapped, + "Grant dropped while still mapped: {:#x?}", + self + ); } } @@ -1458,11 +1987,12 @@ impl Drop for Table { /// Allocates a new empty utable #[cfg(target_arch = "aarch64")] pub fn setup_new_utable() -> Result { - let utable = unsafe { PageMapper::create(TableKind::User, crate::rmm::FRAME_ALLOCATOR).ok_or(Error::new(ENOMEM))? }; + let utable = unsafe { + PageMapper::create(TableKind::User, crate::rmm::FRAME_ALLOCATOR) + .ok_or(Error::new(ENOMEM))? + }; - Ok(Table { - utable, - }) + Ok(Table { utable }) } /// Allocates a new identically mapped ktable and empty utable (same memory on x86) @@ -1470,13 +2000,18 @@ pub fn setup_new_utable() -> Result
{ pub fn setup_new_utable() -> Result
{ use crate::paging::KernelMapper; - let utable = unsafe { PageMapper::create(TableKind::User, crate::rmm::FRAME_ALLOCATOR).ok_or(Error::new(ENOMEM))? }; + let utable = unsafe { + PageMapper::create(TableKind::User, crate::rmm::FRAME_ALLOCATOR) + .ok_or(Error::new(ENOMEM))? + }; { let active_ktable = KernelMapper::lock(); let copy_mapping = |p4_no| unsafe { - let entry = active_ktable.table().entry(p4_no) + let entry = active_ktable + .table() + .entry(p4_no) .unwrap_or_else(|| panic!("expected kernel PML {} to be mapped", p4_no)); utable.table().set_entry(p4_no, entry) @@ -1488,9 +2023,7 @@ pub fn setup_new_utable() -> Result
{ } } - Ok(Table { - utable, - }) + Ok(Table { utable }) } /// Allocates a new identically mapped ktable and empty utable (same memory on x86_64). @@ -1498,13 +2031,18 @@ pub fn setup_new_utable() -> Result
{ pub fn setup_new_utable() -> Result
{ use crate::paging::KernelMapper; - let utable = unsafe { PageMapper::create(TableKind::User, crate::rmm::FRAME_ALLOCATOR).ok_or(Error::new(ENOMEM))? }; + let utable = unsafe { + PageMapper::create(TableKind::User, crate::rmm::FRAME_ALLOCATOR) + .ok_or(Error::new(ENOMEM))? + }; { let active_ktable = KernelMapper::lock(); let copy_mapping = |p4_no| unsafe { - let entry = active_ktable.table().entry(p4_no) + let entry = active_ktable + .table() + .entry(p4_no) .unwrap_or_else(|| panic!("expected kernel PML {} to be mapped", p4_no)); utable.table().set_entry(p4_no, entry) @@ -1523,9 +2061,7 @@ pub fn setup_new_utable() -> Result
{ copy_mapping(crate::PHYS_PML4); } - Ok(Table { - utable, - }) + Ok(Table { utable }) } #[derive(Clone, Copy, PartialEq)] pub enum AccessMode { @@ -1561,7 +2097,9 @@ fn cow(old_frame: Frame, old_info: &PageInfo, initial_ref_kind: RefKind) -> Resu // This reference, that is being copied on write, was the only reference to old_frame, so // no copy is necessary. if initial_ref_kind == RefKind::Shared { - old_info.refcount.store(initial_rc.to_raw(), Ordering::Relaxed); + old_info + .refcount + .store(initial_rc.to_raw(), Ordering::Relaxed); } return Ok(old_frame); } @@ -1569,7 +2107,9 @@ fn cow(old_frame: Frame, old_info: &PageInfo, initial_ref_kind: RefKind) -> Resu let new_frame = init_frame(initial_rc)?; if old_frame != the_zeroed_frame().0 { - unsafe { copy_frame_to_frame_directly(new_frame, old_frame); } + unsafe { + copy_frame_to_frame_directly(new_frame, old_frame); + } } if old_info.remove_ref() == RefCount::Zero { @@ -1579,11 +2119,19 @@ fn cow(old_frame: Frame, old_info: &PageInfo, initial_ref_kind: RefKind) -> Resu Ok(new_frame) } -fn map_zeroed(mapper: &mut PageMapper, page: Page, page_flags: PageFlags, _writable: bool) -> Result { +fn map_zeroed( + mapper: &mut PageMapper, + page: Page, + page_flags: PageFlags, + _writable: bool, +) -> Result { let new_frame = init_frame(RefCount::One)?; unsafe { - mapper.map_phys(page.start_address(), new_frame.start_address(), page_flags).ok_or(PfError::Oom)?.ignore(); + mapper + .map_phys(page.start_address(), new_frame.start_address(), page_flags) + .ok_or(PfError::Oom)? + .ignore(); } Ok(new_frame) @@ -1616,7 +2164,13 @@ pub fn try_correcting_page_tables(faulting_page: Page, access: AccessMode) -> Re Ok(()) } -fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space_guard: RwLockWriteGuard<'l, AddrSpace>, faulting_page: Page, access: AccessMode, recursion_level: u32) -> Result<(Frame, PageFlush, RwLockWriteGuard<'l, AddrSpace>), PfError> { +fn correct_inner<'l>( + addr_space_lock: &'l Arc>, + mut addr_space_guard: RwLockWriteGuard<'l, AddrSpace>, + faulting_page: Page, + access: AccessMode, + recursion_level: u32, +) -> Result<(Frame, PageFlush, RwLockWriteGuard<'l, AddrSpace>), PfError> { let mut addr_space = &mut *addr_space_guard; let Some((grant_base, grant_info)) = addr_space.grants.contains(faulting_page) else { @@ -1646,7 +2200,9 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space // By now, the memory at the faulting page is actually valid, but simply not yet mapped, either // at all, or with the required flags. - let faulting_frame_opt = addr_space.table.utable + let faulting_frame_opt = addr_space + .table + .utable .translate(faulting_page.start_address()) .map(|(phys, _page_flags)| Frame::containing_address(phys)); let faulting_pageinfo_opt = faulting_frame_opt.map(|frame| (frame, get_page_info(frame))); @@ -1661,7 +2217,9 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space let mut allow_writable = true; let frame = match grant_info.provider { - Provider::Allocated { .. } | Provider::AllocatedShared { .. } if access == AccessMode::Write => { + Provider::Allocated { .. } | Provider::AllocatedShared { .. } + if access == AccessMode::Write => + { match faulting_pageinfo_opt { Some((_, None)) => unreachable!("allocated page needs frame to be valid"), Some((frame, Some(info))) => { @@ -1670,8 +2228,13 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space } else { cow(frame, info, RefKind::Cow)? } - }, - _ => map_zeroed(&mut addr_space.table.utable, faulting_page, grant_flags, true)?, + } + _ => map_zeroed( + &mut addr_space.table.utable, + faulting_page, + grant_flags, + true, + )?, } } @@ -1692,14 +2255,21 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space None => { // TODO: the zeroed page first, readonly? - map_zeroed(&mut addr_space.table.utable, faulting_page, grant_flags, false)? + map_zeroed( + &mut addr_space.table.utable, + faulting_page, + grant_flags, + false, + )? } } } - Provider::PhysBorrowed { base } => { - base.next_by(pages_from_grant_start) - } - Provider::External { address_space: ref foreign_address_space, src_base, .. } => { + Provider::PhysBorrowed { base } => base.next_by(pages_from_grant_start), + Provider::External { + address_space: ref foreign_address_space, + src_base, + .. + } => { let foreign_address_space = Arc::clone(foreign_address_space); if Arc::ptr_eq(addr_space_lock, &foreign_address_space) { @@ -1710,14 +2280,19 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space let src_page = src_base.next_by(pages_from_grant_start); if let Some(_) = guard.grants.contains(src_page) { - let src_frame = if let Some((phys, _)) = guard.table.utable.translate(src_page.start_address()) { + let src_frame = if let Some((phys, _)) = + guard.table.utable.translate(src_page.start_address()) + { Frame::containing_address(phys) } else { // Grant was valid (TODO check), but we need to correct the underlying page. // TODO: Access mode // TODO: Reasonable maximum? - let new_recursion_level = recursion_level.checked_add(1).filter(|new_lvl| *new_lvl < 16).ok_or(PfError::RecursionLimitExceeded)?; + let new_recursion_level = recursion_level + .checked_add(1) + .filter(|new_lvl| *new_lvl < 16) + .ok_or(PfError::RecursionLimitExceeded)?; drop(guard); drop(addr_space_guard); @@ -1725,7 +2300,13 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space let ext_addrspace = &foreign_address_space; let (frame, _, _) = { let g = ext_addrspace.write(); - correct_inner(ext_addrspace, g, src_page, AccessMode::Read, new_recursion_level)? + correct_inner( + ext_addrspace, + g, + src_page, + AccessMode::Read, + new_recursion_level, + )? }; addr_space_guard = addr_space_lock.write(); @@ -1746,7 +2327,12 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space // TODO: flusher unsafe { - guard.table.utable.remap_with_full(src_page.start_address(), |_, f| (new_frame.start_address(), f)); + guard + .table + .utable + .remap_with_full(src_page.start_address(), |_, f| { + (new_frame.start_address(), f) + }); } new_frame @@ -1762,11 +2348,15 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space // TODO: Should this be called? log::warn!("Mapped zero page since grant didn't exist"); - map_zeroed(&mut guard.table.utable, src_page, grant_flags, access == AccessMode::Write)? + map_zeroed( + &mut guard.table.utable, + src_page, + grant_flags, + access == AccessMode::Write, + )? } } // TODO: NonfatalInternalError if !MAP_LAZY and this page fault occurs. - Provider::FmapBorrowed { ref file_ref, .. } => { let file_ref = file_ref.clone(); let flags = map_flags(grant_info.flags()); @@ -1776,7 +2366,8 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space ref desc => (desc.scheme, desc.number), }; let user_inner = scheme::schemes() - .get(scheme_id).and_then(|s| { + .get(scheme_id) + .and_then(|s| { if let KernelSchemes::User(user) = s { user.inner.upgrade() } else { @@ -1786,14 +2377,24 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space .ok_or(PfError::Segv)?; let offset = file_ref.base_offset as u64 + (pages_from_grant_start * PAGE_SIZE) as u64; - user_inner.request_fmap(scheme_number, offset, 1, flags).unwrap(); + user_inner + .request_fmap(scheme_number, offset, 1, flags) + .unwrap(); let context_lock = super::current().map_err(|_| PfError::NonfatalInternalError)?; - context_lock.write().hard_block(HardBlockedReason::AwaitingMmap { file_ref }); + context_lock + .write() + .hard_block(HardBlockedReason::AwaitingMmap { file_ref }); - unsafe { super::switch(); } + unsafe { + super::switch(); + } - let frame = context_lock.write().fmap_ret.take().ok_or(PfError::NonfatalInternalError)?; + let frame = context_lock + .write() + .fmap_ret + .take() + .ok_or(PfError::NonfatalInternalError)?; addr_space_guard = addr_space_lock.write(); addr_space = &mut *addr_space_guard; @@ -1805,7 +2406,13 @@ fn correct_inner<'l>(addr_space_lock: &'l Arc>, mut addr_space }; let new_flags = grant_flags.write(grant_flags.has_write() && allow_writable); - let Some(flush) = (unsafe { addr_space.table.utable.map_phys(faulting_page.start_address(), frame.start_address(), new_flags) }) else { + let Some(flush) = (unsafe { + addr_space.table.utable.map_phys( + faulting_page.start_address(), + frame.start_address(), + new_flags, + ) + }) else { // TODO return Err(PfError::Oom); }; diff --git a/src/context/mod.rs b/src/context/mod.rs index 00b80050d1..d01b6fe3fa 100644 --- a/src/context/mod.rs +++ b/src/context/mod.rs @@ -2,19 +2,22 @@ //! //! For resources on contexts, please consult [wikipedia](https://en.wikipedia.org/wiki/Context_switch) and [osdev](https://wiki.osdev.org/Context_Switching) -use alloc::borrow::Cow; -use alloc::sync::Arc; +use alloc::{borrow::Cow, sync::Arc}; use spin::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use crate::LogicalCpuSet; -use crate::paging::{RmmA, RmmArch, TableKind}; -use crate::percpu::PercpuBlock; -use crate::syscall::error::{Error, ESRCH, Result}; +use crate::{ + paging::{RmmA, RmmArch, TableKind}, + percpu::PercpuBlock, + syscall::error::{Error, Result, ESRCH}, + LogicalCpuSet, +}; -pub use self::context::{BorrowedHtBuf, Context, ContextId, Status, WaitpidKey}; -pub use self::list::ContextList; -pub use self::switch::switch; +pub use self::{ + context::{BorrowedHtBuf, Context, ContextId, Status, WaitpidKey}, + list::ContextList, + switch::switch, +}; #[cfg(target_arch = "aarch64")] #[path = "arch/aarch64.rs"] @@ -65,7 +68,9 @@ pub use self::arch::empty_cr3; pub fn init() { let mut contexts = contexts_mut(); let id = ContextId::from(crate::cpu_id().get() as usize + 1); - let context_lock = contexts.insert_context_raw(id).expect("could not initialize first context"); + let context_lock = contexts + .insert_context_raw(id) + .expect("could not initialize first context"); let mut context = context_lock.write(); context.sched_affinity = LogicalCpuSet::single(crate::cpu_id()); context.name = Cow::Borrowed("kmain"); @@ -77,7 +82,9 @@ pub fn init() { context.cpu_id = Some(crate::cpu_id()); unsafe { - PercpuBlock::current().switch_internals.set_context_id(context.id); + PercpuBlock::current() + .switch_internals + .set_context_id(context.id); } } @@ -96,5 +103,8 @@ pub fn context_id() -> ContextId { } pub fn current() -> Result>> { - contexts().current().ok_or(Error::new(ESRCH)).map(Arc::clone) + contexts() + .current() + .ok_or(Error::new(ESRCH)) + .map(Arc::clone) } diff --git a/src/context/signal.rs b/src/context/signal.rs index de19966da1..bbc04d14ef 100644 --- a/src/context/signal.rs +++ b/src/context/signal.rs @@ -1,12 +1,19 @@ use alloc::sync::Arc; use core::mem; -use syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_SIGNAL, SIG_DFL, SIG_IGN, SIGCHLD, SIGCONT, SIGKILL, SIGSTOP, SIGTSTP, SIGTTIN, SIGTTOU}; -use syscall::ptrace_event; +use syscall::{ + flag::{ + PTRACE_FLAG_IGNORE, PTRACE_STOP_SIGNAL, SIGCHLD, SIGCONT, SIGKILL, SIGSTOP, SIGTSTP, + SIGTTIN, SIGTTOU, SIG_DFL, SIG_IGN, + }, + ptrace_event, +}; -use crate::context::{contexts, switch, Status, WaitpidKey}; -use crate::start::usermode; -use crate::ptrace; -use crate::syscall::usercopy::UserSlice; +use crate::{ + context::{contexts, switch, Status, WaitpidKey}, + ptrace, + start::usermode, + syscall::usercopy::UserSlice, +}; pub fn is_user_handled(handler: Option) -> bool { let handler = handler.map(|ptr| ptr as usize).unwrap_or(0); @@ -16,7 +23,9 @@ pub fn is_user_handled(handler: Option) -> bool { pub extern "C" fn signal_handler(sig: usize) { let ((action, restorer), sigstack) = { let contexts = contexts(); - let context_lock = contexts.current().expect("context::signal_handler not inside of context"); + let context_lock = contexts + .current() + .expect("context::signal_handler not inside of context"); let context = context_lock.read(); let actions = context.actions.read(); (actions[sig], context.sigstack) @@ -24,8 +33,11 @@ pub extern "C" fn signal_handler(sig: usize) { let handler = action.sa_handler.map(|ptr| ptr as usize).unwrap_or(0); - let thumbs_down = ptrace::breakpoint_callback(PTRACE_STOP_SIGNAL, Some(ptrace_event!(PTRACE_STOP_SIGNAL, sig, handler))) - .and_then(|_| ptrace::next_breakpoint().map(|f| f.contains(PTRACE_FLAG_IGNORE))); + let thumbs_down = ptrace::breakpoint_callback( + PTRACE_STOP_SIGNAL, + Some(ptrace_event!(PTRACE_STOP_SIGNAL, sig, handler)), + ) + .and_then(|_| ptrace::next_breakpoint().map(|f| f.contains(PTRACE_FLAG_IGNORE))); if sig != SIGKILL && thumbs_down.unwrap_or(false) { // If signal can be and was ignored @@ -37,7 +49,7 @@ pub extern "C" fn signal_handler(sig: usize) { match sig { SIGCHLD => { // println!("SIGCHLD"); - }, + } SIGCONT => { // println!("Continue"); @@ -45,7 +57,9 @@ pub extern "C" fn signal_handler(sig: usize) { let contexts = contexts(); let (pid, pgid, ppid) = { - let context_lock = contexts.current().expect("context::signal_handler not inside of context"); + let context_lock = contexts + .current() + .expect("context::signal_handler not inside of context"); let mut context = context_lock.write(); context.status = Status::Runnable; (context.id, context.pgid, context.ppid) @@ -57,15 +71,18 @@ pub extern "C" fn signal_handler(sig: usize) { Arc::clone(&parent.waitpid) }; - waitpid.send(WaitpidKey { - pid: Some(pid), - pgid: Some(pgid) - }, (pid, 0xFFFF)); + waitpid.send( + WaitpidKey { + pid: Some(pid), + pgid: Some(pgid), + }, + (pid, 0xFFFF), + ); } else { println!("{}: {} not found for continue", pid.get(), ppid.get()); } } - }, + } SIGSTOP | SIGTSTP | SIGTTIN | SIGTTOU => { // println!("Stop {}", sig); @@ -73,7 +90,9 @@ pub extern "C" fn signal_handler(sig: usize) { let contexts = contexts(); let (pid, pgid, ppid) = { - let context_lock = contexts.current().expect("context::signal_handler not inside of context"); + let context_lock = contexts + .current() + .expect("context::signal_handler not inside of context"); let mut context = context_lock.write(); context.status = Status::Stopped(sig); (context.id, context.pgid, context.ppid) @@ -85,17 +104,20 @@ pub extern "C" fn signal_handler(sig: usize) { Arc::clone(&parent.waitpid) }; - waitpid.send(WaitpidKey { - pid: Some(pid), - pgid: Some(pgid) - }, (pid, (sig << 8) | 0x7F)); + waitpid.send( + WaitpidKey { + pid: Some(pid), + pgid: Some(pgid), + }, + (pid, (sig << 8) | 0x7F), + ); } else { println!("{}: {} not found for stop", pid.get(), ppid.get()); } } unsafe { switch() }; - }, + } _ => { // println!("Exit {}", sig); crate::syscall::exit(sig); @@ -108,10 +130,14 @@ pub extern "C" fn signal_handler(sig: usize) { let singlestep = { let contexts = contexts(); - let context = contexts.current().expect("context::signal_handler userspace not inside of context"); + let context = contexts + .current() + .expect("context::signal_handler userspace not inside of context"); let context = context.read(); unsafe { - ptrace::regs_for(&context).map(|s| s.is_singlestep()).unwrap_or(false) + ptrace::regs_for(&context) + .map(|s| s.is_singlestep()) + .unwrap_or(false) } }; @@ -123,7 +149,9 @@ pub extern "C" fn signal_handler(sig: usize) { sp -= mem::size_of::(); - match UserSlice::wo(sp, core::mem::size_of::()).and_then(|buf| buf.write_usize(restorer)) { + match UserSlice::wo(sp, core::mem::size_of::()) + .and_then(|buf| buf.write_usize(restorer)) + { Ok(()) => usermode(handler, sp, sig, usize::from(singlestep)), Err(error) => { log::error!("Failed to signal: {}", error); diff --git a/src/context/switch.rs b/src/context/switch.rs index e2e9d29212..a213836168 100644 --- a/src/context/switch.rs +++ b/src/context/switch.rs @@ -1,17 +1,15 @@ -use core::cell::Cell; -use core::ops::Bound; -use core::sync::atomic::Ordering; +use core::{cell::Cell, ops::Bound, sync::atomic::Ordering}; use alloc::sync::Arc; use spin::{RwLock, RwLockWriteGuard}; -use crate::context::signal::signal_handler; -use crate::context::{arch, contexts, Context}; -use crate::{interrupt, LogicalCpuId}; -use crate::percpu::PercpuBlock; -use crate::ptrace; -use crate::time; +use crate::{ + context::{arch, contexts, signal::signal_handler, Context}, + interrupt, + percpu::PercpuBlock, + ptrace, time, LogicalCpuId, +}; use super::ContextId; @@ -33,15 +31,24 @@ unsafe fn update_runnable(context: &mut Context, cpu_id: LogicalCpuId) -> bool { // Restore from signal, must only be done from another context to avoid overwriting the stack! if context.ksig_restore { - let was_singlestep = ptrace::regs_for(context).map(|s| s.is_singlestep()).unwrap_or(false); + let was_singlestep = ptrace::regs_for(context) + .map(|s| s.is_singlestep()) + .unwrap_or(false); - let ksig = context.ksig.take().expect("context::switch: ksig not set with ksig_restore"); + let ksig = context + .ksig + .take() + .expect("context::switch: ksig not set with ksig_restore"); context.arch = ksig.0; context.kfx.copy_from_slice(&*ksig.1); if let Some(ref mut kstack) = context.kstack { - kstack.copy_from_slice(&ksig.2.expect("context::switch: ksig kstack not set with ksig_restore")); + kstack.copy_from_slice( + &ksig + .2 + .expect("context::switch: ksig kstack not set with ksig_restore"), + ); } else { panic!("context::switch: kstack not set with ksig_restore"); } @@ -81,7 +88,6 @@ struct SwitchResult { next_lock: Arc>, } - pub fn tick() { let ticks_cell = &PercpuBlock::current().switch_internals.pit_ticks; @@ -95,7 +101,11 @@ pub fn tick() { } pub unsafe extern "C" fn switch_finish_hook() { - if let Some(SwitchResult { prev_lock, next_lock }) = PercpuBlock::current().switch_internals.switch_result.take() { + if let Some(SwitchResult { + prev_lock, + next_lock, + }) = PercpuBlock::current().switch_internals.switch_result.take() + { prev_lock.force_write_unlock(); next_lock.force_write_unlock(); } else { @@ -118,7 +128,10 @@ pub unsafe fn switch() -> bool { // Set the global lock to avoid the unsafe operations below from causing issues // TODO: Better memory orderings? - while arch::CONTEXT_SWITCH_LOCK.compare_exchange_weak(false, true, Ordering::SeqCst, Ordering::Relaxed).is_err() { + while arch::CONTEXT_SWITCH_LOCK + .compare_exchange_weak(false, true, Ordering::SeqCst, Ordering::Relaxed) + .is_err() + { interrupt::pause(); } @@ -130,20 +143,21 @@ pub unsafe fn switch() -> bool { let contexts = contexts(); // Lock previous context - let prev_context_lock = contexts.current().expect("context::switch: not inside of context"); + let prev_context_lock = contexts + .current() + .expect("context::switch: not inside of context"); let prev_context_guard = prev_context_lock.write(); // Locate next context for (_pid, next_context_lock) in contexts // Include all contexts with IDs greater than the current... - .range( - (Bound::Excluded(prev_context_guard.id), Bound::Unbounded) + .range((Bound::Excluded(prev_context_guard.id), Bound::Unbounded)) + .chain( + contexts + // ... and all contexts with IDs less than the current... + .range((Bound::Unbounded, Bound::Excluded(prev_context_guard.id))), ) - .chain(contexts - // ... and all contexts with IDs less than the current... - .range((Bound::Unbounded, Bound::Excluded(prev_context_guard.id))) - ) - // ... but not the current context, which is already locked + // ... but not the current context, which is already locked { // Lock next context let mut next_context_guard = next_context_lock.write(); @@ -165,7 +179,9 @@ pub unsafe fn switch() -> bool { }; // Switch process states, TSS stack pointer, and store new context ID - if let Some((prev_context_lock, prev_context_ptr, next_context_lock, next_context_ptr)) = switch_context_opt { + if let Some((prev_context_lock, prev_context_ptr, next_context_lock, next_context_ptr)) = + switch_context_opt + { // Set old context as not running and update CPU time let prev_context = &mut *prev_context_ptr; prev_context.running = false; @@ -198,10 +214,13 @@ pub unsafe fn switch() -> bool { } } - percpu.switch_internals.switch_result.set(Some(SwitchResult { - prev_lock: prev_context_lock, - next_lock: next_context_lock, - })); + percpu + .switch_internals + .switch_result + .set(Some(SwitchResult { + prev_lock: prev_context_lock, + next_lock: next_context_lock, + })); arch::switch_to(prev_context, next_context); diff --git a/src/context/timeout.rs b/src/context/timeout.rs index 220811dafc..a1c23d5b6b 100644 --- a/src/context/timeout.rs +++ b/src/context/timeout.rs @@ -1,11 +1,15 @@ use alloc::collections::VecDeque; -use spin::{Once, Mutex, MutexGuard}; +use spin::{Mutex, MutexGuard, Once}; -use crate::event; -use crate::scheme::SchemeId; -use crate::syscall::data::TimeSpec; -use crate::syscall::flag::{CLOCK_MONOTONIC, CLOCK_REALTIME, EVENT_READ}; -use crate::time; +use crate::{ + event, + scheme::SchemeId, + syscall::{ + data::TimeSpec, + flag::{CLOCK_MONOTONIC, CLOCK_REALTIME, EVENT_READ}, + }, + time, +}; #[derive(Debug)] struct Timeout { @@ -35,7 +39,7 @@ pub fn register(scheme_id: SchemeId, event_id: usize, clock: usize, time: TimeSp scheme_id, event_id, clock, - time: (time.tv_sec as u128 * time::NANOS_PER_SEC) + (time.tv_nsec as u128) + time: (time.tv_sec as u128 * time::NANOS_PER_SEC) + (time.tv_nsec as u128), }); } @@ -51,11 +55,11 @@ pub fn trigger() { CLOCK_MONOTONIC => { let time = registry[i].time; mono >= time - }, + } CLOCK_REALTIME => { let time = registry[i].time; real >= time - }, + } clock => { println!("timeout::trigger: unknown clock {}", clock); true diff --git a/src/debugger.rs b/src/debugger.rs index 7d31bd8200..08aa134114 100644 --- a/src/debugger.rs +++ b/src/debugger.rs @@ -9,7 +9,7 @@ use crate::paging::{RmmA, RmmArch, TableKind, PAGE_SIZE}; pub unsafe fn debugger(target_id: Option) { use hashbrown::HashSet; - use crate::memory::{RefCount, get_page_info}; + use crate::memory::{get_page_info, RefCount}; println!("DEBUGGER START"); println!(); @@ -21,12 +21,14 @@ pub unsafe fn debugger(target_id: Option) { let mut spaces = HashSet::new(); for (id, context_lock) in crate::context::contexts().iter() { - if target_id.map_or(false, |target_id| *id != target_id) { continue; } + if target_id.map_or(false, |target_id| *id != target_id) { + continue; + } let context = context_lock.read(); println!("{}: {}", (*id).get(), context.name); println!("status: {:?}", context.status); - if ! context.status_reason.is_empty() { + if !context.status_reason.is_empty() { println!("reason: {}", context.status_reason); } @@ -38,18 +40,23 @@ pub unsafe fn debugger(target_id: Option) { check_consistency(&mut *space.write(), new_as, &mut tree); if let Some((a, b, c, d, e, f)) = context.syscall { - println!("syscall: {}", crate::syscall::debug::format_call(a, b, c, d, e, f)); + println!( + "syscall: {}", + crate::syscall::debug::format_call(a, b, c, d, e, f) + ); } { let space = space.read(); - if ! space.grants.is_empty() { + if !space.grants.is_empty() { println!("grants:"); for (base, grant) in space.grants.iter() { println!( " virt 0x{:016x}:0x{:016x} size 0x{:08x} {:?}", - base.start_address().data(), base.next_by(grant.page_count() - 1).start_address().data() + 0xFFF, grant.page_count() * PAGE_SIZE, grant.provider, - + base.start_address().data(), + base.next_by(grant.page_count() - 1).start_address().data() + 0xFFF, + grant.page_count() * PAGE_SIZE, + grant.provider, ); } } @@ -63,7 +70,14 @@ pub unsafe fn debugger(target_id: Option) { println!("stack: {:>016x}", sp); //Maximum 64 usizes for _ in 0..64 { - if context.addr_space.as_ref().map_or(false, |space| space.read().table.utable.translate(crate::paging::VirtualAddress::new(sp)).is_some()) { + if context.addr_space.as_ref().map_or(false, |space| { + space + .read() + .table + .utable + .translate(crate::paging::VirtualAddress::new(sp)) + .is_some() + }) { let value = *(sp as *const usize); println!(" {:>016x}: {:>016x}", sp, value); if let Some(next_sp) = sp.checked_add(core::mem::size_of::()) { @@ -110,7 +124,9 @@ pub unsafe fn debugger(target_id: Option) { let old_table = RmmA::table(TableKind::User); for (id, context_lock) in crate::context::contexts().iter() { - if target_id.map_or(false, |target_id| *id != target_id) { continue; } + if target_id.map_or(false, |target_id| *id != target_id) { + continue; + } let context = context_lock.read(); println!("{}: {}", (*id).get(), context.name); @@ -121,20 +137,25 @@ pub unsafe fn debugger(target_id: Option) { } println!("status: {:?}", context.status); - if ! context.status_reason.is_empty() { + if !context.status_reason.is_empty() { println!("reason: {}", context.status_reason); } if let Some((a, b, c, d, e, f)) = context.syscall { - println!("syscall: {}", crate::syscall::debug::format_call(a, b, c, d, e, f)); + println!( + "syscall: {}", + crate::syscall::debug::format_call(a, b, c, d, e, f) + ); } if let Some(ref addr_space) = context.addr_space { let addr_space = addr_space.read(); - if ! addr_space.grants.is_empty() { + if !addr_space.grants.is_empty() { println!("grants:"); for (base, grant) in addr_space.grants.iter() { println!( " virt 0x{:08x}:0x{:08x} size 0x{:08x} {:?}", - base.start_address().data(), base.next_by(grant.page_count()).start_address().data() + 0xFFF, grant.page_count() * crate::memory::PAGE_SIZE, + base.start_address().data(), + base.next_by(grant.page_count()).start_address().data() + 0xFFF, + grant.page_count() * crate::memory::PAGE_SIZE, grant.provider, ); } @@ -148,7 +169,14 @@ pub unsafe fn debugger(target_id: Option) { println!("stack: {:>08x}", sp); //Maximum 64 dwords for _ in 0..64 { - if context.addr_space.as_ref().map_or(false, |space| space.read().table.utable.translate(crate::paging::VirtualAddress::new(sp)).is_some()) { + if context.addr_space.as_ref().map_or(false, |space| { + space + .read() + .table + .utable + .translate(crate::paging::VirtualAddress::new(sp)) + .is_some() + }) { let value = *(sp as *const usize); println!(" {:>08x}: {:>08x}", sp, value); if let Some(next_sp) = sp.checked_add(core::mem::size_of::()) { @@ -178,9 +206,11 @@ pub unsafe fn debugger(target_id: Option) { pub unsafe fn debugger(target_id: Option) { use hashbrown::HashSet; - use crate::memory::{RefCount, get_page_info, the_zeroed_frame}; + use crate::memory::{get_page_info, the_zeroed_frame, RefCount}; - unsafe { x86::bits64::rflags::stac(); } + unsafe { + x86::bits64::rflags::stac(); + } println!("DEBUGGER START"); println!(); @@ -195,7 +225,9 @@ pub unsafe fn debugger(target_id: Option) { let old_table = RmmA::table(TableKind::User); for (id, context_lock) in crate::context::contexts().iter() { - if target_id.map_or(false, |target_id| *id != target_id) { continue; } + if target_id.map_or(false, |target_id| *id != target_id) { + continue; + } let context = context_lock.read(); println!("{}: {}", (*id).get(), context.name); @@ -218,21 +250,26 @@ pub unsafe fn debugger(target_id: Option) { } println!("status: {:?}", context.status); - if ! context.status_reason.is_empty() { + if !context.status_reason.is_empty() { println!("reason: {}", context.status_reason); } if let Some((a, b, c, d, e, f)) = context.syscall { - println!("syscall: {}", crate::syscall::debug::format_call(a, b, c, d, e, f)); + println!( + "syscall: {}", + crate::syscall::debug::format_call(a, b, c, d, e, f) + ); } if let Some(ref addr_space) = context.addr_space { let addr_space = addr_space.read(); - if ! addr_space.grants.is_empty() { + if !addr_space.grants.is_empty() { println!("grants:"); for (base, info) in addr_space.grants.iter() { let size = info.page_count() * PAGE_SIZE; println!( " virt 0x{:016x}:0x{:016x} size 0x{:08x} {:?}", - base.start_address().data(), base.start_address().data() + size - 1, size, + base.start_address().data(), + base.start_address().data() + size - 1, + size, info.provider, ); } @@ -246,7 +283,14 @@ pub unsafe fn debugger(target_id: Option) { println!("stack: {:>016x}", rsp); //Maximum 64 qwords for _ in 0..64 { - if context.addr_space.as_ref().map_or(false, |space| space.read().table.utable.translate(crate::paging::VirtualAddress::new(rsp)).is_some()) { + if context.addr_space.as_ref().map_or(false, |space| { + space + .read() + .table + .utable + .translate(crate::paging::VirtualAddress::new(rsp)) + .is_some() + }) { let value = *(rsp as *const usize); println!(" {:>016x}: {:>016x}", rsp, value); if let Some(next_rsp) = rsp.checked_add(core::mem::size_of::()) { @@ -280,22 +324,36 @@ pub unsafe fn debugger(target_id: Option) { RefCount::Shared(s) => s.get(), }; if c != count { - println!("frame refcount mismatch for {:?} ({} != {})", frame, c, count); + println!( + "frame refcount mismatch for {:?} ({} != {})", + frame, c, count + ); } } - println!("({} kernel-owned references were not counted)", temporarily_taken_htbufs); + println!( + "({} kernel-owned references were not counted)", + temporarily_taken_htbufs + ); println!("DEBUGGER END"); - unsafe { x86::bits64::rflags::clac(); } + unsafe { + x86::bits64::rflags::clac(); + } } #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -use {hashbrown::HashMap, crate::memory::Frame}; +use {crate::memory::Frame, hashbrown::HashMap}; #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] -pub unsafe fn check_consistency(addr_space: &mut crate::context::memory::AddrSpace, new_as: bool, tree: &mut HashMap) { - use crate::context::memory::{PageSpan, Provider}; - use crate::memory::{get_page_info, RefCount}; - use crate::paging::*; +pub unsafe fn check_consistency( + addr_space: &mut crate::context::memory::AddrSpace, + new_as: bool, + tree: &mut HashMap, +) { + use crate::{ + context::memory::{PageSpan, Provider}, + memory::{get_page_info, RefCount}, + paging::*, + }; let p4 = addr_space.table.utable.table(); @@ -319,28 +377,52 @@ pub unsafe fn check_consistency(addr_space: &mut crate::context::memory::AddrSpa for p1i in 0..512 { let (physaddr, flags) = match p1.entry(p1i) { - Some(e) => if let Ok(address) = e.address() { - (address, e.flags()) - } else { - continue; + Some(e) => { + if let Ok(address) = e.address() { + (address, e.flags()) + } else { + continue; + } } _ => continue, }; - let address = VirtualAddress::new((p1i << 12) | (p2i << 21) | (p3i << 30) | (p4i << 39)); + let address = + VirtualAddress::new((p1i << 12) | (p2i << 21) | (p3i << 30) | (p4i << 39)); - let (base, grant) = match addr_space.grants.contains(Page::containing_address(address)) { + let (base, grant) = match addr_space + .grants + .contains(Page::containing_address(address)) + { Some(g) => g, None => { - log::error!("ADDRESS {:p} LACKING GRANT BUT MAPPED TO {:#0x} FLAGS {:?}!", address.data() as *const u8, physaddr.data(), flags); + log::error!( + "ADDRESS {:p} LACKING GRANT BUT MAPPED TO {:#0x} FLAGS {:?}!", + address.data() as *const u8, + physaddr.data(), + flags + ); continue; } }; const EXCLUDE: usize = (1 << 5) | (1 << 6); // accessed+dirty+writable - if grant.flags().write(false).data() & !EXCLUDE != flags.write(false).data() & !EXCLUDE { - log::error!("FLAG MISMATCH: {:?} != {:?}, address {:p} in grant at {:?}", grant.flags(), flags, address.data() as *const u8, PageSpan::new(base, grant.page_count())); + if grant.flags().write(false).data() & !EXCLUDE + != flags.write(false).data() & !EXCLUDE + { + log::error!( + "FLAG MISMATCH: {:?} != {:?}, address {:p} in grant at {:?}", + grant.flags(), + flags, + address.data() as *const u8, + PageSpan::new(base, grant.page_count()) + ); } - let p = matches!(grant.provider, Provider::PhysBorrowed { .. } | Provider::External { .. } | Provider::FmapBorrowed { .. }); + let p = matches!( + grant.provider, + Provider::PhysBorrowed { .. } + | Provider::External { .. } + | Provider::FmapBorrowed { .. } + ); let frame = Frame::containing_address(physaddr); if new_as { tree.entry(frame).or_insert((0, p)).0 += 1; @@ -350,8 +432,13 @@ pub unsafe fn check_consistency(addr_space: &mut crate::context::memory::AddrSpa match page.refcount() { RefCount::Zero => panic!("mapped page with zero refcount"), - RefCount::One | RefCount::Shared(_) => assert!(!(flags.has_write() && !grant.flags().has_write()), "page entry has higher permissions than grant!"), - RefCount::Cow(_) => assert!(!flags.has_write(), "directly writable CoW page!"), + RefCount::One | RefCount::Shared(_) => assert!( + !(flags.has_write() && !grant.flags().has_write()), + "page entry has higher permissions than grant!" + ), + RefCount::Cow(_) => { + assert!(!flags.has_write(), "directly writable CoW page!") + } } } else { //println!("!OWNED {:?}", frame); diff --git a/src/devices/graphical_debug/debug.rs b/src/devices/graphical_debug/debug.rs index 427c997c4f..93f115d625 100644 --- a/src/devices/graphical_debug/debug.rs +++ b/src/devices/graphical_debug/debug.rs @@ -1,7 +1,7 @@ use super::Display; pub struct DebugDisplay { - pub (crate) display: Display, + pub(crate) display: Display, x: usize, y: usize, w: usize, @@ -10,8 +10,8 @@ pub struct DebugDisplay { impl DebugDisplay { pub fn new(display: Display) -> DebugDisplay { - let w = display.width/8; - let h = display.height/16; + let w = display.width / 8; + let h = display.height / 16; DebugDisplay { display, x: 0, @@ -34,18 +34,15 @@ impl DebugDisplay { self.display.scroll(d_y * 16); unsafe { - self.display.sync(0, 0, self.display.width, self.display.height); + self.display + .sync(0, 0, self.display.width, self.display.height); } self.y = new_y; } if c != '\n' { - self.display.char( - self.x * 8, self.y * 16, - c, - 0xFFFFFF - ); + self.display.char(self.x * 8, self.y * 16, c, 0xFFFFFF); unsafe { self.display.sync(self.x * 8, self.y * 16, 8, 16); diff --git a/src/devices/graphical_debug/display.rs b/src/devices/graphical_debug/display.rs index d3d93ec17f..2997f4453d 100644 --- a/src/devices/graphical_debug/display.rs +++ b/src/devices/graphical_debug/display.rs @@ -9,7 +9,7 @@ pub struct Display { pub height: usize, pub stride: usize, pub onscreen: &'static mut [u32], - pub offscreen: Option> + pub offscreen: Option>, } impl Display { @@ -46,7 +46,9 @@ impl Display { let row_data = FONT[font_i + row]; for col in 0..8 { if (row_data >> (7 - col)) & 1 == 1 { - unsafe { *((dst + col * 4) as *mut u32) = color; } + unsafe { + *((dst + col * 4) as *mut u32) = color; + } } } dst += self.stride * 4; @@ -71,9 +73,7 @@ impl Display { if let Some(offscreen) = &self.offscreen { let mut offset = y * self.stride + x; while h > 0 { - self.onscreen[offset..offset+w].copy_from_slice( - &offscreen[offset..offset+w] - ); + self.onscreen[offset..offset + w].copy_from_slice(&offscreen[offset..offset + w]); offset += self.stride; h -= 1; } diff --git a/src/devices/graphical_debug/mod.rs b/src/devices/graphical_debug/mod.rs index 32ac85c06e..e3cfb62484 100644 --- a/src/devices/graphical_debug/mod.rs +++ b/src/devices/graphical_debug/mod.rs @@ -56,7 +56,10 @@ pub fn init(env: &[u8]) { return; } - println!("Framebuffer {}x{} stride {} at {:X} mapped to {:X}", width, height, stride, phys, virt); + println!( + "Framebuffer {}x{} stride {} at {:X} mapped to {:X}", + width, height, stride, phys, virt + ); { let display = Display::new(width, height, stride, virt as *mut u32); @@ -67,9 +70,8 @@ pub fn init(env: &[u8]) { pub fn init_heap() { if let Some(debug_display) = &mut *DEBUG_DISPLAY.lock() { - debug_display.display.offscreen = Some( - debug_display.display.onscreen.to_vec().into_boxed_slice() - ); + debug_display.display.offscreen = + Some(debug_display.display.onscreen.to_vec().into_boxed_slice()); } } diff --git a/src/devices/uart_16550.rs b/src/devices/uart_16550.rs index 51a0b09d5c..628cac58c5 100644 --- a/src/devices/uart_16550.rs +++ b/src/devices/uart_16550.rs @@ -1,9 +1,11 @@ -use core::convert::TryInto; -use core::ptr::{addr_of, addr_of_mut}; +use core::{ + convert::TryInto, + ptr::{addr_of, addr_of_mut}, +}; -use crate::syscall::io::{Io, Mmio, ReadOnly}; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use crate::syscall::io::Pio; +use crate::syscall::io::{Io, Mmio, ReadOnly}; bitflags! { /// Interrupt enable flags diff --git a/src/elf.rs b/src/elf.rs index 70e4610713..ed3f886cfd 100644 --- a/src/elf.rs +++ b/src/elf.rs @@ -17,22 +17,34 @@ pub use goblin::elf64::{header, program_header, section_header, sym}; /// An ELF executable pub struct Elf<'a> { pub data: &'a [u8], - header: &'a header::Header + header: &'a header::Header, } impl<'a> Elf<'a> { /// Create a ELF executable from data pub fn from(data: &'a [u8]) -> Result, String> { if data.len() < header::SIZEOF_EHDR { - Err(format!("Elf: Not enough data: {} < {}", data.len(), header::SIZEOF_EHDR)) + Err(format!( + "Elf: Not enough data: {} < {}", + data.len(), + header::SIZEOF_EHDR + )) } else if &data[..header::SELFMAG] != header::ELFMAG { - Err(format!("Elf: Invalid magic: {:?} != {:?}", &data[..header::SELFMAG], header::ELFMAG)) + Err(format!( + "Elf: Invalid magic: {:?} != {:?}", + &data[..header::SELFMAG], + header::ELFMAG + )) } else if data.get(header::EI_CLASS) != Some(&header::ELFCLASS) { - Err(format!("Elf: Invalid architecture: {:?} != {:?}", data.get(header::EI_CLASS), header::ELFCLASS)) + Err(format!( + "Elf: Invalid architecture: {:?} != {:?}", + data.get(header::EI_CLASS), + header::ELFCLASS + )) } else { Ok(Elf { data, - header: unsafe { &*(data.as_ptr() as usize as *const header::Header) } + header: unsafe { &*(data.as_ptr() as usize as *const header::Header) }, }) } } @@ -41,7 +53,7 @@ impl<'a> Elf<'a> { ElfSections { data: self.data, header: self.header, - i: 0 + i: 0, } } @@ -58,7 +70,7 @@ impl<'a> Elf<'a> { Some(ElfSymbols { data: self.data, symtab, - i: 0 + i: 0, }) } else { None @@ -69,7 +81,7 @@ impl<'a> Elf<'a> { pub struct ElfSections<'a> { data: &'a [u8], header: &'a header::Header, - i: usize + i: usize, } impl<'a> Iterator for ElfSections<'a> { @@ -77,11 +89,10 @@ impl<'a> Iterator for ElfSections<'a> { fn next(&mut self) -> Option { if self.i < self.header.e_shnum as usize { let item = unsafe { - &* (( - self.data.as_ptr() as usize - + self.header.e_shoff as usize - + self.i * self.header.e_shentsize as usize - ) as *const section_header::SectionHeader) + &*((self.data.as_ptr() as usize + + self.header.e_shoff as usize + + self.i * self.header.e_shentsize as usize) + as *const section_header::SectionHeader) }; self.i += 1; Some(item) @@ -94,7 +105,7 @@ impl<'a> Iterator for ElfSections<'a> { pub struct ElfSegments<'a> { data: &'a [u8], header: &'a header::Header, - i: usize + i: usize, } impl<'a> Iterator for ElfSegments<'a> { @@ -102,11 +113,10 @@ impl<'a> Iterator for ElfSegments<'a> { fn next(&mut self) -> Option { if self.i < self.header.e_phnum as usize { let item = unsafe { - &* (( - self.data.as_ptr() as usize - + self.header.e_phoff as usize - + self.i * self.header.e_phentsize as usize - ) as *const program_header::ProgramHeader) + &*((self.data.as_ptr() as usize + + self.header.e_phoff as usize + + self.i * self.header.e_phentsize as usize) + as *const program_header::ProgramHeader) }; self.i += 1; Some(item) @@ -119,7 +129,7 @@ impl<'a> Iterator for ElfSegments<'a> { pub struct ElfSymbols<'a> { data: &'a [u8], symtab: &'a section_header::SectionHeader, - i: usize + i: usize, } impl<'a> Iterator for ElfSymbols<'a> { @@ -127,11 +137,9 @@ impl<'a> Iterator for ElfSymbols<'a> { fn next(&mut self) -> Option { if self.i < (self.symtab.sh_size as usize) / sym::SIZEOF_SYM { let item = unsafe { - &* (( - self.data.as_ptr() as usize - + self.symtab.sh_offset as usize - + self.i * sym::SIZEOF_SYM - ) as *const sym::Sym) + &*((self.data.as_ptr() as usize + + self.symtab.sh_offset as usize + + self.i * sym::SIZEOF_SYM) as *const sym::Sym) }; self.i += 1; Some(item) diff --git a/src/event.rs b/src/event.rs index dbfb16be54..ec74065cf0 100644 --- a/src/event.rs +++ b/src/event.rs @@ -3,13 +3,17 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use hashbrown::HashMap; use spin::{Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use crate::context; -use crate::scheme::{self, SchemeId}; -use crate::sync::WaitQueue; -use crate::syscall::data::Event; -use crate::syscall::error::{Error, Result, EBADF, ESRCH}; -use crate::syscall::flag::EventFlags; -use crate::syscall::usercopy::UserSliceWo; +use crate::{ + context, + scheme::{self, SchemeId}, + sync::WaitQueue, + syscall::{ + data::Event, + error::{Error, Result, EBADF, ESRCH}, + flag::EventFlags, + usercopy::UserSliceWo, + }, +}; int_like!(EventQueueId, AtomicEventQueueId, usize, AtomicUsize); @@ -22,7 +26,7 @@ impl EventQueue { pub fn new(id: EventQueueId) -> EventQueue { EventQueue { id, - queue: WaitQueue::new() + queue: WaitQueue::new(), } } @@ -39,7 +43,7 @@ impl EventQueue { let files = context.files.read(); match files.get(event.id).ok_or(Error::new(EBADF))? { Some(file) => file.clone(), - None => return Err(Error::new(EBADF)) + None => return Err(Error::new(EBADF)), } }; @@ -50,8 +54,12 @@ impl EventQueue { register( RegKey { scheme, number }, - QueueKey { queue: self.id, id: event.id, data: event.data }, - event.flags + QueueKey { + queue: self.id, + id: event.id, + data: event.data, + }, + event.flags, ); let flags = sync(RegKey { scheme, number })?; @@ -127,9 +135,7 @@ pub fn registry_mut() -> RwLockWriteGuard<'static, Registry> { pub fn register(reg_key: RegKey, queue_key: QueueKey, flags: EventFlags) { let mut registry = registry_mut(); - let entry = registry.entry(reg_key).or_insert_with(|| { - HashMap::new() - }); + let entry = registry.entry(reg_key).or_insert_with(|| HashMap::new()); if flags.is_empty() { entry.remove(&queue_key); @@ -152,7 +158,8 @@ pub fn sync(reg_key: RegKey) -> Result { } let scheme = scheme::schemes() - .get(reg_key.scheme).ok_or(Error::new(EBADF))? + .get(reg_key.scheme) + .ok_or(Error::new(EBADF))? .clone(); scheme.fevent(reg_key.number, flags) @@ -181,7 +188,7 @@ pub fn trigger(scheme: SchemeId, number: usize, flags: EventFlags) { queue.queue.send(Event { id: queue_key.id, flags: common_flags, - data: queue_key.data + data: queue_key.data, }); } } diff --git a/src/log.rs b/src/log.rs index b53733764c..d4151bb7dd 100644 --- a/src/log.rs +++ b/src/log.rs @@ -17,7 +17,7 @@ impl Log { pub fn new(size: usize) -> Log { Log { data: VecDeque::with_capacity(size), - size + size, } } @@ -55,14 +55,16 @@ pub fn init_logger(func: fn(&log::Record)) { match LOGGER.initialized.load(Ordering::SeqCst) { false => { ::log::set_max_level(::log::LevelFilter::Info); - LOGGER.log_func = func; - match ::log::set_logger(&LOGGER) { - Ok(_) => ::log::info!("Logger initialized."), - Err(e) => println!("Logger setup failed! error: {}", e), - } + LOGGER.log_func = func; + match ::log::set_logger(&LOGGER) { + Ok(_) => ::log::info!("Logger initialized."), + Err(e) => println!("Logger setup failed! error: {}", e), + } LOGGER.initialized.store(true, Ordering::SeqCst); - }, - true => ::log::info!("Tried to reinitialize the logger, which is not possible. Ignoring."), + } + true => { + ::log::info!("Tried to reinitialize the logger, which is not possible. Ignoring.") + } } } } diff --git a/src/main.rs b/src/main.rs index 046a8d893f..333c5ebf78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,6 @@ #![allow(clippy::too_many_arguments)] // There is no harm in this being done #![allow(clippy::useless_format)] - // TODO: address ocurrances and then deny #![warn(clippy::not_unsafe_ptr_arg_deref)] // TODO: address ocurrances and then deny @@ -35,13 +34,11 @@ // Avoid panicking in the kernel without information about the panic. Use expect // TODO: address ocurrances and then deny #![warn(clippy::result_unwrap_used)] - // This is usually a serious issue - a missing import of a define where it is interpreted // as a catch-all variable in a match, for example #![deny(unreachable_patterns)] // Ensure that all must_use results are used #![deny(unused_must_use)] - #![feature(allocator_api)] #![feature(asm_const)] // TODO: Relax requirements of most asm invocations #![feature(int_roundings)] @@ -97,7 +94,7 @@ mod debugger; mod devices; /// ELF file parsing -#[cfg(not(feature="doc"))] +#[cfg(not(feature = "doc"))] mod elf; /// Event handling @@ -113,7 +110,7 @@ mod log; mod memory; /// Panic -#[cfg(not(any(feature="doc", test)))] +#[cfg(not(any(feature = "doc", test)))] mod panic; mod percpu; @@ -202,7 +199,7 @@ fn kmain(cpu_count: u32, bootstrap: Bootstrap) -> ! { context.ens = SchemeNamespace::from(1); context.status = context::Status::Runnable; context.name = "bootstrap".into(); - }, + } Err(err) => { panic!("failed to spawn userspace_init: {:?}", err); } @@ -261,8 +258,13 @@ fn kmain_ap(cpu_id: LogicalCpuId) -> ! { /// Allow exception handlers to send signal to arch-independant kernel #[no_mangle] -extern fn ksignal(signal: usize) { - info!("SIGNAL {}, CPU {}, PID {:?}", signal, cpu_id(), context::context_id()); +extern "C" fn ksignal(signal: usize) { + info!( + "SIGNAL {}, CPU {}, PID {:?}", + signal, + cpu_id(), + context::context_id() + ); { let contexts = context::contexts(); if let Some(context_lock) = contexts.current() { @@ -296,7 +298,18 @@ macro_rules! linker_offsets( } ); mod kernel_executable_offsets { - linker_offsets!(__text_start, __text_end, __rodata_start, __rodata_end, __data_start, __data_end, __bss_start, __bss_end, __usercopy_start, __usercopy_end); + linker_offsets!( + __text_start, + __text_end, + __rodata_start, + __rodata_end, + __data_start, + __data_end, + __bss_start, + __bss_end, + __usercopy_start, + __usercopy_end + ); #[cfg(target_arch = "x86_64")] linker_offsets!(__altrelocs_start, __altrelocs_end); @@ -315,8 +328,12 @@ struct LogicalCpuId(u32); impl LogicalCpuId { const BSP: Self = Self::new(0); - const fn new(inner: u32) -> Self { Self(inner) } - const fn get(self) -> u32 { self.0 } + const fn new(inner: u32) -> Self { + Self(inner) + } + const fn get(self) -> u32 { + self.0 + } } impl core::fmt::Debug for LogicalCpuId { @@ -338,12 +355,22 @@ impl core::fmt::Display for LogicalCpuId { struct LogicalCpuSet(u128); impl LogicalCpuSet { - const fn new(inner: u128) -> Self { Self(inner) } - const fn get(self) -> u128 { self.0 } + const fn new(inner: u128) -> Self { + Self(inner) + } + const fn get(self) -> u128 { + self.0 + } - const fn empty() -> Self { Self::new(0) } - const fn all() -> Self { Self::new(!0) } - const fn single(id: LogicalCpuId) -> Self { Self::new(1 << id.get()) } + const fn empty() -> Self { + Self::new(0) + } + const fn all() -> Self { + Self::new(!0) + } + const fn single(id: LogicalCpuId) -> Self { + Self::new(1 << id.get()) + } const fn contains(&self, id: LogicalCpuId) -> bool { self.0 & (1 << id.get()) != 0 } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 45c9d8bdab..2d2b1ac705 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -1,25 +1,29 @@ //! # Memory management //! Some code was borrowed from [Phil Opp's Blog](http://os.phil-opp.com/allocating-frames.html) -use core::cell::SyncUnsafeCell; -use core::mem; -use core::num::NonZeroUsize; -use core::sync::atomic::{AtomicUsize, Ordering}; - -use crate::arch::rmm::LockedAllocator; -use crate::context::{self, memory::{AccessMode, PfError}}; -use crate::kernel_executable_offsets::{__usercopy_start, __usercopy_end}; -use crate::paging::Page; -pub use crate::paging::{PAGE_SIZE, PhysicalAddress, RmmA, RmmArch}; -use crate::rmm::areas; - -use alloc::vec::Vec; -use rmm::{ - FrameAllocator, - FrameCount, VirtualAddress, TableKind, +use core::{ + cell::SyncUnsafeCell, + mem, + num::NonZeroUsize, + sync::atomic::{AtomicUsize, Ordering}, }; + +pub use crate::paging::{PhysicalAddress, RmmA, RmmArch, PAGE_SIZE}; +use crate::{ + arch::rmm::LockedAllocator, + context::{ + self, + memory::{AccessMode, PfError}, + }, + kernel_executable_offsets::{__usercopy_end, __usercopy_start}, + paging::Page, + rmm::areas, +}; + +use crate::syscall::error::{Error, ENOMEM}; +use alloc::vec::Vec; +use rmm::{FrameAllocator, FrameCount, TableKind, VirtualAddress}; use spin::RwLock; -use crate::syscall::error::{ENOMEM, Error}; /// A memory map area #[derive(Copy, Clone, Debug, Default)] @@ -28,29 +32,25 @@ pub struct MemoryArea { pub base_addr: u64, pub length: u64, pub _type: u32, - pub acpi: u32 + pub acpi: u32, } /// Get the number of frames available pub fn free_frames() -> usize { - unsafe { - LockedAllocator.usage().free().data() - } + unsafe { LockedAllocator.usage().free().data() } } /// Get the number of frames used pub fn used_frames() -> usize { - unsafe { - LockedAllocator.usage().used().data() - } + unsafe { LockedAllocator.usage().used().data() } } /// Allocate a range of frames pub fn allocate_frames(count: usize) -> Option { unsafe { - LockedAllocator.allocate(FrameCount::new(count)).map(|phys| { - Frame::containing_address(PhysicalAddress::new(phys.data())) - }) + LockedAllocator + .allocate(FrameCount::new(count)) + .map(|phys| Frame::containing_address(PhysicalAddress::new(phys.data()))) } } // TODO: allocate_frames_complex @@ -61,7 +61,7 @@ pub fn deallocate_frames(frame: Frame, count: usize) { unsafe { LockedAllocator.free( rmm::PhysicalAddress::new(frame.start_address().data()), - FrameCount::new(count) + FrameCount::new(count), ); } } @@ -74,7 +74,11 @@ pub struct Frame { } impl core::fmt::Debug for Frame { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "[frame at {:p}]", self.start_address().data() as *const u8) + write!( + f, + "[frame at {:p}]", + self.start_address().data() as *const u8 + ) } } @@ -97,11 +101,19 @@ impl Frame { } pub fn next_by(self, n: usize) -> Self { Self { - number: self.number.get().checked_add(n).and_then(NonZeroUsize::new).expect("overflow in Frame::next_by"), + number: self + .number + .get() + .checked_add(n) + .and_then(NonZeroUsize::new) + .expect("overflow in Frame::next_by"), } } pub fn offset_from(self, from: Self) -> usize { - self.number.get().checked_sub(from.number.get()).expect("overflow in Frame::offset_from") + self.number + .get() + .checked_sub(from.number.get()) + .expect("overflow in Frame::offset_from") } } @@ -139,7 +151,9 @@ pub struct RaiiFrame { } impl RaiiFrame { pub fn allocate() -> Result { - init_frame(RefCount::One).map_err(|_| Enomem).map(|inner| Self { inner }) + init_frame(RefCount::One) + .map_err(|_| Enomem) + .map(|inner| Self { inner }) } pub fn get(&self) -> Frame { self.inner @@ -150,7 +164,8 @@ impl Drop for RaiiFrame { fn drop(&mut self) { if get_page_info(self.inner) .expect("RaiiFrame lacking PageInfo") - .remove_ref() == RefCount::Zero + .remove_ref() + == RefCount::Zero { crate::memory::deallocate_frames(self.inner, 1); } @@ -223,7 +238,10 @@ fn init_sections() { while let Some(mut memory_map_area) = iter.next() { // TODO: NonZeroUsize - assert_ne!(memory_map_area.size, 0, "RMM should enforce areas are not zeroed"); + assert_ne!( + memory_map_area.size, 0, + "RMM should enforce areas are not zeroed" + ); // TODO: Would it make sense to naturally align the sections? // TODO: Should RMM do this? @@ -233,8 +251,16 @@ fn init_sections() { let _ = iter.next(); } - assert_eq!(memory_map_area.base.data() % PAGE_SIZE, 0, "RMM should enforce area alignment"); - assert_eq!(memory_map_area.size % PAGE_SIZE, 0, "RMM should enforce area length alignment"); + assert_eq!( + memory_map_area.base.data() % PAGE_SIZE, + 0, + "RMM should enforce area alignment" + ); + assert_eq!( + memory_map_area.size % PAGE_SIZE, + 0, + "RMM should enforce area length alignment" + ); let mut pages_left = memory_map_area.size.div_floor(PAGE_SIZE); let mut base = Frame::containing_address(memory_map_area.base); @@ -262,10 +288,7 @@ fn init_sections() { core::slice::from_raw_parts_mut(virt as *mut PageInfo, section_page_count) }; - sections.push(Section { - base, - frames, - }); + sections.push(Section { base, frames }); pages_left -= section_page_count; base = base.next_by(section_page_count); @@ -291,7 +314,9 @@ pub fn init_mm() { unsafe { let the_frame = allocate_frames(1).expect("failed to allocate static zeroed frame"); let the_info = get_page_info(the_frame).expect("static zeroed frame had no PageInfo"); - the_info.refcount.store(RefCount::One.to_raw(), Ordering::Relaxed); + the_info + .refcount + .store(RefCount::One.to_raw(), Ordering::Relaxed); THE_ZEROED_FRAME.get().write(Some((the_frame, the_info))); } @@ -306,7 +331,9 @@ impl PageInfo { match (self.refcount(), kind) { (RefCount::Zero, _) => self.refcount.store(1, Ordering::Relaxed), (RefCount::One, RefKind::Cow) => self.refcount.store(2, Ordering::Relaxed), - (RefCount::One, RefKind::Shared) => self.refcount.store(2 | RC_SHARED_NOT_COW, Ordering::Relaxed), + (RefCount::One, RefKind::Shared) => self + .refcount + .store(2 | RC_SHARED_NOT_COW, Ordering::Relaxed), (RefCount::Cow(_), RefKind::Cow) | (RefCount::Shared(_), RefKind::Shared) => { self.refcount.fetch_add(1, Ordering::Relaxed); } @@ -324,7 +351,9 @@ impl PageInfo { 0 } - RefCount::Cow(_) | RefCount::Shared(_) => self.refcount.fetch_sub(1, Ordering::Relaxed) - 1, + RefCount::Cow(_) | RefCount::Shared(_) => { + self.refcount.fetch_sub(1, Ordering::Relaxed) - 1 + } }) } pub fn allows_writable(&self) -> bool { @@ -382,8 +411,7 @@ impl RefCount { pub fn get_page_info(frame: Frame) -> Option<&'static PageInfo> { let sections = SECTIONS.read(); - let idx_res = sections - .binary_search_by_key(&frame, |section| section.base); + let idx_res = sections.binary_search_by_key(&frame, |section| section.base); if idx_res == Err(0) || idx_res == Err(sections.len()) { return None; @@ -426,7 +454,11 @@ pub trait ArchIntCtx { fn recover_and_efault(&mut self); } -pub fn page_fault_handler(stack: &mut impl ArchIntCtx, code: GenericPfFlags, faulting_address: VirtualAddress) -> Result<(), Segv> { +pub fn page_fault_handler( + stack: &mut impl ArchIntCtx, + code: GenericPfFlags, + faulting_address: VirtualAddress, +) -> Result<(), Segv> { let faulting_page = Page::containing_address(faulting_address); let usercopy_region = __usercopy_start()..__usercopy_end(); @@ -446,7 +478,9 @@ pub fn page_fault_handler(stack: &mut impl ArchIntCtx, code: GenericPfFlags, fau (true, false) => AccessMode::Write, (false, false) => AccessMode::Read, (false, true) => AccessMode::InstrFetch, - (true, true) => unreachable!("page fault cannot be caused by both instruction fetch and write"), + (true, true) => { + unreachable!("page fault cannot be caused by both instruction fetch and write") + } }; if invalid_page_tables { @@ -470,19 +504,30 @@ pub fn page_fault_handler(stack: &mut impl ArchIntCtx, code: GenericPfFlags, fau Err(Segv) } -static THE_ZEROED_FRAME: SyncUnsafeCell> = SyncUnsafeCell::new(None); +static THE_ZEROED_FRAME: SyncUnsafeCell> = + SyncUnsafeCell::new(None); pub fn the_zeroed_frame() -> (Frame, &'static PageInfo) { unsafe { - THE_ZEROED_FRAME.get().read().expect("zeroed frame must be initialized") + THE_ZEROED_FRAME + .get() + .read() + .expect("zeroed frame must be initialized") } } pub fn init_frame(init_rc: RefCount) -> Result { let new_frame = crate::memory::allocate_frames(1).ok_or(PfError::Oom)?; - let page_info = get_page_info(new_frame).unwrap_or_else(|| panic!("all allocated frames need an associated page info, {:?} didn't", new_frame)); + let page_info = get_page_info(new_frame).unwrap_or_else(|| { + panic!( + "all allocated frames need an associated page info, {:?} didn't", + new_frame + ) + }); assert_eq!(page_info.refcount(), RefCount::Zero); - page_info.refcount.store(init_rc.to_raw(), Ordering::Relaxed); + page_info + .refcount + .store(init_rc.to_raw(), Ordering::Relaxed); Ok(new_frame) } diff --git a/src/panic.rs b/src/panic.rs index d0ae67faa2..97293642bd 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,14 +2,16 @@ use core::panic::PanicInfo; -use crate::{cpu_id, context, interrupt, syscall}; +use crate::{context, cpu_id, interrupt, syscall}; /// Required to handle panics #[panic_handler] fn rust_begin_unwind(info: &PanicInfo) -> ! { println!("KERNEL PANIC: {}", info); - unsafe { interrupt::stack_trace(); } + unsafe { + interrupt::stack_trace(); + } println!("CPU {}, PID {:?}", cpu_id(), context::context_id()); @@ -28,6 +30,8 @@ fn rust_begin_unwind(info: &PanicInfo) -> ! { println!("HALT"); loop { - unsafe { interrupt::halt(); } + unsafe { + interrupt::halt(); + } } } diff --git a/src/percpu.rs b/src/percpu.rs index 2107ec60d0..98be1ce5b1 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -1,5 +1,4 @@ -use crate::LogicalCpuId; -use crate::context::switch::ContextSwitchPercpu; +use crate::{context::switch::ContextSwitchPercpu, LogicalCpuId}; /// The percpu block, that stored all percpu variables. pub struct PercpuBlock { @@ -11,7 +10,6 @@ pub struct PercpuBlock { // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. - #[cfg(feature = "profiling")] pub profiling: Option<&'static crate::profiling::RingBuffer>, } diff --git a/src/profiling.rs b/src/profiling.rs index 5ae8621c1f..83b3d64f4a 100644 --- a/src/profiling.rs +++ b/src/profiling.rs @@ -1,16 +1,19 @@ -use core::cell::UnsafeCell; -use core::mem::size_of; -use core::sync::atomic::{AtomicUsize, Ordering, AtomicPtr, AtomicBool, AtomicU32}; +use core::{ + cell::UnsafeCell, + mem::size_of, + sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicUsize, Ordering}, +}; use alloc::boxed::Box; -use crate::idt::Idt; -use crate::interrupt::irq::aux_timer; -use crate::{LogicalCpuId, interrupt}; -use crate::interrupt::InterruptStack; -use crate::percpu::PercpuBlock; -use crate::syscall::error::*; -use crate::syscall::usercopy::UserSliceWo; +use crate::{ + idt::Idt, + interrupt, + interrupt::{irq::aux_timer, InterruptStack}, + percpu::PercpuBlock, + syscall::{error::*, usercopy::UserSliceWo}, + LogicalCpuId, +}; const N: usize = 64 * 1024 * 1024; @@ -28,10 +31,16 @@ pub struct RingBuffer { impl RingBuffer { unsafe fn advance_head(&self, n: usize) { - self.head.store(self.head.load(Ordering::Acquire).wrapping_add(n), Ordering::Release); + self.head.store( + self.head.load(Ordering::Acquire).wrapping_add(n), + Ordering::Release, + ); } unsafe fn advance_tail(&self, n: usize) { - self.tail.store(self.tail.load(Ordering::Acquire).wrapping_add(n), Ordering::Release); + self.tail.store( + self.tail.load(Ordering::Acquire).wrapping_add(n), + Ordering::Release, + ); } unsafe fn sender_owned(&self) -> [&[UnsafeCell]; 2] { let head = self.head.load(Ordering::Acquire) % N; @@ -67,13 +76,13 @@ impl RingBuffer { n } pub unsafe fn peek(&self) -> [&[usize]; 2] { - self.receiver_owned().map(|slice| core::slice::from_raw_parts(slice.as_ptr().cast(), slice.len())) + self.receiver_owned() + .map(|slice| core::slice::from_raw_parts(slice.as_ptr().cast(), slice.len())) } pub unsafe fn advance(&self, n: usize) { self.advance_head(n) } pub fn create() -> &'static Self { - Box::leak(Box::new(Self { head: AtomicUsize::new(0), tail: AtomicUsize::new(0), @@ -103,10 +112,20 @@ pub fn serio_command(index: usize, data: u8) { pub fn drain_buffer(cpu_num: LogicalCpuId, buf: UserSliceWo) -> Result { unsafe { - let Some(src) = BUFS.get(cpu_num.get() as usize).ok_or(Error::new(EBADFD))?.load(Ordering::Relaxed).as_ref() else { + let Some(src) = BUFS + .get(cpu_num.get() as usize) + .ok_or(Error::new(EBADFD))? + .load(Ordering::Relaxed) + .as_ref() + else { return Ok(0); }; - let byte_slices = src.peek().map(|words| core::slice::from_raw_parts(words.as_ptr().cast::(), words.len() * size_of::())); + let byte_slices = src.peek().map(|words| { + core::slice::from_raw_parts( + words.as_ptr().cast::(), + words.len() * size_of::(), + ) + }); let copied_1 = buf.copy_common_bytes_from_slice(byte_slices[0])?; src.advance(copied_1 / size_of::()); @@ -130,17 +149,23 @@ pub unsafe fn nmi_handler(stack: &InterruptStack) { return; } if stack.iret.cs & 0b00 == 0b11 { - profiling.nmi_ucount.store(profiling.nmi_ucount.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + profiling.nmi_ucount.store( + profiling.nmi_ucount.load(Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); return; } else if stack.iret.rflags & (1 << 9) != 0 { // Interrupts were enabled, i.e. we were in kmain, so ignore. return; } else { - profiling.nmi_kcount.store(profiling.nmi_kcount.load(Ordering::Relaxed) + 1, Ordering::Relaxed); + profiling.nmi_kcount.store( + profiling.nmi_kcount.load(Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); }; let mut buf = [0_usize; 32]; - buf[0] = stack.iret.rip & !(1<<63); + buf[0] = stack.iret.rip & !(1 << 63); buf[1] = x86::time::rdtsc() as usize; let mut bp = stack.preserved.rbp; @@ -148,13 +173,17 @@ pub unsafe fn nmi_handler(stack: &InterruptStack) { let mut len = 2; for i in 2..32 { - if bp.saturating_add(16) < crate::KERNEL_HEAP_OFFSET || bp >= crate::KERNEL_HEAP_OFFSET + crate::PML4_SIZE { + if bp.saturating_add(16) < crate::KERNEL_HEAP_OFFSET + || bp >= crate::KERNEL_HEAP_OFFSET + crate::PML4_SIZE + { break; } let ip = ((bp + 8) as *const usize).read(); bp = (bp as *const usize).read(); - if ip < crate::kernel_executable_offsets::__text_start() || ip >= crate::kernel_executable_offsets::__text_end() { + if ip < crate::kernel_executable_offsets::__text_start() + || ip >= crate::kernel_executable_offsets::__text_end() + { break; } buf[i] = ip; @@ -167,12 +196,18 @@ pub unsafe fn nmi_handler(stack: &InterruptStack) { pub unsafe fn init() { let percpu = PercpuBlock::current(); - if percpu.cpu_id == PROFILER_CPU { return } + if percpu.cpu_id == PROFILER_CPU { + return; + } let profiling = RingBuffer::create(); - BUFS[percpu.cpu_id.get() as usize].store(profiling as *const _ as *mut _, core::sync::atomic::Ordering::SeqCst); - (core::ptr::addr_of!(percpu.profiling) as *mut Option<&'static RingBuffer>).write(Some(profiling)); + BUFS[percpu.cpu_id.get() as usize].store( + profiling as *const _ as *mut _, + core::sync::atomic::Ordering::SeqCst, + ); + (core::ptr::addr_of!(percpu.profiling) as *mut Option<&'static RingBuffer>) + .write(Some(profiling)); } static ACK: AtomicU32 = AtomicU32::new(0); @@ -187,7 +222,14 @@ pub fn maybe_run_profiling_helper_forever(cpu_id: LogicalCpuId) { } unsafe { for i in 33..255 { - crate::idt::IDTS.write().as_mut().unwrap().get_mut(&cpu_id).unwrap().entries[i].set_func(crate::interrupt::ipi::wakeup); + crate::idt::IDTS + .write() + .as_mut() + .unwrap() + .get_mut(&cpu_id) + .unwrap() + .entries[i] + .set_func(crate::interrupt::ipi::wakeup); } let apic = &mut crate::device::local_apic::LOCAL_APIC; diff --git a/src/ptrace.rs b/src/ptrace.rs index 94336c330d..0fec42df95 100644 --- a/src/ptrace.rs +++ b/src/ptrace.rs @@ -9,19 +9,10 @@ use crate::{ event, scheme::GlobalSchemes, sync::WaitCondition, - syscall::{ - data::PtraceEvent, - error::*, - flag::*, - ptrace_event - }, + syscall::{data::PtraceEvent, error::*, flag::*, ptrace_event}, }; -use alloc::{ - boxed::Box, - collections::VecDeque, - sync::Arc, -}; +use alloc::{boxed::Box, collections::VecDeque, sync::Arc}; use core::cmp; use hashbrown::hash_map::{Entry, HashMap}; use spin::{Mutex, Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; @@ -54,7 +45,7 @@ impl SessionData { pub fn set_breakpoint(&mut self, flags: Option) { self.breakpoint = flags.map(|flags| Breakpoint { reached: false, - flags + flags, }); } @@ -215,7 +206,7 @@ pub fn send_event(event: PtraceEvent) -> Option<()> { #[derive(Debug, Clone, Copy)] struct Breakpoint { reached: bool, - flags: PtraceFlags + flags: PtraceFlags, } /// Wait for the tracee to stop, or return immediately if there's an unread @@ -230,7 +221,7 @@ pub fn wait(pid: ContextId) -> Result<()> { match sessions.get(&pid) { Some(session) => Arc::clone(session), - _ => return Ok(()) + _ => return Ok(()), } }; @@ -260,7 +251,10 @@ pub fn wait(pid: ContextId) -> Result<()> { /// /// Note: Don't call while holding any locks or allocated data, this /// will switch contexts and may in fact just never terminate. -pub fn breakpoint_callback(match_flags: PtraceFlags, event: Option) -> Option { +pub fn breakpoint_callback( + match_flags: PtraceFlags, + event: Option, +) -> Option { loop { let session = { let contexts = context::contexts(); @@ -282,7 +276,8 @@ pub fn breakpoint_callback(match_flags: PtraceFlags, event: Option) } // In case no tracer is waiting, make sure the next one gets the memo - data.breakpoint.as_mut() + data.breakpoint + .as_mut() .expect("already checked that breakpoint isn't None") .reached = true; @@ -362,7 +357,7 @@ impl Drop for ProcessRegsGuard { /// stack instead of the original. pub unsafe fn rebase_regs_ptr( regs: Option<(usize, Unique)>, - kstack: Option<&Box<[u8]>> + kstack: Option<&Box<[u8]>>, ) -> Option<*const InterruptStack> { let (old_base, ptr) = regs?; let new_base = kstack?.as_ptr() as usize; @@ -372,7 +367,7 @@ pub unsafe fn rebase_regs_ptr( /// stack instead of the original. pub unsafe fn rebase_regs_ptr_mut( regs: Option<(usize, Unique)>, - kstack: Option<&mut Box<[u8]>> + kstack: Option<&mut Box<[u8]>>, ) -> Option<*mut InterruptStack> { let (old_base, ptr) = regs?; let new_base = kstack?.as_mut_ptr() as usize; diff --git a/src/scheme/acpi.rs b/src/scheme/acpi.rs index 6c3ed154a9..91aef9b9ef 100644 --- a/src/scheme/acpi.rs +++ b/src/scheme/acpi.rs @@ -1,28 +1,30 @@ -use core::convert::TryInto; -use core::str; -use core::sync::atomic::{self, AtomicUsize}; +use core::{ + convert::TryInto, + str, + sync::atomic::{self, AtomicUsize}, +}; -use alloc::boxed::Box; -use alloc::collections::BTreeMap; +use alloc::{boxed::Box, collections::BTreeMap}; use spin::{Mutex, Once, RwLock}; -use crate::acpi::{RXSDT_ENUM, RxsdtEnum}; -use crate::event; -use crate::sync::WaitCondition; - -use crate::syscall::data::Stat; -use crate::syscall::error::{EACCES, EBADF, EBADFD, EINTR, EINVAL, EISDIR, ENOENT, ENOTDIR, EROFS}; -use crate::syscall::flag::{ - EventFlags, EVENT_READ, - MODE_CHR, MODE_DIR, MODE_FILE, - O_ACCMODE, O_CREAT, O_DIRECTORY, O_EXCL, O_RDONLY, O_STAT, O_SYMLINK, - SEEK_SET, SEEK_CUR, SEEK_END, +use crate::{ + acpi::{RxsdtEnum, RXSDT_ENUM}, + event, + sync::WaitCondition, }; -use crate::syscall::error::{Error, Result}; -use crate::syscall::usercopy::{UserSliceRo, UserSliceWo}; -use super::{GlobalSchemes, KernelScheme, CallerCtx, OpenResult}; +use crate::syscall::{ + data::Stat, + error::{Error, Result, EACCES, EBADF, EBADFD, EINTR, EINVAL, EISDIR, ENOENT, ENOTDIR, EROFS}, + flag::{ + EventFlags, EVENT_READ, MODE_CHR, MODE_DIR, MODE_FILE, O_ACCMODE, O_CREAT, O_DIRECTORY, + O_EXCL, O_RDONLY, O_STAT, O_SYMLINK, SEEK_CUR, SEEK_END, SEEK_SET, + }, + usercopy::{UserSliceRo, UserSliceWo}, +}; + +use super::{CallerCtx, GlobalSchemes, KernelScheme, OpenResult}; /// A scheme used to access the RSDT or XSDT, which is needed for e.g. `acpid` to function. pub struct AcpiScheme; @@ -56,7 +58,10 @@ pub fn register_kstop() -> bool { let handles = HANDLES.read(); - for (&fd, _) in handles.iter().filter(|(_, handle)| handle.kind == HandleKind::ShutdownPipe) { + for (&fd, _) in handles + .iter() + .filter(|(_, handle)| handle.kind == HandleKind::ShutdownPipe) + { event::trigger(GlobalSchemes::Acpi.scheme_id(), fd, EVENT_READ); waiters_awoken += 1; } @@ -142,11 +147,14 @@ impl KernelScheme for AcpiScheme { let fd = NEXT_FD.fetch_add(1, atomic::Ordering::Relaxed); let mut handles_guard = HANDLES.write(); - let _ = handles_guard.insert(fd, Handle { - offset: 0, - kind: handle_kind, - stat: flags & O_STAT == O_STAT, - }); + let _ = handles_guard.insert( + fd, + Handle { + offset: 0, + kind: handle_kind, + stat: flags & O_STAT == O_STAT, + }, + ); Ok(OpenResult::SchemeLocal(fd)) } @@ -166,15 +174,24 @@ impl KernelScheme for AcpiScheme { let new_offset = match whence { SEEK_SET => pos as usize, - SEEK_CUR => if pos < 0 { - handle.offset.checked_sub((-pos) as usize).ok_or(Error::new(EINVAL))? - } else { - handle.offset.saturating_add(pos as usize) + SEEK_CUR => { + if pos < 0 { + handle + .offset + .checked_sub((-pos) as usize) + .ok_or(Error::new(EINVAL))? + } else { + handle.offset.saturating_add(pos as usize) + } } - SEEK_END => if pos < 0 { - file_len.checked_sub((-pos) as usize).ok_or(Error::new(EINVAL))? - } else { - file_len + SEEK_END => { + if pos < 0 { + file_len + .checked_sub((-pos) as usize) + .ok_or(Error::new(EINVAL))? + } else { + file_len + } } _ => return Err(Error::new(EINVAL)), }; @@ -222,7 +239,7 @@ impl KernelScheme for AcpiScheme { if *flag_guard { break; - } else if ! KSTOP_WAITCOND.wait(flag_guard, "waiting for kstop") { + } else if !KSTOP_WAITCOND.wait(flag_guard, "waiting for kstop") { return Err(Error::new(EINTR)); } } @@ -258,10 +275,13 @@ impl KernelScheme for AcpiScheme { st_size: data.len().try_into().unwrap_or(u64::max_value()), ..Default::default() } - }, + } HandleKind::TopLevel => Stat { st_mode: MODE_DIR, - st_size: TOPLEVEL_CONTENTS.len().try_into().unwrap_or(u64::max_value()), + st_size: TOPLEVEL_CONTENTS + .len() + .try_into() + .unwrap_or(u64::max_value()), ..Default::default() }, HandleKind::ShutdownPipe => Stat { diff --git a/src/scheme/debug.rs b/src/scheme/debug.rs index a00daaeb8e..29ed755db9 100644 --- a/src/scheme/debug.rs +++ b/src/scheme/debug.rs @@ -1,12 +1,16 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use spin::RwLock; -use crate::arch::debug::Writer; -use crate::event; -use crate::scheme::*; -use crate::sync::WaitQueue; -use crate::syscall::flag::{EventFlags, EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK}; -use crate::syscall::usercopy::{UserSliceRo, UserSliceWo}; +use crate::{ + arch::debug::Writer, + event, + scheme::*, + sync::WaitQueue, + syscall::{ + flag::{EventFlags, EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK}, + usercopy::{UserSliceRo, UserSliceWo}, + }, +}; static NEXT_ID: AtomicUsize = AtomicUsize::new(0); @@ -54,10 +58,13 @@ impl KernelScheme for DebugScheme { }; let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, Handle { - flags: flags & ! O_ACCMODE, - num, - }); + HANDLES.write().insert( + id, + Handle { + flags: flags & !O_ACCMODE, + num, + }, + ); Ok(OpenResult::SchemeLocal(id)) } @@ -68,10 +75,10 @@ impl KernelScheme for DebugScheme { match cmd { F_GETFL => Ok(handle.flags), F_SETFL => { - handle.flags = arg & ! O_ACCMODE; + handle.flags = arg & !O_ACCMODE; Ok(0) - }, - _ => Err(Error::new(EINVAL)) + } + _ => Err(Error::new(EINVAL)), } } else { Err(Error::new(EBADF)) @@ -113,11 +120,17 @@ impl KernelScheme for DebugScheme { #[cfg(feature = "profiling")] if handle.num != !0 { - return crate::profiling::drain_buffer(crate::LogicalCpuId::new(handle.num as u32), buf); + return crate::profiling::drain_buffer( + crate::LogicalCpuId::new(handle.num as u32), + buf, + ); } - INPUT - .receive_into_user(buf, handle.flags & O_NONBLOCK != O_NONBLOCK, "DebugScheme::read") + INPUT.receive_into_user( + buf, + handle.flags & O_NONBLOCK != O_NONBLOCK, + "DebugScheme::read", + ) } fn kwrite(&self, id: usize, buf: UserSliceRo) -> Result { @@ -155,7 +168,9 @@ impl KernelScheme for DebugScheme { // TODO: Copy elsewhere in the kernel? const SRC: &[u8] = b"debug:"; let byte_count = core::cmp::min(buf.len(), SRC.len()); - buf.limit(byte_count).expect("must succeed").copy_from_slice(&SRC[..byte_count])?; + buf.limit(byte_count) + .expect("must succeed") + .copy_from_slice(&SRC[..byte_count])?; Ok(byte_count) } diff --git a/src/scheme/dtb.rs b/src/scheme/dtb.rs index a37d20efec..716d1009b0 100644 --- a/src/scheme/dtb.rs +++ b/src/scheme/dtb.rs @@ -1,21 +1,19 @@ use core::sync::atomic::{self, AtomicUsize}; -use alloc::boxed::Box; -use alloc::collections::BTreeMap; +use alloc::{boxed::Box, collections::BTreeMap}; use spin::{Once, RwLock}; -use crate::dtb::DTB_BINARY; -use crate::scheme::SchemeId; -use crate::syscall::flag::{ - MODE_FILE, - O_STAT, - SEEK_CUR, SEEK_END, SEEK_SET, +use super::{CallerCtx, KernelScheme, OpenResult}; +use crate::{ + dtb::DTB_BINARY, + scheme::SchemeId, + syscall::{ + data::Stat, + error::*, + flag::{MODE_FILE, O_STAT, SEEK_CUR, SEEK_END, SEEK_SET}, + usercopy::{UserSliceRo, UserSliceWo}, + }, }; -use crate::syscall::data::Stat; -use crate::syscall::error::*; -use crate::syscall::usercopy::{UserSliceRo, UserSliceWo}; -use super::{KernelScheme, OpenResult, CallerCtx}; - pub struct DtbScheme; @@ -43,7 +41,7 @@ impl DtbScheme { let dtb = match DTB_BINARY.get() { Some(dtb) => dtb.as_slice(), - None => &[] + None => &[], }; Box::from(dtb) @@ -56,9 +54,7 @@ impl DtbScheme { } impl KernelScheme for DtbScheme { - fn kopen(&self, path: &str, _flags: usize, _ctx: CallerCtx) -> Result { - let path = path.trim_matches('/'); if path.is_empty() { @@ -66,12 +62,15 @@ impl KernelScheme for DtbScheme { let mut handles_guard = HANDLES.write(); - let _ = handles_guard.insert(id, Handle { - offset: 0, - kind: HandleKind::RawData, - stat: _flags & O_STAT == O_STAT, - }); - return Ok(OpenResult::SchemeLocal(id)) + let _ = handles_guard.insert( + id, + Handle { + offset: 0, + kind: HandleKind::RawData, + stat: _flags & O_STAT == O_STAT, + }, + ); + return Ok(OpenResult::SchemeLocal(id)); } Err(Error::new(ENOENT)) @@ -91,15 +90,24 @@ impl KernelScheme for DtbScheme { let new_offset = match whence { SEEK_SET => pos as usize, - SEEK_CUR => if pos < 0 { - handle.offset.checked_sub((-pos) as usize).ok_or(Error::new(EINVAL))? - } else { - handle.offset.saturating_add(pos as usize) + SEEK_CUR => { + if pos < 0 { + handle + .offset + .checked_sub((-pos) as usize) + .ok_or(Error::new(EINVAL))? + } else { + handle.offset.saturating_add(pos as usize) + } } - SEEK_END => if pos < 0 { - file_len.checked_sub((-pos) as usize).ok_or(Error::new(EINVAL))? - } else { - file_len + SEEK_END => { + if pos < 0 { + file_len + .checked_sub((-pos) as usize) + .ok_or(Error::new(EINVAL))? + } else { + file_len + } } _ => return Err(Error::new(EINVAL)), }; @@ -144,11 +152,10 @@ impl KernelScheme for DtbScheme { } fn kfstat(&self, id: usize, buf: UserSliceWo) -> Result<()> { - let handles = HANDLES.read(); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; buf.copy_exactly(&match handle.kind { - HandleKind::RawData => { + HandleKind::RawData => { let data = DATA.get().ok_or(Error::new(EBADFD))?; Stat { st_mode: MODE_FILE, diff --git a/src/scheme/event.rs b/src/scheme/event.rs index 2bcc2c5ae7..2d6476662e 100644 --- a/src/scheme/event.rs +++ b/src/scheme/event.rs @@ -1,12 +1,16 @@ use alloc::sync::Arc; use core::mem; -use crate::event::{EventQueue, EventQueueId, next_queue_id, queues, queues_mut}; -use crate::syscall::data::Event; -use crate::syscall::error::*; -use crate::syscall::usercopy::{UserSliceWo, UserSliceRo}; +use crate::{ + event::{next_queue_id, queues, queues_mut, EventQueue, EventQueueId}, + syscall::{ + data::Event, + error::*, + usercopy::{UserSliceRo, UserSliceWo}, + }, +}; -use super::{KernelScheme, CallerCtx, OpenResult}; +use super::{CallerCtx, KernelScheme, OpenResult}; pub struct EventScheme; @@ -34,7 +38,10 @@ impl KernelScheme for EventScheme { fn close(&self, id: usize) -> Result<()> { let id = EventQueueId::from(id); - queues_mut().remove(&id).ok_or(Error::new(EBADF)).and(Ok(())) + queues_mut() + .remove(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(())) } fn kread(&self, id: usize, buf: UserSliceWo) -> Result { let id = EventQueueId::from(id); diff --git a/src/scheme/irq.rs b/src/scheme/irq.rs index be35dad4cf..703a473b1c 100644 --- a/src/scheme/irq.rs +++ b/src/scheme/irq.rs @@ -1,23 +1,28 @@ -use core::{mem, str}; -use core::str::FromStr; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::{ + mem, str, + str::FromStr, + sync::atomic::{AtomicUsize, Ordering}, +}; -use alloc::collections::BTreeMap; -use alloc::vec::Vec; -use alloc::string::String; +use alloc::{collections::BTreeMap, string::String, vec::Vec}; -use spin::{Mutex, RwLock, Once}; +use spin::{Mutex, Once, RwLock}; use crate::arch::interrupt::{available_irqs_iter, bsp_apic_id, is_reserved, set_reserved}; -use crate::{event, LogicalCpuId}; -use crate::interrupt::irq::acknowledge; -use crate::syscall::data::Stat; -use crate::syscall::error::*; -use crate::syscall::flag::{EventFlags, EVENT_READ, O_DIRECTORY, O_CREAT, O_STAT, MODE_CHR, MODE_DIR}; -use crate::syscall::usercopy::{UserSliceWo, UserSliceRo}; +use crate::{ + event, + interrupt::irq::acknowledge, + syscall::{ + data::Stat, + error::*, + flag::{EventFlags, EVENT_READ, MODE_CHR, MODE_DIR, O_CREAT, O_DIRECTORY, O_STAT}, + usercopy::{UserSliceRo, UserSliceWo}, + }, + LogicalCpuId, +}; -use super::{GlobalSchemes, OpenResult, CallerCtx, calc_seek_offset}; +use super::{calc_seek_offset, CallerCtx, GlobalSchemes, OpenResult}; /// IRQ queues pub(super) static COUNTS: Mutex<[usize; 224]> = Mutex::new([0; 224]); @@ -42,21 +47,23 @@ const INO_BSP: u64 = 0x8001_0000_0000_0000; /// Add to the input queue #[no_mangle] -pub extern fn irq_trigger(irq: u8) { +pub extern "C" fn irq_trigger(irq: u8) { COUNTS.lock()[irq as usize] += 1; - for (fd, _) in HANDLES.read().iter().filter_map(|(fd, handle)| Some((fd, handle.as_irq_handle()?))).filter(|&(_, (_, handle_irq))| handle_irq == irq) { + for (fd, _) in HANDLES + .read() + .iter() + .filter_map(|(fd, handle)| Some((fd, handle.as_irq_handle()?))) + .filter(|&(_, (_, handle_irq))| handle_irq == irq) + { event::trigger(GlobalSchemes::Irq.scheme_id(), *fd, EVENT_READ); } } enum Handle { - Irq { - ack: AtomicUsize, - irq: u8, - }, - Avail(u8, Vec, AtomicUsize), // CPU id, data, offset - TopLevel(Vec, AtomicUsize), // data, offset + Irq { ack: AtomicUsize, irq: u8 }, + Avail(u8, Vec, AtomicUsize), // CPU id, data, offset + TopLevel(Vec, AtomicUsize), // data, offset Bsp, } impl Handle { @@ -80,49 +87,59 @@ impl IrqScheme { use crate::acpi::madt::*; match unsafe { MADT.as_ref() } { - Some(madt) => { - madt.iter().filter_map(|entry| match entry { + Some(madt) => madt + .iter() + .filter_map(|entry| match entry { MadtEntry::LocalApic(apic) => Some(apic.id), _ => None, - }).collect::>() - }, + }) + .collect::>(), None => { log::warn!("no MADT found, defaulting to 1 CPU"); - vec!(0) + vec![0] } } }; #[cfg(not(all(feature = "acpi", any(target_arch = "x86", target_arch = "x86_64"))))] - let cpus = vec!(0); + let cpus = vec![0]; CPUS.call_once(|| cpus); } fn open_ext_irq(flags: usize, cpu_id: u8, path_str: &str) -> Result { let irq_number = u8::from_str(path_str).or(Err(Error::new(ENOENT)))?; - Ok(if irq_number < BASE_IRQ_COUNT && Some(u32::from(cpu_id)) == bsp_apic_id() { - // Give legacy IRQs only to `irq:{0..15}` and `irq:cpu-/{0..15}` (same handles). - // - // The only CPUs don't have the legacy IRQs in their IDTs. + Ok( + if irq_number < BASE_IRQ_COUNT && Some(u32::from(cpu_id)) == bsp_apic_id() { + // Give legacy IRQs only to `irq:{0..15}` and `irq:cpu-/{0..15}` (same handles). + // + // The only CPUs don't have the legacy IRQs in their IDTs. - Handle::Irq { - ack: AtomicUsize::new(0), - irq: irq_number, - } - } else if irq_number < TOTAL_IRQ_COUNT { - if flags & O_CREAT == 0 && flags & O_STAT == 0 { - return Err(Error::new(EINVAL)); - } - if flags & O_STAT == 0 { - if is_reserved(LogicalCpuId::new(cpu_id.into()), irq_to_vector(irq_number)) { - return Err(Error::new(EEXIST)); + Handle::Irq { + ack: AtomicUsize::new(0), + irq: irq_number, } - set_reserved(LogicalCpuId::new(cpu_id.into()), irq_to_vector(irq_number), true); - } - Handle::Irq { ack: AtomicUsize::new(0), irq: irq_number } - } else { - return Err(Error::new(ENOENT)); - }) + } else if irq_number < TOTAL_IRQ_COUNT { + if flags & O_CREAT == 0 && flags & O_STAT == 0 { + return Err(Error::new(EINVAL)); + } + if flags & O_STAT == 0 { + if is_reserved(LogicalCpuId::new(cpu_id.into()), irq_to_vector(irq_number)) { + return Err(Error::new(EEXIST)); + } + set_reserved( + LogicalCpuId::new(cpu_id.into()), + irq_to_vector(irq_number), + true, + ); + } + Handle::Irq { + ack: AtomicUsize::new(0), + irq: irq_number, + } + } else { + return Err(Error::new(ENOENT)); + }, + ) } } @@ -135,12 +152,16 @@ const fn vector_to_irq(vector: u8) -> u8 { impl crate::scheme::KernelScheme for IrqScheme { fn kopen(&self, path: &str, flags: usize, ctx: CallerCtx) -> Result { - if ctx.uid != 0 { return Err(Error::new(EACCES)) } + if ctx.uid != 0 { + return Err(Error::new(EACCES)); + } let path_str = path.trim_start_matches('/'); let handle: Handle = if path_str.is_empty() { - if flags & O_DIRECTORY == 0 && flags & O_STAT == 0 { return Err(Error::new(EISDIR)) } + if flags & O_DIRECTORY == 0 && flags & O_STAT == 0 { + return Err(Error::new(EISDIR)); + } // list every logical CPU in the format of e.g. `cpu-1b` @@ -192,7 +213,10 @@ impl crate::scheme::KernelScheme for IrqScheme { } } else if let Ok(plain_irq_number) = u8::from_str(path_str) { if plain_irq_number < BASE_IRQ_COUNT { - Handle::Irq { ack: AtomicUsize::new(0), irq: plain_irq_number } + Handle::Irq { + ack: AtomicUsize::new(0), + irq: plain_irq_number, + } } else { return Err(Error::new(ENOENT)); } @@ -220,7 +244,6 @@ impl crate::scheme::KernelScheme for IrqScheme { } } - fn fcntl(&self, _id: usize, _cmd: usize, _arg: usize) -> Result { Ok(0) } @@ -229,7 +252,6 @@ impl crate::scheme::KernelScheme for IrqScheme { Ok(EventFlags::empty()) } - fn fsync(&self, _file: usize) -> Result<()> { Ok(()) } @@ -238,7 +260,10 @@ impl crate::scheme::KernelScheme for IrqScheme { let handles_guard = HANDLES.read(); let handle = handles_guard.get(&id).ok_or(Error::new(EBADF))?; - if let &Handle::Irq { irq: handle_irq, .. } = handle { + if let &Handle::Irq { + irq: handle_irq, .. + } = handle + { if handle_irq > BASE_IRQ_COUNT { set_reserved(LogicalCpuId::BSP, irq_to_vector(handle_irq), false); } @@ -250,19 +275,26 @@ impl crate::scheme::KernelScheme for IrqScheme { let handle = handles_guard.get(&file).ok_or(Error::new(EBADF))?; match handle { - &Handle::Irq { irq: handle_irq, ack: ref handle_ack } => if buffer.len() >= mem::size_of::() { - let ack = buffer.read_usize()?; - let current = COUNTS.lock()[handle_irq as usize]; + &Handle::Irq { + irq: handle_irq, + ack: ref handle_ack, + } => { + if buffer.len() >= mem::size_of::() { + let ack = buffer.read_usize()?; + let current = COUNTS.lock()[handle_irq as usize]; - if ack == current { - handle_ack.store(ack, Ordering::SeqCst); - unsafe { acknowledge(handle_irq as usize); } - Ok(mem::size_of::()) + if ack == current { + handle_ack.store(ack, Ordering::SeqCst); + unsafe { + acknowledge(handle_irq as usize); + } + Ok(mem::size_of::()) + } else { + Ok(0) + } } else { - Ok(0) + Err(Error::new(EINVAL)) } - } else { - Err(Error::new(EINVAL)) } _ => Err(Error::new(EBADF)), } @@ -273,7 +305,9 @@ impl crate::scheme::KernelScheme for IrqScheme { let handle = handles_guard.get(&id).ok_or(Error::new(EBADF))?; buf.copy_exactly(&match *handle { - Handle::Irq { irq: handle_irq, .. } => Stat { + Handle::Irq { + irq: handle_irq, .. + } => Stat { st_mode: MODE_CHR | 0o600, st_size: mem::size_of::() as u64, st_blocks: 1, @@ -318,7 +352,8 @@ impl crate::scheme::KernelScheme for IrqScheme { Handle::Bsp => format!("irq:bsp"), Handle::Avail(cpu_id, _, _) => format!("irq:cpu-{:2x}", cpu_id), Handle::TopLevel(_, _) => format!("irq:"), - }.into_bytes(); + } + .into_bytes(); buf.copy_common_bytes_from_slice(&scheme_path) } @@ -328,16 +363,21 @@ impl crate::scheme::KernelScheme for IrqScheme { match *handle { // Ensures that the length of the buffer is larger than the size of a usize - Handle::Irq { irq: handle_irq, ack: ref handle_ack } => if buffer.len() >= mem::size_of::() { - let current = COUNTS.lock()[handle_irq as usize]; - if handle_ack.load(Ordering::SeqCst) != current { - buffer.write_usize(current)?; - Ok(mem::size_of::()) + Handle::Irq { + irq: handle_irq, + ack: ref handle_ack, + } => { + if buffer.len() >= mem::size_of::() { + let current = COUNTS.lock()[handle_irq as usize]; + if handle_ack.load(Ordering::SeqCst) != current { + buffer.write_usize(current)?; + Ok(mem::size_of::()) + } else { + Ok(0) + } } else { - Ok(0) + Err(Error::new(EINVAL)) } - } else { - Err(Error::new(EINVAL)) } Handle::Bsp => { if buffer.len() < mem::size_of::() { @@ -359,5 +399,4 @@ impl crate::scheme::KernelScheme for IrqScheme { } } } - } diff --git a/src/scheme/itimer.rs b/src/scheme/itimer.rs index fca21d4845..0ac65fc3ae 100644 --- a/src/scheme/itimer.rs +++ b/src/scheme/itimer.rs @@ -1,14 +1,18 @@ use alloc::collections::BTreeMap; -use core::{mem, str}; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::{ + mem, str, + sync::atomic::{AtomicUsize, Ordering}, +}; use spin::RwLock; -use crate::syscall::data::ITimerSpec; -use crate::syscall::error::*; -use crate::syscall::flag::{CLOCK_REALTIME, CLOCK_MONOTONIC, EventFlags}; -use crate::syscall::usercopy::{UserSliceWo, UserSliceRo}; +use crate::syscall::{ + data::ITimerSpec, + error::*, + flag::{EventFlags, CLOCK_MONOTONIC, CLOCK_REALTIME}, + usercopy::{UserSliceRo, UserSliceWo}, +}; -use super::{KernelScheme, CallerCtx, OpenResult}; +use super::{CallerCtx, KernelScheme, OpenResult}; pub struct ITimerScheme; static NEXT_ID: AtomicUsize = AtomicUsize::new(1); @@ -22,7 +26,7 @@ impl KernelScheme for ITimerScheme { match clock { CLOCK_REALTIME => (), CLOCK_MONOTONIC => (), - _ => return Err(Error::new(ENOENT)) + _ => return Err(Error::new(ENOENT)), } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); @@ -35,9 +39,12 @@ impl KernelScheme for ITimerScheme { Ok(0) } - fn fevent(&self, id: usize, _flags: EventFlags) -> Result { + fn fevent(&self, id: usize, _flags: EventFlags) -> Result { let handles = HANDLES.read(); - handles.get(&id).ok_or(Error::new(EBADF)).and(Ok(EventFlags::empty())) + handles + .get(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(EventFlags::empty())) } fn fsync(&self, id: usize) -> Result<()> { @@ -45,7 +52,11 @@ impl KernelScheme for ITimerScheme { } fn close(&self, id: usize) -> Result<()> { - HANDLES.write().remove(&id).ok_or(Error::new(EBADF)).and(Ok(())) + HANDLES + .write() + .remove(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(())) } fn kread(&self, id: usize, buf: UserSliceWo) -> Result { let _clock = { @@ -89,5 +100,4 @@ impl KernelScheme for ITimerScheme { buf.copy_common_bytes_from_slice(format!("time:{}", clock).as_bytes()) } - } diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs index 480c1cb715..0febd9e59a 100644 --- a/src/scheme/memory.rs +++ b/src/scheme/memory.rs @@ -1,22 +1,25 @@ use core::num::NonZeroUsize; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use rmm::PhysicalAddress; -use alloc::vec::Vec; use spin::RwLock; -use crate::memory::{free_frames, used_frames, PAGE_SIZE, Frame}; -use crate::context::memory::{AddrSpace, Grant, PageSpan, handle_notify_files}; -use crate::paging::VirtualAddress; +use crate::{ + context::memory::{handle_notify_files, AddrSpace, Grant, PageSpan}, + memory::{free_frames, used_frames, Frame, PAGE_SIZE}, + paging::VirtualAddress, +}; use crate::paging::entry::EntryFlags; -use crate::syscall::data::{Map, StatVfs}; -use crate::syscall::flag::MapFlags; -use crate::syscall::error::*; -use crate::syscall::usercopy::UserSliceWo; +use crate::syscall::{ + data::{Map, StatVfs}, + error::*, + flag::MapFlags, + usercopy::UserSliceWo, +}; -use super::{KernelScheme, CallerCtx, OpenResult}; +use super::{CallerCtx, KernelScheme, OpenResult}; pub struct MemoryScheme; @@ -59,14 +62,18 @@ fn from_raw(raw: u32) -> Option<(HandleTy, MemoryType, HandleFlags)> { _ => return None, }, - HandleFlags::from_bits_truncate((raw >> 16) as u16) + HandleFlags::from_bits_truncate((raw >> 16) as u16), )) - } impl MemoryScheme { - pub fn fmap_anonymous(addr_space: &Arc>, map: &Map, is_phys_contiguous: bool) -> Result { - let span = PageSpan::validate_nonempty(VirtualAddress::new(map.address), map.size).ok_or(Error::new(EINVAL))?; + pub fn fmap_anonymous( + addr_space: &Arc>, + map: &Map, + is_phys_contiguous: bool, + ) -> Result { + let span = PageSpan::validate_nonempty(VirtualAddress::new(map.address), map.size) + .ok_or(Error::new(EINVAL))?; let page_count = NonZeroUsize::new(span.count).ok_or(Error::new(EINVAL))?; let mut notify_files = Vec::new(); @@ -76,65 +83,93 @@ impl MemoryScheme { return Err(Error::new(EOPNOTSUPP)); } - let page = addr_space - .write() - .mmap((map.address != 0).then_some(span.base), page_count, map.flags, &mut notify_files, |dst_page, flags, mapper, flusher| { + let page = addr_space.write().mmap( + (map.address != 0).then_some(span.base), + page_count, + map.flags, + &mut notify_files, + |dst_page, flags, mapper, flusher| { let span = PageSpan::new(dst_page, page_count.get()); if is_phys_contiguous { Ok(Grant::zeroed_phys_contiguous(span, flags, mapper, flusher)?) } else { - Ok(Grant::zeroed(span, flags, mapper, flusher, map.flags.contains(MapFlags::MAP_SHARED))?) + Ok(Grant::zeroed( + span, + flags, + mapper, + flusher, + map.flags.contains(MapFlags::MAP_SHARED), + )?) } - })?; + }, + )?; handle_notify_files(notify_files); Ok(page.start_address().data()) } - pub fn physmap(physical_address: usize, size: usize, flags: MapFlags, memory_type: MemoryType) -> Result { + pub fn physmap( + physical_address: usize, + size: usize, + flags: MapFlags, + memory_type: MemoryType, + ) -> Result { // TODO: Check physical_address against the real MAXPHYADDR. let end = 1 << 52; - if (physical_address.saturating_add(size) as u64) > end || physical_address % PAGE_SIZE != 0 { + if (physical_address.saturating_add(size) as u64) > end || physical_address % PAGE_SIZE != 0 + { return Err(Error::new(EINVAL)); } if size % PAGE_SIZE != 0 { - log::warn!("physmap size {} is not multiple of PAGE_SIZE {}", size, PAGE_SIZE); + log::warn!( + "physmap size {} is not multiple of PAGE_SIZE {}", + size, + PAGE_SIZE + ); return Err(Error::new(EINVAL)); } let page_count = NonZeroUsize::new(size.div_ceil(PAGE_SIZE)).ok_or(Error::new(EINVAL))?; - AddrSpace::current()?.write().mmap_anywhere(page_count, flags, |dst_page, mut page_flags, dst_mapper, dst_flusher| { - match memory_type { - // Default - MemoryType::Writeback => (), + AddrSpace::current()? + .write() + .mmap_anywhere( + page_count, + flags, + |dst_page, mut page_flags, dst_mapper, dst_flusher| { + match memory_type { + // Default + MemoryType::Writeback => (), - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] // TODO: AARCH64 - MemoryType::WriteCombining => page_flags = page_flags.custom_flag(EntryFlags::HUGE_PAGE.bits(), true), + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] // TODO: AARCH64 + MemoryType::WriteCombining => { + page_flags = page_flags.custom_flag(EntryFlags::HUGE_PAGE.bits(), true) + } - MemoryType::Uncacheable => page_flags = page_flags.custom_flag(EntryFlags::NO_CACHE.bits(), true), + MemoryType::Uncacheable => { + page_flags = page_flags.custom_flag(EntryFlags::NO_CACHE.bits(), true) + } - // MemoryType::DeviceMemory doesn't exist on x86 && x86_64, which instead support - // uncacheable, write-combining, write-through, write-protect, and write-back. + // MemoryType::DeviceMemory doesn't exist on x86 && x86_64, which instead support + // uncacheable, write-combining, write-through, write-protect, and write-back. + #[cfg(target_arch = "aarch64")] + MemoryType::DeviceMemory => { + page_flags = page_flags.custom_flag(EntryFlags::DEV_MEM.bits(), true) + } - #[cfg(target_arch = "aarch64")] - MemoryType::DeviceMemory => page_flags = page_flags.custom_flag(EntryFlags::DEV_MEM.bits(), true), + _ => (), + } - _ => (), - } - - Grant::physmap( - Frame::containing_address(PhysicalAddress::new(physical_address)), - PageSpan::new( - dst_page, - page_count.get(), - ), - page_flags, - dst_mapper, - dst_flusher, + Grant::physmap( + Frame::containing_address(PhysicalAddress::new(physical_address)), + PageSpan::new(dst_page, page_count.get()), + page_flags, + dst_mapper, + dst_flusher, + ) + }, ) - }).map(|page| page.start_address().data()) - + .map(|page| page.start_address().data()) } } impl KernelScheme for MemoryScheme { @@ -162,19 +197,31 @@ impl KernelScheme for MemoryScheme { _ => return Err(Error::new(ENOENT)), }; - let flags = type_str.split(',').filter_map(|ty_str| match ty_str { - //"32" => HandleFlags::BELOW_4G, - "phys_contiguous" => Some(Some(HandleFlags::PHYS_CONTIGUOUS)), - "" => None, - _ => Some(None), - }).collect::>().ok_or(Error::new(ENOENT))?; + let flags = type_str + .split(',') + .filter_map(|ty_str| match ty_str { + //"32" => HandleFlags::BELOW_4G, + "phys_contiguous" => Some(Some(HandleFlags::PHYS_CONTIGUOUS)), + "" => None, + _ => Some(None), + }) + .collect::>() + .ok_or(Error::new(ENOENT))?; // TODO: Support arches with other default memory types? - if ctx.uid != 0 && (!flags.is_empty() || !matches!((handle_ty, mem_ty), (HandleTy::Allocated, MemoryType::Writeback))) { + if ctx.uid != 0 + && (!flags.is_empty() + || !matches!( + (handle_ty, mem_ty), + (HandleTy::Allocated, MemoryType::Writeback) + )) + { return Err(Error::new(EACCES)); } - Ok(OpenResult::SchemeLocal((handle_ty as usize) | ((mem_ty as usize) << 8) | (usize::from(flags.bits()) << 16))) + Ok(OpenResult::SchemeLocal( + (handle_ty as usize) | ((mem_ty as usize) << 8) | (usize::from(flags.bits()) << 16), + )) } fn fcntl(&self, _id: usize, _cmd: usize, _arg: usize) -> Result { @@ -184,11 +231,24 @@ impl KernelScheme for MemoryScheme { fn close(&self, _id: usize) -> Result<()> { Ok(()) } - fn kfmap(&self, id: usize, addr_space: &Arc>, map: &Map, _consume: bool) -> Result { - let (handle_ty, mem_ty, flags) = u32::try_from(id).ok().and_then(from_raw).ok_or(Error::new(EBADF))?; + fn kfmap( + &self, + id: usize, + addr_space: &Arc>, + map: &Map, + _consume: bool, + ) -> Result { + let (handle_ty, mem_ty, flags) = u32::try_from(id) + .ok() + .and_then(from_raw) + .ok_or(Error::new(EBADF))?; match handle_ty { - HandleTy::Allocated => Self::fmap_anonymous(addr_space, map, flags.contains(HandleFlags::PHYS_CONTIGUOUS)), + HandleTy::Allocated => Self::fmap_anonymous( + addr_space, + map, + flags.contains(HandleFlags::PHYS_CONTIGUOUS), + ), HandleTy::PhysBorrow => Self::physmap(map.offset, map.size, map.flags, mem_ty), } } diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index 68f06a69b4..181d25b8eb 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -6,40 +6,30 @@ //! The kernel validates paths and file descriptors before they are passed to schemes, //! also stripping the scheme identifier of paths if necessary. -use alloc::{ - boxed::Box, - collections::BTreeMap, - string::ToString, - sync::Arc, - vec::Vec, -}; -use hashbrown::HashMap; -use syscall::{MunmapFlags, SendFdFlags, EventFlags, SEEK_SET, SEEK_CUR, SEEK_END}; +use alloc::{boxed::Box, collections::BTreeMap, string::ToString, sync::Arc, vec::Vec}; use core::sync::atomic::AtomicUsize; +use hashbrown::HashMap; use spin::{Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use syscall::{EventFlags, MunmapFlags, SendFdFlags, SEEK_CUR, SEEK_END, SEEK_SET}; -use crate::context::file::FileDescription; -use crate::context::memory::AddrSpace; -use crate::syscall::error::*; -use crate::syscall::usercopy::{UserSliceRo, UserSliceWo}; +use crate::{ + context::{file::FileDescription, memory::AddrSpace}, + syscall::{ + error::*, + usercopy::{UserSliceRo, UserSliceWo}, + }, +}; #[cfg(all(feature = "acpi", any(target_arch = "x86", target_arch = "x86_64")))] use self::acpi::AcpiScheme; #[cfg(all(any(target_arch = "aarch64")))] use self::dtb::DtbScheme; -use self::debug::DebugScheme; -use self::event::EventScheme; -use self::irq::IrqScheme; -use self::itimer::ITimerScheme; -use self::memory::MemoryScheme; -use self::pipe::PipeScheme; -use self::proc::ProcScheme; -use self::root::RootScheme; -use self::serio::SerioScheme; -use self::sys::SysScheme; -use self::time::TimeScheme; -use self::user::UserScheme; +use self::{ + debug::DebugScheme, event::EventScheme, irq::IrqScheme, itimer::ITimerScheme, + memory::MemoryScheme, pipe::PipeScheme, proc::ProcScheme, root::RootScheme, serio::SerioScheme, + sys::SysScheme, time::TimeScheme, user::UserScheme, +}; /// When compiled with the "acpi" feature - `acpi:` - allows drivers to read a limited set of ACPI tables. #[cfg(all(feature = "acpi", any(target_arch = "x86", target_arch = "x86_64")))] @@ -96,7 +86,7 @@ int_like!(SchemeId, usize); int_like!(FileHandle, AtomicFileHandle, usize, AtomicUsize); pub struct SchemeIter<'a> { - inner: Option, SchemeId>> + inner: Option, SchemeId>>, } impl<'a> Iterator for SchemeIter<'a> { @@ -127,14 +117,27 @@ impl SchemeList { let mut insert_globals = |globals: &[GlobalSchemes]| { for &g in globals { - list.map.insert(SchemeId::from(g as usize), KernelSchemes::Global(g)); + list.map + .insert(SchemeId::from(g as usize), KernelSchemes::Global(g)); } }; // TODO: impl TryFrom and bypass map for global schemes? { use GlobalSchemes::*; - insert_globals(&[Debug, Event, Memory, Pipe, Serio, Irq, Time, ITimer, Sys, ProcFull, ProcRestricted]); + insert_globals(&[ + Debug, + Event, + Memory, + Pipe, + Serio, + Irq, + Time, + ITimer, + Sys, + ProcFull, + ProcRestricted, + ]); #[cfg(all(feature = "acpi", any(target_arch = "x86", target_arch = "x86_64")))] insert_globals(&[Acpi]); @@ -155,8 +158,10 @@ impl SchemeList { //TODO: Only memory: is in the null namespace right now. It should be removed when //anonymous mmap's are implemented - self.insert_global(ns, "memory", GlobalSchemes::Memory).unwrap(); - self.insert_global(ns, "thisproc", GlobalSchemes::ProcRestricted).unwrap(); + self.insert_global(ns, "memory", GlobalSchemes::Memory) + .unwrap(); + self.insert_global(ns, "thisproc", GlobalSchemes::ProcRestricted) + .unwrap(); self.insert_global(ns, "pipe", GlobalSchemes::Pipe).unwrap(); } @@ -166,10 +171,16 @@ impl SchemeList { self.next_ns += 1; self.names.insert(ns, HashMap::new()); - self.insert(ns, "", |scheme_id| KernelSchemes::Root(Arc::new(RootScheme::new(ns, scheme_id)))).unwrap(); - self.insert_global(ns, "event", GlobalSchemes::Event).unwrap(); - self.insert_global(ns, "itimer", GlobalSchemes::ITimer).unwrap(); - self.insert_global(ns, "memory", GlobalSchemes::Memory).unwrap(); + self.insert(ns, "", |scheme_id| { + KernelSchemes::Root(Arc::new(RootScheme::new(ns, scheme_id))) + }) + .unwrap(); + self.insert_global(ns, "event", GlobalSchemes::Event) + .unwrap(); + self.insert_global(ns, "itimer", GlobalSchemes::ITimer) + .unwrap(); + self.insert_global(ns, "memory", GlobalSchemes::Memory) + .unwrap(); self.insert_global(ns, "pipe", GlobalSchemes::Pipe).unwrap(); self.insert_global(ns, "sys", GlobalSchemes::Sys).unwrap(); self.insert_global(ns, "time", GlobalSchemes::Time).unwrap(); @@ -185,20 +196,30 @@ impl SchemeList { // These schemes should only be available on the root #[cfg(all(any(target_arch = "aarch64")))] { - self.insert_global(ns, "kernel.dtb", GlobalSchemes::Dtb).unwrap(); + self.insert_global(ns, "kernel.dtb", GlobalSchemes::Dtb) + .unwrap(); } #[cfg(all(feature = "acpi", any(target_arch = "x86", target_arch = "x86_64")))] { - self.insert_global(ns, "kernel.acpi", GlobalSchemes::Acpi).unwrap(); + self.insert_global(ns, "kernel.acpi", GlobalSchemes::Acpi) + .unwrap(); } - self.insert_global(ns, "debug", GlobalSchemes::Debug).unwrap(); + self.insert_global(ns, "debug", GlobalSchemes::Debug) + .unwrap(); self.insert_global(ns, "irq", GlobalSchemes::Irq).unwrap(); - self.insert_global(ns, "proc", GlobalSchemes::ProcFull).unwrap(); - self.insert_global(ns, "thisproc", GlobalSchemes::ProcRestricted).unwrap(); - self.insert_global(ns, "serio", GlobalSchemes::Serio).unwrap(); + self.insert_global(ns, "proc", GlobalSchemes::ProcFull) + .unwrap(); + self.insert_global(ns, "thisproc", GlobalSchemes::ProcRestricted) + .unwrap(); + self.insert_global(ns, "serio", GlobalSchemes::Serio) + .unwrap(); } - pub fn make_ns(&mut self, from: SchemeNamespace, names: impl IntoIterator>) -> Result { + pub fn make_ns( + &mut self, + from: SchemeNamespace, + names: impl IntoIterator>, + ) -> Result { // Create an empty namespace let to = self.new_ns(); @@ -209,7 +230,10 @@ impl SchemeList { }; if let Some(ref mut names) = self.names.get_mut(&to) { - if names.insert(name.to_string().into_boxed_str(), id).is_some() { + if names + .insert(name.to_string().into_boxed_str(), id) + .is_some() + { return Err(Error::new(EEXIST)); } } else { @@ -222,7 +246,7 @@ impl SchemeList { pub fn iter_name(&self, ns: SchemeNamespace) -> SchemeIter { SchemeIter { - inner: self.names.get(&ns).map(|names| names.iter()) + inner: self.names.get(&ns).map(|names| names.iter()), } } @@ -240,8 +264,17 @@ impl SchemeList { None } - pub fn insert_global(&mut self, ns: SchemeNamespace, name: &str, global: GlobalSchemes) -> Result<()> { - let prev = self.names.get_mut(&ns).ok_or(Error::new(ENODEV))?.insert(name.into(), global.scheme_id()); + pub fn insert_global( + &mut self, + ns: SchemeNamespace, + name: &str, + global: GlobalSchemes, + ) -> Result<()> { + let prev = self + .names + .get_mut(&ns) + .ok_or(Error::new(ENODEV))? + .insert(name.into(), global.scheme_id()); if prev.is_some() { return Err(Error::new(EEXIST)); @@ -251,11 +284,22 @@ impl SchemeList { } /// Create a new scheme. - pub fn insert(&mut self, ns: SchemeNamespace, name: &str, scheme_fn: impl FnOnce(SchemeId) -> KernelSchemes) -> Result { - self.insert_and_pass(ns, name, |id| (scheme_fn(id), ())).map(|(id, ())| id) + pub fn insert( + &mut self, + ns: SchemeNamespace, + name: &str, + scheme_fn: impl FnOnce(SchemeId) -> KernelSchemes, + ) -> Result { + self.insert_and_pass(ns, name, |id| (scheme_fn(id), ())) + .map(|(id, ())| id) } - pub fn insert_and_pass(&mut self, ns: SchemeNamespace, name: &str, scheme_fn: impl FnOnce(SchemeId) -> (KernelSchemes, T)) -> Result<(SchemeId, T)> { + pub fn insert_and_pass( + &mut self, + ns: SchemeNamespace, + name: &str, + scheme_fn: impl FnOnce(SchemeId) -> (KernelSchemes, T), + ) -> Result<(SchemeId, T)> { if let Some(names) = self.names.get(&ns) { if names.contains_key(name) { return Err(Error::new(EEXIST)); @@ -283,7 +327,9 @@ impl SchemeList { assert!(self.map.insert(id, new_scheme).is_none()); if let Some(ref mut names) = self.names.get_mut(&ns) { - assert!(names.insert(name.to_string().into_boxed_str(), id).is_none()); + assert!(names + .insert(name.to_string().into_boxed_str(), id) + .is_none()); } else { // Nonexistent namespace, posssibly null namespace return Err(Error::new(ENODEV)); @@ -332,7 +378,13 @@ pub trait KernelScheme: Send + Sync + 'static { Err(Error::new(ENOENT)) } - fn kfmap(&self, number: usize, addr_space: &Arc>, map: &crate::syscall::data::Map, consume: bool) -> Result { + fn kfmap( + &self, + number: usize, + addr_space: &Arc>, + map: &crate::syscall::data::Map, + consume: bool, + ) -> Result { Err(Error::new(EOPNOTSUPP)) } fn kfunmap(&self, number: usize, offset: usize, size: usize, flags: MunmapFlags) -> Result<()> { @@ -361,7 +413,13 @@ pub trait KernelScheme: Send + Sync + 'static { Err(Error::new(EBADF)) } - fn ksendfd(&self, id: usize, desc: Arc>, flags: SendFdFlags, arg: u64) -> Result { + fn ksendfd( + &self, + id: usize, + desc: Arc>, + flags: SendFdFlags, + arg: u64, + ) -> Result { Err(Error::new(EOPNOTSUPP)) } @@ -411,10 +469,17 @@ pub struct CallerCtx { pub gid: u32, } -pub fn calc_seek_offset(cur_pos: usize, rel_pos: isize, whence: usize, len: usize) -> Result { +pub fn calc_seek_offset( + cur_pos: usize, + rel_pos: isize, + whence: usize, + len: usize, +) -> Result { match whence { SEEK_SET => usize::try_from(rel_pos).map_err(|_| Error::new(EINVAL)), - SEEK_CUR => cur_pos.checked_add_signed(rel_pos).ok_or(Error::new(EOVERFLOW)), + SEEK_CUR => cur_pos + .checked_add_signed(rel_pos) + .ok_or(Error::new(EOVERFLOW)), SEEK_END => len.checked_add_signed(rel_pos).ok_or(Error::new(EOVERFLOW)), _ => return Err(Error::new(EINVAL)), diff --git a/src/scheme/pipe.rs b/src/scheme/pipe.rs index 1938541a85..628af1d7da 100644 --- a/src/scheme/pipe.rs +++ b/src/scheme/pipe.rs @@ -1,18 +1,26 @@ -use core::sync::atomic::{AtomicUsize, Ordering, AtomicBool}; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use alloc::sync::Arc; -use alloc::collections::{BTreeMap, VecDeque}; +use alloc::{ + collections::{BTreeMap, VecDeque}, + sync::Arc, +}; use spin::{Mutex, RwLock}; -use crate::event; -use crate::sync::WaitCondition; -use crate::syscall::error::{Error, Result, EAGAIN, EBADF, EINTR, EINVAL, ENOENT, EPIPE, ESPIPE}; -use crate::syscall::flag::{EventFlags, EVENT_READ, EVENT_WRITE, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK, MODE_FIFO}; -use crate::syscall::data::Stat; -use crate::syscall::usercopy::{UserSliceWo, UserSliceRo}; +use crate::{ + event, + sync::WaitCondition, + syscall::{ + data::Stat, + error::{Error, Result, EAGAIN, EBADF, EINTR, EINVAL, ENOENT, EPIPE, ESPIPE}, + flag::{ + EventFlags, EVENT_READ, EVENT_WRITE, F_GETFL, F_SETFL, MODE_FIFO, O_ACCMODE, O_NONBLOCK, + }, + usercopy::{UserSliceRo, UserSliceWo}, + }, +}; -use super::{KernelScheme, OpenResult, CallerCtx, GlobalSchemes}; +use super::{CallerCtx, GlobalSchemes, KernelScheme, OpenResult}; // TODO: Preallocate a number of scheme IDs, since there can only be *one* root namespace, and // therefore only *one* pipe scheme. @@ -35,16 +43,19 @@ fn from_raw_id(id: usize) -> (bool, usize) { pub fn pipe(flags: usize) -> Result<(usize, usize)> { let id = PIPE_NEXT_ID.fetch_add(1, Ordering::Relaxed); - PIPES.write().insert(id, Arc::new(Pipe { - read_flags: AtomicUsize::new(flags), - write_flags: AtomicUsize::new(flags), - queue: Mutex::new(VecDeque::new()), - read_condition: WaitCondition::new(), - write_condition: WaitCondition::new(), - writer_is_alive: AtomicBool::new(true), - reader_is_alive: AtomicBool::new(true), - has_run_dup: AtomicBool::new(false), - })); + PIPES.write().insert( + id, + Arc::new(Pipe { + read_flags: AtomicUsize::new(flags), + write_flags: AtomicUsize::new(flags), + queue: Mutex::new(VecDeque::new()), + read_condition: WaitCondition::new(), + write_condition: WaitCondition::new(), + writer_is_alive: AtomicBool::new(true), + reader_is_alive: AtomicBool::new(true), + has_run_dup: AtomicBool::new(false), + }), + ); Ok((id, id | WRITE_NOT_READ_BIT)) } @@ -56,15 +67,19 @@ impl KernelScheme for PipeScheme { let (is_writer_not_reader, key) = from_raw_id(id); let pipe = Arc::clone(PIPES.read().get(&key).ok_or(Error::new(EBADF))?); - let flags = if is_writer_not_reader { &pipe.write_flags } else { &pipe.read_flags }; + let flags = if is_writer_not_reader { + &pipe.write_flags + } else { + &pipe.read_flags + }; match cmd { F_GETFL => Ok(flags.load(Ordering::SeqCst)), F_SETFL => { flags.store(arg & !O_ACCMODE, Ordering::SeqCst); Ok(0) - }, - _ => Err(Error::new(EINVAL)) + } + _ => Err(Error::new(EINVAL)), } } @@ -172,17 +187,26 @@ impl KernelScheme for PipeScheme { let (s1, s2) = vec.as_slices(); let s1_count = core::cmp::min(user_buf.len(), s1.len()); - let (s1_dst, s2_buf) = user_buf.split_at(s1_count).expect("s1_count <= user_buf.len()"); + let (s1_dst, s2_buf) = user_buf + .split_at(s1_count) + .expect("s1_count <= user_buf.len()"); s1_dst.copy_from_slice(&s1[..s1_count])?; let s2_count = core::cmp::min(s2_buf.len(), s2.len()); - s2_buf.limit(s2_count).expect("s2_count <= s2_buf.len()").copy_from_slice(&s2[..s2_count])?; + s2_buf + .limit(s2_count) + .expect("s2_count <= s2_buf.len()") + .copy_from_slice(&s2[..s2_count])?; let bytes_read = s1_count + s2_count; let _ = vec.drain(..bytes_read); if bytes_read > 0 { - event::trigger(GlobalSchemes::Pipe.scheme_id(), key | WRITE_NOT_READ_BIT, EVENT_WRITE); + event::trigger( + GlobalSchemes::Pipe.scheme_id(), + key | WRITE_NOT_READ_BIT, + EVENT_WRITE, + ); pipe.write_condition.notify(); return Ok(bytes_read); @@ -212,7 +236,9 @@ impl KernelScheme for PipeScheme { let bytes_left = MAX_QUEUE_SIZE.saturating_sub(vec.len()); let bytes_to_write = core::cmp::min(bytes_left, user_buf.len()); - let src_buf = user_buf.limit(bytes_to_write).expect("bytes_to_write <= user_buf.len()"); + let src_buf = user_buf + .limit(bytes_to_write) + .expect("bytes_to_write <= user_buf.len()"); const TMPBUF_SIZE: usize = 512; let mut tmp_buf = [0_u8; TMPBUF_SIZE]; @@ -259,9 +285,9 @@ impl KernelScheme for PipeScheme { } pub struct Pipe { - read_flags: AtomicUsize, // fcntl read flags - write_flags: AtomicUsize, // fcntl write flags - read_condition: WaitCondition, // signals whether there are available bytes to read + read_flags: AtomicUsize, // fcntl read flags + write_flags: AtomicUsize, // fcntl write flags + read_condition: WaitCondition, // signals whether there are available bytes to read write_condition: WaitCondition, // signals whether there is room for additional bytes queue: Mutex>, reader_is_alive: AtomicBool, // starts set, unset when reader closes diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 096154d72f..27063ba9eb 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -1,18 +1,23 @@ use crate::{ arch::paging::{Page, RmmA, RmmArch, VirtualAddress}, - context::{self, Context, ContextId, Status, file::FileDescriptor, memory::{AddrSpace, Grant, new_addrspace, PageSpan, handle_notify_files}}, + context::{ + self, + file::FileDescriptor, + memory::{handle_notify_files, new_addrspace, AddrSpace, Grant, PageSpan}, + Context, ContextId, Status, + }, memory::PAGE_SIZE, ptrace, scheme::{self, FileHandle, KernelScheme}, syscall::{ - FloatRegisters, - IntRegisters, - EnvRegisters, + self, data::{GrantDesc, Map, PtraceEvent, SigAction, Stat}, error::*, flag::*, - self, usercopy::{UserSliceWo, UserSliceRo}, - }, LogicalCpuId, LogicalCpuSet, + usercopy::{UserSliceRo, UserSliceWo}, + EnvRegisters, FloatRegisters, IntRegisters, + }, + LogicalCpuId, LogicalCpuSet, }; use alloc::{ @@ -24,18 +29,20 @@ use alloc::{ }; use core::{ mem, - slice, - str, - sync::atomic::{AtomicUsize, Ordering}, num::NonZeroUsize, + num::NonZeroUsize, + slice, str, + sync::atomic::{AtomicUsize, Ordering}, }; use spin::RwLock; -use super::{OpenResult, CallerCtx, KernelSchemes, GlobalSchemes}; +use super::{CallerCtx, GlobalSchemes, KernelSchemes, OpenResult}; fn read_from(dst: UserSliceWo, src: &[u8], offset: &mut usize) -> Result { let avail_src = src.get(*offset..).unwrap_or(&[]); let bytes_copied = dst.copy_common_bytes_from_slice(avail_src)?; - *offset = offset.checked_add(bytes_copied).ok_or(Error::new(EOVERFLOW))?; + *offset = offset + .checked_add(bytes_copied) + .ok_or(Error::new(EOVERFLOW))?; Ok(bytes_copied) } @@ -80,15 +87,18 @@ where // Wait until stopped while running { - unsafe { context::switch(); } + unsafe { + context::switch(); + } - running = with_context(pid, |context| { - Ok(context.running) - })?; + running = with_context(pid, |context| Ok(context.running))?; } with_context_mut(pid, |context| { - assert!(!context.running, "process can't have been restarted, we stopped it!"); + assert!( + !context.running, + "process can't have been restarted, we stopped it!" + ); let ret = callback(context); @@ -113,8 +123,12 @@ enum Operation { SessionId, Sigstack, Attr(Attr), - Filetable { filetable: Arc>>> }, - AddrSpace { addrspace: Arc> }, + Filetable { + filetable: Arc>>>, + }, + AddrSpace { + addrspace: Arc>, + }, CurrentAddrSpace, // "operations CAN change". The reason we split changing the address space into two handle @@ -149,7 +163,19 @@ enum Attr { } impl Operation { fn needs_child_process(&self) -> bool { - matches!(self, Self::Regs(_) | Self::Trace | Self::SessionId | Self::Filetable { .. } | Self::AddrSpace { .. } | Self::CurrentAddrSpace | Self::CurrentFiletable | Self::Sigactions(_) | Self::CurrentSigactions | Self::AwaitingSigactionsChange(_)) + matches!( + self, + Self::Regs(_) + | Self::Trace + | Self::SessionId + | Self::Filetable { .. } + | Self::AddrSpace { .. } + | Self::CurrentAddrSpace + | Self::CurrentFiletable + | Self::Sigactions(_) + | Self::CurrentSigactions + | Self::AwaitingSigactionsChange(_) + ) } fn needs_root(&self) -> bool { matches!(self, Self::Attr(_)) @@ -165,10 +191,7 @@ struct StaticData { } impl StaticData { fn new(buf: Box<[u8]>) -> Self { - Self { - buf, - offset: 0, - } + Self { buf, offset: 0 } } } enum OperationData { @@ -243,14 +266,33 @@ fn new_handle(handle: Handle) -> Result { } fn get_context(id: ContextId) -> Result>> { - context::contexts().get(id).ok_or(Error::new(ENOENT)).map(Arc::clone) + context::contexts() + .get(id) + .ok_or(Error::new(ENOENT)) + .map(Arc::clone) } impl ProcScheme { - fn open_inner(&self, pid: ContextId, operation_str: Option<&str>, flags: usize, uid: u32, gid: u32) -> Result { + fn open_inner( + &self, + pid: ContextId, + operation_str: Option<&str>, + flags: usize, + uid: u32, + gid: u32, + ) -> Result { let operation = match operation_str { - Some("addrspace") => Operation::AddrSpace { addrspace: Arc::clone(get_context(pid)?.read().addr_space().map_err(|_| Error::new(ENOENT))?) }, - Some("filetable") => Operation::Filetable { filetable: Arc::clone(&get_context(pid)?.read().files) }, + Some("addrspace") => Operation::AddrSpace { + addrspace: Arc::clone( + get_context(pid)? + .read() + .addr_space() + .map_err(|_| Error::new(ENOENT))?, + ), + }, + Some("filetable") => Operation::Filetable { + filetable: Arc::clone(&get_context(pid)?.read().files), + }, Some("current-addrspace") => Operation::CurrentAddrSpace, Some("current-filetable") => Operation::CurrentFiletable, Some("regs/float") => Operation::Regs(RegsKind::Float), @@ -264,11 +306,18 @@ impl ProcScheme { Some("uid") => Operation::Attr(Attr::Uid), Some("gid") => Operation::Attr(Attr::Gid), Some("open_via_dup") => Operation::OpenViaDup, - Some("sigactions") => Operation::Sigactions(Arc::clone(&get_context(pid)?.read().actions)), + Some("sigactions") => { + Operation::Sigactions(Arc::clone(&get_context(pid)?.read().actions)) + } Some("current-sigactions") => Operation::CurrentSigactions, - Some("mmap-min-addr") => Operation::MmapMinAddr(Arc::clone(get_context(pid)?.read().addr_space().map_err(|_| Error::new(ENOENT))?)), + Some("mmap-min-addr") => Operation::MmapMinAddr(Arc::clone( + get_context(pid)? + .read() + .addr_space() + .map_err(|_| Error::new(ENOENT))?, + )), Some("sched-affinity") => Operation::SchedAffinity, - _ => return Err(Error::new(EINVAL)) + _ => return Err(Error::new(EINVAL)), }; let contexts = context::contexts(); @@ -282,7 +331,7 @@ impl ProcScheme { data = match operation { Operation::Trace => OperationData::Trace(TraceData::default()), Operation::Static(_) => OperationData::Static(StaticData::new( - target.name.clone().into_owned().into_bytes().into() + target.name.clone().into_owned().into_bytes().into(), )), Operation::AddrSpace { .. } => OperationData::Offset(0), _ => OperationData::Other, @@ -306,13 +355,16 @@ impl ProcScheme { // Is it a subprocess of us? In the future, a capability could // bypass this check. - match contexts.ancestors(target.ppid).find(|&(id, _context)| id == current.id) { + match contexts + .ancestors(target.ppid) + .find(|&(id, _context)| id == current.id) + { Some((id, context)) => { // Paranoid sanity check, as ptrace security holes // wouldn't be fun assert_eq!(id, current.id); assert_eq!(id, context.read().id); - }, + } None => return Err(Error::new(EPERM)), } } @@ -325,7 +377,13 @@ impl ProcScheme { use core::fmt::Write; let mut data = String::new(); - for index in target.files.read().iter().enumerate().filter_map(|(idx, val)| val.as_ref().map(|_| idx)) { + for index in target + .files + .read() + .iter() + .enumerate() + .filter_map(|(idx, val)| val.as_ref().map(|_| idx)) + { writeln!(data, "{}", index).unwrap(); } data.into_bytes().into_boxed_slice() @@ -371,10 +429,7 @@ impl ProcScheme { } } else { try_stop_context(info.pid, |context| { - Ok(( - context.arch.tpidr_el0, - context.arch.tpidrro_el0, - )) + Ok((context.arch.tpidr_el0, context.arch.tpidrro_el0)) })? }; Ok(EnvRegisters { @@ -389,7 +444,7 @@ impl ProcScheme { unsafe { ( (&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_FS].offset() as u64, - (&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].offset() as u64 + (&*crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].offset() as u64, ) } } else { @@ -397,7 +452,10 @@ impl ProcScheme { Ok((context.arch.fsbase as u64, context.arch.gsbase as u64)) })? }; - Ok(EnvRegisters { fsbase: fsbase as _, gsbase: gsbase as _ }) + Ok(EnvRegisters { + fsbase: fsbase as _, + gsbase: gsbase as _, + }) } #[cfg(target_arch = "x86_64")] @@ -415,7 +473,10 @@ impl ProcScheme { Ok((context.arch.fsbase as u64, context.arch.gsbase as u64)) })? }; - Ok(EnvRegisters { fsbase: fsbase as _, gsbase: gsbase as _ }) + Ok(EnvRegisters { + fsbase: fsbase as _, + gsbase: gsbase as _, + }) } #[cfg(target_arch = "aarch64")] @@ -439,7 +500,9 @@ impl ProcScheme { #[cfg(target_arch = "x86")] fn write_env_regs(&self, info: &Info, regs: EnvRegisters) -> Result<()> { - if !(RmmA::virt_is_valid(VirtualAddress::new(regs.fsbase as usize)) && RmmA::virt_is_valid(VirtualAddress::new(regs.gsbase as usize))) { + if !(RmmA::virt_is_valid(VirtualAddress::new(regs.fsbase as usize)) + && RmmA::virt_is_valid(VirtualAddress::new(regs.gsbase as usize))) + { return Err(Error::new(EINVAL)); } @@ -448,7 +511,12 @@ impl ProcScheme { (&mut *crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_FS].set_offset(regs.fsbase); (&mut *crate::gdt::pcr()).gdt[crate::gdt::GDT_USER_GS].set_offset(regs.gsbase); - match context::contexts().current().ok_or(Error::new(ESRCH))?.write().arch { + match context::contexts() + .current() + .ok_or(Error::new(ESRCH))? + .write() + .arch + { ref mut arch => { arch.fsbase = regs.fsbase as usize; arch.gsbase = regs.gsbase as usize; @@ -467,7 +535,9 @@ impl ProcScheme { #[cfg(target_arch = "x86_64")] fn write_env_regs(&self, info: &Info, regs: EnvRegisters) -> Result<()> { - if !(RmmA::virt_is_valid(VirtualAddress::new(regs.fsbase as usize)) && RmmA::virt_is_valid(VirtualAddress::new(regs.gsbase as usize))) { + if !(RmmA::virt_is_valid(VirtualAddress::new(regs.fsbase as usize)) + && RmmA::virt_is_valid(VirtualAddress::new(regs.gsbase as usize))) + { return Err(Error::new(EINVAL)); } @@ -478,7 +548,12 @@ impl ProcScheme { // userspace, it will have executed SWAPGS first. x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, regs.gsbase as u64); - match context::contexts().current().ok_or(Error::new(ESRCH))?.write().arch { + match context::contexts() + .current() + .ok_or(Error::new(ESRCH))? + .write() + .arch + { ref mut arch => { arch.fsbase = regs.fsbase as usize; arch.gsbase = regs.gsbase as usize; @@ -499,8 +574,7 @@ impl ProcScheme { impl KernelScheme for ProcScheme { fn kopen(&self, path: &str, flags: usize, ctx: CallerCtx) -> Result { let mut parts = path.splitn(2, '/'); - let pid_str = parts.next() - .ok_or(Error::new(ENOENT))?; + let pid_str = parts.next().ok_or(Error::new(ENOENT))?; let pid = if pid_str == "current" { context::context_id() @@ -512,7 +586,8 @@ impl KernelScheme for ProcScheme { ContextId::new(pid_str.parse().map_err(|_| Error::new(ENOENT))?) }; - self.open_inner(pid, parts.next(), flags, ctx.uid, ctx.gid).map(OpenResult::SchemeLocal) + self.open_inner(pid, parts.next(), flags, ctx.uid, ctx.gid) + .map(OpenResult::SchemeLocal) } fn fcntl(&self, id: usize, cmd: usize, arg: usize) -> Result { @@ -520,9 +595,12 @@ impl KernelScheme for ProcScheme { let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; match cmd { - F_SETFL => { handle.info.flags = arg; Ok(0) }, + F_SETFL => { + handle.info.flags = arg; + Ok(0) + } F_GETFL => Ok(handle.info.flags), - _ => Err(Error::new(EINVAL)) + _ => Err(Error::new(EINVAL)), } } @@ -538,15 +616,22 @@ impl KernelScheme for ProcScheme { } } - fn close(&self, id: usize) -> Result<()> { let mut handle = HANDLES.write().remove(&id).ok_or(Error::new(EBADF))?; handle.continue_ignored_children(); - let stop_context = if handle.info.pid == context::context_id() { with_context_mut } else { try_stop_context }; + let stop_context = if handle.info.pid == context::context_id() { + with_context_mut + } else { + try_stop_context + }; match handle.info.operation { - Operation::AwaitingAddrSpaceChange { new, new_sp, new_ip } => { + Operation::AwaitingAddrSpaceChange { + new, + new_sp, + new_ip, + } => { let _ = stop_context(handle.info.pid, |context: &mut Context| unsafe { if let Some(saved_regs) = ptrace::regs_for_mut(context) { #[cfg(target_arch = "aarch64")] @@ -572,18 +657,27 @@ impl KernelScheme for ProcScheme { Ok(context.set_addr_space(new)) })?; - let _ = ptrace::send_event(crate::syscall::ptrace_event!(PTRACE_EVENT_ADDRSPACE_SWITCH, 0)); + let _ = ptrace::send_event(crate::syscall::ptrace_event!( + PTRACE_EVENT_ADDRSPACE_SWITCH, + 0 + )); + } + Operation::AddrSpace { addrspace } | Operation::MmapMinAddr(addrspace) => { + drop(addrspace) } - Operation::AddrSpace { addrspace } | Operation::MmapMinAddr(addrspace) => drop(addrspace), - Operation::AwaitingFiletableChange(new) => with_context_mut(handle.info.pid, |context: &mut Context| { - context.files = new; - Ok(()) - })?, - Operation::AwaitingSigactionsChange(new) => with_context_mut(handle.info.pid, |context: &mut Context| { - context.actions = new; - Ok(()) - })?, + Operation::AwaitingFiletableChange(new) => { + with_context_mut(handle.info.pid, |context: &mut Context| { + context.files = new; + Ok(()) + })? + } + Operation::AwaitingSigactionsChange(new) => { + with_context_mut(handle.info.pid, |context: &mut Context| { + context.actions = new; + Ok(()) + })? + } Operation::Trace => { ptrace::close_session(handle.info.pid); @@ -601,8 +695,19 @@ impl KernelScheme for ProcScheme { } Ok(()) } - fn kfmap(&self, id: usize, dst_addr_space: &Arc>, map: &crate::syscall::data::Map, consume: bool) -> Result { - let info = HANDLES.read().get(&id).ok_or(Error::new(EBADF))?.info.clone(); + fn kfmap( + &self, + id: usize, + dst_addr_space: &Arc>, + map: &crate::syscall::data::Map, + consume: bool, + ) -> Result { + let info = HANDLES + .read() + .get(&id) + .ok_or(Error::new(EBADF))? + .info + .clone(); match info.operation { Operation::AddrSpace { ref addrspace } => { @@ -610,8 +715,11 @@ impl KernelScheme for ProcScheme { return Err(Error::new(EBUSY)); } - let (requested_dst_page, _) = crate::syscall::validate_region(map.address, map.size)?; - let src_span = PageSpan::validate_nonempty(VirtualAddress::new(map.offset), map.size).ok_or(Error::new(EINVAL))?; + let (requested_dst_page, _) = + crate::syscall::validate_region(map.address, map.size)?; + let src_span = + PageSpan::validate_nonempty(VirtualAddress::new(map.offset), map.size) + .ok_or(Error::new(EINVAL))?; let requested_dst_base = (map.address != 0).then_some(requested_dst_page); @@ -624,9 +732,37 @@ impl KernelScheme for ProcScheme { // TODO: Validate flags let result_base = if consume { - AddrSpace::r#move(&mut *dst_addr_space, Some(&mut *src_addr_space), src_span, requested_dst_base, src_page_count.get(), map.flags, &mut notify_files)? + AddrSpace::r#move( + &mut *dst_addr_space, + Some(&mut *src_addr_space), + src_span, + requested_dst_base, + src_page_count.get(), + map.flags, + &mut notify_files, + )? } else { - dst_addr_space.mmap(requested_dst_base, src_page_count, map.flags, &mut notify_files, |dst_page, _, dst_mapper, flusher| Ok(Grant::borrow(Arc::clone(addrspace), &mut *src_addr_space, src_span.base, dst_page, src_span.count, map.flags, dst_mapper, flusher, true, true, false)?))? + dst_addr_space.mmap( + requested_dst_base, + src_page_count, + map.flags, + &mut notify_files, + |dst_page, _, dst_mapper, flusher| { + Ok(Grant::borrow( + Arc::clone(addrspace), + &mut *src_addr_space, + src_span.base, + dst_page, + src_span.count, + map.flags, + dst_mapper, + flusher, + true, + true, + false, + )?) + }, + )? }; handle_notify_files(notify_files); @@ -654,7 +790,7 @@ impl KernelScheme for ProcScheme { let len = buf.copy_common_bytes_from_slice(src_buf)?; data.offset += len; Ok(len) - }, + } Operation::Regs(kind) => { union Output { float: FloatRegisters, @@ -666,34 +802,44 @@ impl KernelScheme for ProcScheme { RegsKind::Float => with_context(info.pid, |context| { // NOTE: The kernel will never touch floats - Ok((Output { float: context.get_fx_regs() }, mem::size_of::())) + Ok(( + Output { + float: context.get_fx_regs(), + }, + mem::size_of::(), + )) })?, - RegsKind::Int => try_stop_context(info.pid, |context| match unsafe { ptrace::regs_for(context) } { - None => { - assert!(!context.running, "try_stop_context is broken, clearly"); - println!("{}:{}: Couldn't read registers from stopped process", file!(), line!()); - Err(Error::new(ENOTRECOVERABLE)) - }, - Some(stack) => { - let mut regs = IntRegisters::default(); - stack.save(&mut regs); - Ok((Output { int: regs }, mem::size_of::())) + RegsKind::Int => try_stop_context(info.pid, |context| { + match unsafe { ptrace::regs_for(context) } { + None => { + assert!(!context.running, "try_stop_context is broken, clearly"); + println!( + "{}:{}: Couldn't read registers from stopped process", + file!(), + line!() + ); + Err(Error::new(ENOTRECOVERABLE)) + } + Some(stack) => { + let mut regs = IntRegisters::default(); + stack.save(&mut regs); + Ok((Output { int: regs }, mem::size_of::())) + } } })?, - RegsKind::Env => { - ( - Output { env: self.read_env_regs(&info)? }, - mem::size_of::() - ) - } + RegsKind::Env => ( + Output { + env: self.read_env_regs(&info)?, + }, + mem::size_of::(), + ), }; - let src_buf = unsafe { - slice::from_raw_parts(&output as *const _ as *const u8, size) - }; + let src_buf = + unsafe { slice::from_raw_parts(&output as *const _ as *const u8, size) }; buf.copy_common_bytes_from_slice(src_buf) - }, + } Operation::Trace => { let mut handles = HANDLES.write(); let handle = handles.get_mut(&id).ok_or(Error::new(EBADF))?; @@ -711,7 +857,8 @@ impl KernelScheme for ProcScheme { // Read events let src_len = src_buf.len(); - let slice = &mut src_buf[..core::cmp::min(src_len, buf.len() / mem::size_of::())]; + let slice = &mut src_buf + [..core::cmp::min(src_len, buf.len() / mem::size_of::())]; let (read, reached) = ptrace::Session::with_session(info.pid, |session| { let mut data = session.data.lock(); @@ -728,11 +875,17 @@ impl KernelScheme for ProcScheme { // If there are no events, and breakpoint isn't reached, we // must not have waited. if read == 0 && !reached { - assert!(handle.info.flags & O_NONBLOCK == O_NONBLOCK, "wait woke up spuriously??"); + assert!( + handle.info.flags & O_NONBLOCK == O_NONBLOCK, + "wait woke up spuriously??" + ); return Err(Error::new(EAGAIN)); } - for (dst, src) in buf.in_exact_chunks(mem::size_of::()).zip(slice.iter()) { + for (dst, src) in buf + .in_exact_chunks(mem::size_of::()) + .zip(slice.iter()) + { dst.copy_exactly(src)?; } @@ -740,7 +893,9 @@ impl KernelScheme for ProcScheme { Ok(read * mem::size_of::()) } Operation::AddrSpace { ref addrspace } => { - let OperationData::Offset(orig_offset) = HANDLES.read().get(&id).ok_or(Error::new(EBADF))?.data else { + let OperationData::Offset(orig_offset) = + HANDLES.read().get(&id).ok_or(Error::new(EBADF))?.data + else { return Err(Error::new(EBADFD)); }; @@ -750,7 +905,10 @@ impl KernelScheme for ProcScheme { let mut dst = [GrantDesc::default(); 16]; - for (dst, (grant_base, grant_info)) in dst.iter_mut().zip(addrspace.read().grants.iter().skip(orig_offset)) { + for (dst, (grant_base, grant_info)) in dst + .iter_mut() + .zip(addrspace.read().grants.iter().skip(orig_offset)) + { *dst = GrantDesc { base: grant_base.start_address().data(), size: grant_info.page_count() * PAGE_SIZE, @@ -761,7 +919,11 @@ impl KernelScheme for ProcScheme { }; grants_read += 1; } - for (src, chunk) in dst.iter().take(grants_read).zip(buf.in_exact_chunks(mem::size_of::())) { + for (src, chunk) in dst + .iter() + .take(grants_read) + .zip(buf.in_exact_chunks(mem::size_of::())) + { chunk.copy_exactly(src)?; } @@ -772,14 +934,48 @@ impl KernelScheme for ProcScheme { Ok(grants_read * mem::size_of::()) } - Operation::Name => read_from(buf, context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?.read().name.as_bytes(), &mut 0), - Operation::SessionId => read_from(buf, &context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?.read().session_id.get().to_ne_bytes(), &mut 0), - Operation::Sigstack => read_from(buf, &context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?.read().sigstack.unwrap_or(!0).to_ne_bytes(), &mut 0), + Operation::Name => read_from( + buf, + context::contexts() + .get(info.pid) + .ok_or(Error::new(ESRCH))? + .read() + .name + .as_bytes(), + &mut 0, + ), + Operation::SessionId => read_from( + buf, + &context::contexts() + .get(info.pid) + .ok_or(Error::new(ESRCH))? + .read() + .session_id + .get() + .to_ne_bytes(), + &mut 0, + ), + Operation::Sigstack => read_from( + buf, + &context::contexts() + .get(info.pid) + .ok_or(Error::new(ESRCH))? + .read() + .sigstack + .unwrap_or(!0) + .to_ne_bytes(), + &mut 0, + ), Operation::Attr(attr) => { - let src_buf = match (attr, &*Arc::clone(context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?).read()) { + let src_buf = match ( + attr, + &*Arc::clone(context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?) + .read(), + ) { (Attr::Uid, context) => context.euid.to_string(), (Attr::Gid, context) => context.egid.to_string(), - }.into_bytes(); + } + .into_bytes(); read_from(buf, &src_buf, &mut 0) } @@ -796,7 +992,11 @@ impl KernelScheme for ProcScheme { } Operation::SchedAffinity => { // TODO: Improve the sched_affinity interface to allow a full mask. - let set = context::contexts().get(info.pid).ok_or(Error::new(EBADFD))?.read().sched_affinity; + let set = context::contexts() + .get(info.pid) + .ok_or(Error::new(EBADFD))? + .read() + .sched_affinity; let id = if set == LogicalCpuSet::empty() { usize::MAX @@ -837,7 +1037,8 @@ impl KernelScheme for ProcScheme { op @ ADDRSPACE_OP_MMAP | op @ ADDRSPACE_OP_TRANSFER => { let fd = next()??; let offset = next()??; - let (page, page_count) = crate::syscall::validate_region(next()??, next()??)?; + let (page, page_count) = + crate::syscall::validate_region(next()??, next()??)?; let flags = MapFlags::from_bits(next()??).ok_or(Error::new(EINVAL))?; if !flags.contains(MapFlags::MAP_FIXED) { @@ -846,19 +1047,35 @@ impl KernelScheme for ProcScheme { let (scheme, number) = extract_scheme_number(fd)?; - scheme.kfmap(number, &addrspace, &Map { offset, size: page_count * PAGE_SIZE, address: page.start_address().data(), flags }, op == ADDRSPACE_OP_TRANSFER)?; + scheme.kfmap( + number, + &addrspace, + &Map { + offset, + size: page_count * PAGE_SIZE, + address: page.start_address().data(), + flags, + }, + op == ADDRSPACE_OP_TRANSFER, + )?; } ADDRSPACE_OP_MUNMAP => { - let (page, page_count) = crate::syscall::validate_region(next()??, next()??)?; + let (page, page_count) = + crate::syscall::validate_region(next()??, next()??)?; let unpin = false; - addrspace.write().munmap(PageSpan::new(page, page_count), unpin)?; + addrspace + .write() + .munmap(PageSpan::new(page, page_count), unpin)?; } ADDRSPACE_OP_MPROTECT => { - let (page, page_count) = crate::syscall::validate_region(next()??, next()??)?; + let (page, page_count) = + crate::syscall::validate_region(next()??, next()??)?; let flags = MapFlags::from_bits(next()??).ok_or(Error::new(EINVAL))?; - addrspace.write().mprotect(PageSpan::new(page, page_count), flags)?; + addrspace + .write() + .mprotect(PageSpan::new(page, page_count), flags)?; } _ => return Err(Error::new(EINVAL)), } @@ -877,19 +1094,25 @@ impl KernelScheme for ProcScheme { Ok(mem::size_of::()) }) - }, + } RegsKind::Int => { let regs = unsafe { buf.read_exact::()? }; - try_stop_context(info.pid, |context| match unsafe { ptrace::regs_for_mut(context) } { - None => { - println!("{}:{}: Couldn't read registers from stopped process", file!(), line!()); - Err(Error::new(ENOTRECOVERABLE)) - }, - Some(stack) => { - stack.load(®s); + try_stop_context(info.pid, |context| { + match unsafe { ptrace::regs_for_mut(context) } { + None => { + println!( + "{}:{}: Couldn't read registers from stopped process", + file!(), + line!() + ); + Err(Error::new(ENOTRECOVERABLE)) + } + Some(stack) => { + stack.load(®s); - Ok(mem::size_of::()) + Ok(mem::size_of::()) + } } }) } @@ -906,8 +1129,7 @@ impl KernelScheme for ProcScheme { // Set next breakpoint ptrace::Session::with_session(info.pid, |session| { session.data.lock().set_breakpoint( - Some(op) - .filter(|op| op.intersects(PTRACE_STOP_MASK | PTRACE_EVENT_MASK)) + Some(op).filter(|op| op.intersects(PTRACE_STOP_MASK | PTRACE_EVENT_MASK)), ); Ok(()) })?; @@ -916,9 +1138,13 @@ impl KernelScheme for ProcScheme { try_stop_context(info.pid, |context| { match unsafe { ptrace::regs_for_mut(context) } { None => { - println!("{}:{}: Couldn't read registers from stopped process", file!(), line!()); + println!( + "{}:{}: Couldn't read registers from stopped process", + file!(), + line!() + ); Err(Error::new(ENOTRECOVERABLE)) - }, + } Some(stack) => { stack.set_singlestep(true); Ok(()) @@ -940,14 +1166,19 @@ impl KernelScheme for ProcScheme { })?; Ok(mem::size_of::()) - }, + } Operation::Name => { // TODO: What limit? let mut name_buf = [0_u8; 256]; let bytes_copied = buf.copy_common_bytes_to_slice(&mut name_buf)?; - let utf8 = alloc::string::String::from_utf8(name_buf[..bytes_copied].to_vec()).map_err(|_| Error::new(EINVAL))?; - context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?.write().name = utf8.into(); + let utf8 = alloc::string::String::from_utf8(name_buf[..bytes_copied].to_vec()) + .map_err(|_| Error::new(EINVAL))?; + context::contexts() + .get(info.pid) + .ok_or(Error::new(ESRCH))? + .write() + .name = utf8.into(); Ok(buf.len()) } Operation::SessionId => { @@ -965,7 +1196,8 @@ impl KernelScheme for ProcScheme { } } - let context_lock = Arc::clone(context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?); + let context_lock = + Arc::clone(context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?); { let mut context = context_lock.write(); context.pgid = session_id; @@ -976,7 +1208,11 @@ impl KernelScheme for ProcScheme { } Operation::Sigstack => { let sigstack = buf.read_usize()?; - context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?.write().sigstack = (sigstack != !0).then(|| sigstack); + context::contexts() + .get(info.pid) + .ok_or(Error::new(ESRCH))? + .write() + .sigstack = (sigstack != !0).then(|| sigstack); Ok(buf.len()) } Operation::Attr(attr) => { @@ -984,8 +1220,12 @@ impl KernelScheme for ProcScheme { let mut str_buf = [0_u8; 32]; let bytes_copied = buf.copy_common_bytes_to_slice(&mut str_buf)?; - let id = core::str::from_utf8(&str_buf[..bytes_copied]).map_err(|_| Error::new(EINVAL))?.parse::().map_err(|_| Error::new(EINVAL))?; - let context_lock = Arc::clone(context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?); + let id = core::str::from_utf8(&str_buf[..bytes_copied]) + .map_err(|_| Error::new(EINVAL))? + .parse::() + .map_err(|_| Error::new(EINVAL))?; + let context_lock = + Arc::clone(context::contexts().get(info.pid).ok_or(Error::new(ESRCH))?); match attr { Attr::Uid => context_lock.write().euid = id, @@ -1001,11 +1241,20 @@ impl KernelScheme for ProcScheme { verify_scheme(hopefully_this_scheme)?; let mut handles = HANDLES.write(); - let Operation::Filetable { ref filetable } = handles.get(&number).ok_or(Error::new(EBADF))?.info.operation else { + let Operation::Filetable { ref filetable } = handles + .get(&number) + .ok_or(Error::new(EBADF))? + .info + .operation + else { return Err(Error::new(EBADF)); }; - handles.get_mut(&id).ok_or(Error::new(EBADF))?.info.operation = Operation::AwaitingFiletableChange(Arc::clone(filetable)); + handles + .get_mut(&id) + .ok_or(Error::new(EBADF))? + .info + .operation = Operation::AwaitingFiletableChange(Arc::clone(filetable)); Ok(mem::size_of::()) } @@ -1019,11 +1268,24 @@ impl KernelScheme for ProcScheme { verify_scheme(hopefully_this_scheme)?; let mut handles = HANDLES.write(); - let Operation::AddrSpace { ref addrspace } = handles.get(&number).ok_or(Error::new(EBADF))?.info.operation else { + let Operation::AddrSpace { ref addrspace } = handles + .get(&number) + .ok_or(Error::new(EBADF))? + .info + .operation + else { return Err(Error::new(EBADF)); }; - handles.get_mut(&id).ok_or(Error::new(EBADF))?.info.operation = Operation::AwaitingAddrSpaceChange { new: Arc::clone(addrspace), new_sp: sp, new_ip: ip }; + handles + .get_mut(&id) + .ok_or(Error::new(EBADF))? + .info + .operation = Operation::AwaitingAddrSpaceChange { + new: Arc::clone(addrspace), + new_sp: sp, + new_ip: ip, + }; Ok(3 * mem::size_of::()) } @@ -1033,16 +1295,27 @@ impl KernelScheme for ProcScheme { verify_scheme(hopefully_this_scheme)?; let mut handles = HANDLES.write(); - let Operation::Sigactions(ref sigactions) = handles.get(&number).ok_or(Error::new(EBADF))?.info.operation else { + let Operation::Sigactions(ref sigactions) = handles + .get(&number) + .ok_or(Error::new(EBADF))? + .info + .operation + else { return Err(Error::new(EBADF)); }; - handles.get_mut(&id).ok_or(Error::new(EBADF))?.info.operation = Operation::AwaitingSigactionsChange(Arc::clone(sigactions)); + handles + .get_mut(&id) + .ok_or(Error::new(EBADF))? + .info + .operation = Operation::AwaitingSigactionsChange(Arc::clone(sigactions)); Ok(mem::size_of::()) } Operation::MmapMinAddr(ref addrspace) => { let val = buf.read_usize()?; - if val % PAGE_SIZE != 0 || val > crate::USER_END_OFFSET { return Err(Error::new(EINVAL)); } + if val % PAGE_SIZE != 0 || val > crate::USER_END_OFFSET { + return Err(Error::new(EINVAL)); + } addrspace.write().mmap_min = val; Ok(mem::size_of::()) } @@ -1051,13 +1324,15 @@ impl KernelScheme for ProcScheme { // TODO: read_u32 let val = u32::try_from(buf.read_usize()?).map_err(|_| Error::new(EINVAL))?; - context::contexts().get(info.pid) - .ok_or(Error::new(EBADFD))?.write() + context::contexts() + .get(info.pid) + .ok_or(Error::new(EBADFD))? + .write() .sched_affinity = if val == u32::MAX { - LogicalCpuSet::all() - } else { - LogicalCpuSet::single(LogicalCpuId::new(val % crate::cpu_count())) - }; + LogicalCpuSet::all() + } else { + LogicalCpuSet::single(LogicalCpuId::new(val % crate::cpu_count())) + }; Ok(mem::size_of::()) } @@ -1068,28 +1343,32 @@ impl KernelScheme for ProcScheme { let handles = HANDLES.read(); let handle = handles.get(&id).ok_or(Error::new(EBADF))?; - let path = format!("proc:{}/{}", handle.info.pid.get(), match handle.info.operation { - Operation::Regs(RegsKind::Float) => "regs/float", - Operation::Regs(RegsKind::Int) => "regs/int", - Operation::Regs(RegsKind::Env) => "regs/env", - Operation::Trace => "trace", - Operation::Static(path) => path, - Operation::Name => "name", - Operation::Sigstack => "sigstack", - Operation::Attr(Attr::Uid) => "uid", - Operation::Attr(Attr::Gid) => "gid", - Operation::Filetable { .. } => "filetable", - Operation::AddrSpace { .. } => "addrspace", - Operation::Sigactions(_) => "sigactions", - Operation::CurrentAddrSpace => "current-addrspace", - Operation::CurrentFiletable => "current-filetable", - Operation::CurrentSigactions => "current-sigactions", - Operation::OpenViaDup => "open-via-dup", - Operation::MmapMinAddr(_) => "mmap-min-addr", - Operation::SchedAffinity => "sched-affinity", + let path = format!( + "proc:{}/{}", + handle.info.pid.get(), + match handle.info.operation { + Operation::Regs(RegsKind::Float) => "regs/float", + Operation::Regs(RegsKind::Int) => "regs/int", + Operation::Regs(RegsKind::Env) => "regs/env", + Operation::Trace => "trace", + Operation::Static(path) => path, + Operation::Name => "name", + Operation::Sigstack => "sigstack", + Operation::Attr(Attr::Uid) => "uid", + Operation::Attr(Attr::Gid) => "gid", + Operation::Filetable { .. } => "filetable", + Operation::AddrSpace { .. } => "addrspace", + Operation::Sigactions(_) => "sigactions", + Operation::CurrentAddrSpace => "current-addrspace", + Operation::CurrentFiletable => "current-filetable", + Operation::CurrentSigactions => "current-sigactions", + Operation::OpenViaDup => "open-via-dup", + Operation::MmapMinAddr(_) => "mmap-min-addr", + Operation::SchedAffinity => "sched-affinity", - _ => return Err(Error::new(EOPNOTSUPP)), - }); + _ => return Err(Error::new(EOPNOTSUPP)), + } + ); buf.copy_common_bytes_from_slice(path.as_bytes()) } @@ -1136,11 +1415,24 @@ impl KernelScheme for ProcScheme { new_handle(match info.operation { Operation::OpenViaDup => { - let (uid, gid) = match &*context::contexts().current().ok_or(Error::new(ESRCH))?.read() { + let (uid, gid) = match &*context::contexts() + .current() + .ok_or(Error::new(ESRCH))? + .read() + { context => (context.euid, context.egid), }; - return self.open_inner(info.pid, Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?).filter(|s| !s.is_empty()), O_RDWR | O_CLOEXEC, uid, gid).map(OpenResult::SchemeLocal); - }, + return self + .open_inner( + info.pid, + Some(core::str::from_utf8(buf).map_err(|_| Error::new(EINVAL))?) + .filter(|s| !s.is_empty()), + O_RDWR | O_CLOEXEC, + uid, + gid, + ) + .map(OpenResult::SchemeLocal); + } Operation::Filetable { ref filetable } => { // TODO: Maybe allow userspace to either copy or transfer recently dupped file @@ -1148,9 +1440,15 @@ impl KernelScheme for ProcScheme { if buf != b"copy" { return Err(Error::new(EINVAL)); } - let new_filetable = Arc::try_new(RwLock::new(filetable.read().clone())).map_err(|_| Error::new(ENOMEM))?; + let new_filetable = Arc::try_new(RwLock::new(filetable.read().clone())) + .map_err(|_| Error::new(ENOMEM))?; - handle(Operation::Filetable { filetable: new_filetable }, OperationData::Other) + handle( + Operation::Filetable { + filetable: new_filetable, + }, + OperationData::Other, + ) } Operation::AddrSpace { ref addrspace } => { const GRANT_FD_PREFIX: &[u8] = b"grant-fd-"; @@ -1158,13 +1456,19 @@ impl KernelScheme for ProcScheme { let operation = match buf { // TODO: Better way to obtain new empty address spaces, perhaps using SYS_OPEN. But // in that case, what scheme? - b"empty" => Operation::AddrSpace { addrspace: new_addrspace()? }, - b"exclusive" => Operation::AddrSpace { addrspace: addrspace.write().try_clone()? }, + b"empty" => Operation::AddrSpace { + addrspace: new_addrspace()?, + }, + b"exclusive" => Operation::AddrSpace { + addrspace: addrspace.write().try_clone()?, + }, b"mmap-min-addr" => Operation::MmapMinAddr(Arc::clone(addrspace)), _ if buf.starts_with(GRANT_FD_PREFIX) => { - let string = core::str::from_utf8(&buf[GRANT_FD_PREFIX.len()..]).map_err(|_| Error::new(EINVAL))?; - let page_addr = usize::from_str_radix(string, 16).map_err(|_| Error::new(EINVAL))?; + let string = core::str::from_utf8(&buf[GRANT_FD_PREFIX.len()..]) + .map_err(|_| Error::new(EINVAL))?; + let page_addr = + usize::from_str_radix(string, 16).map_err(|_| Error::new(EINVAL))?; if page_addr % PAGE_SIZE != 0 { return Err(Error::new(EINVAL)); @@ -1172,8 +1476,19 @@ impl KernelScheme for ProcScheme { let page = Page::containing_address(VirtualAddress::new(page_addr)); - match addrspace.read().grants.contains(page).ok_or(Error::new(EINVAL))? { - (_, info) => return Ok(OpenResult::External(info.file_ref().map(|r| Arc::clone(&r.description)).ok_or(Error::new(EBADF))?)), + match addrspace + .read() + .grants + .contains(page) + .ok_or(Error::new(EINVAL))? + { + (_, info) => { + return Ok(OpenResult::External( + info.file_ref() + .map(|r| Arc::clone(&r.description)) + .ok_or(Error::new(EBADF))?, + )) + } } } @@ -1191,12 +1506,16 @@ impl KernelScheme for ProcScheme { handle(Operation::Sigactions(new), OperationData::Other) } _ => return Err(Error::new(EINVAL)), - }).map(OpenResult::SchemeLocal) + }) + .map(OpenResult::SchemeLocal) } - } extern "C" fn clone_handler() { - let context_lock = Arc::clone(context::contexts().current().expect("expected the current context to be set in a spawn closure")); + let context_lock = Arc::clone( + context::contexts() + .current() + .expect("expected the current context to be set in a spawn closure"), + ); loop { unsafe { @@ -1213,7 +1532,8 @@ extern "C" fn clone_handler() { fn inherit_context() -> Result { let new_id = { - let current_context_lock = Arc::clone(context::contexts().current().ok_or(Error::new(ESRCH))?); + let current_context_lock = + Arc::clone(context::contexts().current().ok_or(Error::new(ESRCH))?); let new_context_lock = Arc::clone(context::contexts_mut().spawn(clone_handler)?); let current_context = current_context_lock.read(); @@ -1240,11 +1560,18 @@ fn inherit_context() -> Result { new_context.id }; - if ptrace::send_event(crate::syscall::ptrace_event!(PTRACE_EVENT_CLONE, new_id.into())).is_some() { + if ptrace::send_event(crate::syscall::ptrace_event!( + PTRACE_EVENT_CLONE, + new_id.into() + )) + .is_some() + { // Freeze the clone, allow ptrace to put breakpoints // to it before it starts let contexts = context::contexts(); - let context = contexts.get(new_id).expect("Newly created context doesn't exist??"); + let context = contexts + .get(new_id) + .expect("Newly created context doesn't exist??"); let mut context = context.write(); context.ptrace_stop = true; } @@ -1252,15 +1579,29 @@ fn inherit_context() -> Result { Ok(new_id) } fn extract_scheme_number(fd: usize) -> Result<(KernelSchemes, usize)> { - let (scheme_id, number) = match &*context::contexts().current().ok_or(Error::new(ESRCH))?.read().get_file(FileHandle::from(fd)).ok_or(Error::new(EBADF))?.description.read() { - desc => (desc.scheme, desc.number) + let (scheme_id, number) = match &*context::contexts() + .current() + .ok_or(Error::new(ESRCH))? + .read() + .get_file(FileHandle::from(fd)) + .ok_or(Error::new(EBADF))? + .description + .read() + { + desc => (desc.scheme, desc.number), }; - let scheme = scheme::schemes().get(scheme_id).ok_or(Error::new(ENODEV))?.clone(); + let scheme = scheme::schemes() + .get(scheme_id) + .ok_or(Error::new(ENODEV))? + .clone(); Ok((scheme, number)) } fn verify_scheme(scheme: KernelSchemes) -> Result<()> { - if !matches!(scheme, KernelSchemes::Global(GlobalSchemes::ProcFull | GlobalSchemes::ProcRestricted)) { + if !matches!( + scheme, + KernelSchemes::Global(GlobalSchemes::ProcFull | GlobalSchemes::ProcRestricted) + ) { return Err(Error::new(EBADF)); } Ok(()) diff --git a/src/scheme/root.rs b/src/scheme/root.rs index a627b3fcac..ae5bc46070 100644 --- a/src/scheme/root.rs +++ b/src/scheme/root.rs @@ -1,27 +1,31 @@ -use alloc::{ - boxed::Box, - string::ToString, - sync::Arc, - vec::Vec, +use alloc::{boxed::Box, string::ToString, sync::Arc, vec::Vec}; +use core::{ + str, + sync::atomic::{AtomicUsize, Ordering}, }; use hashbrown::HashMap; -use core::str; -use core::sync::atomic::{AtomicUsize, Ordering}; use spin::{Mutex, RwLock}; -use crate::context; -use crate::syscall::data::Stat; -use crate::syscall::error::*; -use crate::syscall::flag::{EventFlags, O_CREAT, MODE_FILE, MODE_DIR}; -use crate::scheme::{self, SchemeNamespace, SchemeId}; -use crate::scheme::user::{UserInner, UserScheme}; -use crate::syscall::usercopy::{UserSliceWo, UserSliceRo}; +use crate::{ + context, + scheme::{ + self, + user::{UserInner, UserScheme}, + SchemeId, SchemeNamespace, + }, + syscall::{ + data::Stat, + error::*, + flag::{EventFlags, MODE_DIR, MODE_FILE, O_CREAT}, + usercopy::{UserSliceRo, UserSliceWo}, + }, +}; -use super::{KernelScheme, KernelSchemes, CallerCtx, OpenResult, calc_seek_offset}; +use super::{calc_seek_offset, CallerCtx, KernelScheme, KernelSchemes, OpenResult}; struct FolderInner { data: Box<[u8]>, - pos: Mutex + pos: Mutex, } impl FolderInner { @@ -47,7 +51,7 @@ impl FolderInner { enum Handle { Scheme(Arc), File(Arc>), - Folder(Arc) + Folder(Arc), } pub struct RootScheme { @@ -94,10 +98,21 @@ impl KernelScheme for RootScheme { let path_box = path.to_string().into_boxed_str(); let mut schemes = scheme::schemes_mut(); - let (_scheme_id, inner) = schemes.insert_and_pass(self.scheme_ns, path, |scheme_id| { - let inner = Arc::new(UserInner::new(self.scheme_id, scheme_id, id, path_box, flags, context)); - (KernelSchemes::User(UserScheme::new(Arc::downgrade(&inner))), inner) - })?; + let (_scheme_id, inner) = + schemes.insert_and_pass(self.scheme_ns, path, |scheme_id| { + let inner = Arc::new(UserInner::new( + self.scheme_id, + scheme_id, + id, + path_box, + flags, + context, + )); + ( + KernelSchemes::User(UserScheme::new(Arc::downgrade(&inner))), + inner, + ) + })?; inner }; @@ -124,16 +139,14 @@ impl KernelScheme for RootScheme { let inner = Arc::new(FolderInner { data: data.into_boxed_slice(), - pos: Mutex::new(0) + pos: Mutex::new(0), }); let id = self.next_id.fetch_add(1, Ordering::Relaxed); self.handles.write().insert(id, Handle::Folder(inner)); Ok(OpenResult::SchemeLocal(id)) } else { - let inner = Arc::new( - path.as_bytes().to_vec().into_boxed_slice() - ); + let inner = Arc::new(path.as_bytes().to_vec().into_boxed_slice()); let id = self.next_id.fetch_add(1, Ordering::Relaxed); self.handles.write().insert(id, Handle::File(inner)); @@ -149,23 +162,25 @@ impl KernelScheme for RootScheme { } let inner = { let handles = self.handles.read(); - handles.iter().find_map(|(_id, handle)| { - match handle { - Handle::Scheme(inner) => { - if path == inner.name.as_ref() { - return Some(inner.clone()); + handles + .iter() + .find_map(|(_id, handle)| { + match handle { + Handle::Scheme(inner) => { + if path == inner.name.as_ref() { + return Some(inner.clone()); + } } - }, - _ => (), - } - None - }).ok_or(Error::new(ENOENT))? + _ => (), + } + None + }) + .ok_or(Error::new(ENOENT))? }; inner.unmount() } - fn seek(&self, file: usize, pos: isize, whence: usize) -> Result { let handle = { let handles = self.handles.read(); @@ -174,15 +189,9 @@ impl KernelScheme for RootScheme { }; match handle { - Handle::Scheme(_) => { - Err(Error::new(EBADF)) - }, - Handle::File(_) => { - Err(Error::new(EBADF)) - }, - Handle::Folder(inner) => { - inner.seek(pos, whence) - } + Handle::Scheme(_) => Err(Error::new(EBADF)), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::Folder(inner) => inner.seek(pos, whence), } } @@ -194,15 +203,9 @@ impl KernelScheme for RootScheme { }; match handle { - Handle::Scheme(inner) => { - inner.fevent(flags) - }, - Handle::File(_) => { - Err(Error::new(EBADF)) - }, - Handle::Folder(_) => { - Err(Error::new(EBADF)) - } + Handle::Scheme(inner) => inner.fevent(flags), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::Folder(_) => Err(Error::new(EBADF)), } } @@ -219,11 +222,11 @@ impl KernelScheme for RootScheme { match handle { Handle::Scheme(inner) => { bytes_copied += buf.copy_common_bytes_from_slice(inner.name.as_bytes())?; - }, + } Handle::File(inner) => { bytes_copied += buf.copy_common_bytes_from_slice(&inner)?; - }, - Handle::Folder(_) => () + } + Handle::Folder(_) => (), } Ok(bytes_copied) @@ -237,25 +240,23 @@ impl KernelScheme for RootScheme { }; match handle { - Handle::Scheme(inner) => { - inner.fsync() - }, - Handle::File(_) => { - Err(Error::new(EBADF)) - }, - Handle::Folder(_) => { - Err(Error::new(EBADF)) - } + Handle::Scheme(inner) => inner.fsync(), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::Folder(_) => Err(Error::new(EBADF)), } } fn close(&self, file: usize) -> Result<()> { - let handle = self.handles.write().remove(&file).ok_or(Error::new(EBADF))?; + let handle = self + .handles + .write() + .remove(&file) + .ok_or(Error::new(EBADF))?; match handle { Handle::Scheme(inner) => { scheme::schemes_mut().remove(inner.scheme_id); - }, - _ => () + } + _ => (), } Ok(()) } @@ -267,15 +268,9 @@ impl KernelScheme for RootScheme { }; match handle { - Handle::Scheme(inner) => { - inner.read(buf) - }, - Handle::File(_) => { - Err(Error::new(EBADF)) - }, - Handle::Folder(inner) => { - inner.read(buf) - } + Handle::Scheme(inner) => inner.read(buf), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::Folder(inner) => inner.read(buf), } } @@ -287,15 +282,9 @@ impl KernelScheme for RootScheme { }; match handle { - Handle::Scheme(inner) => { - inner.write(buf) - }, - Handle::File(_) => { - Err(Error::new(EBADF)) - }, - Handle::Folder(_) => { - Err(Error::new(EBADF)) - } + Handle::Scheme(inner) => inner.write(buf), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::Folder(_) => Err(Error::new(EBADF)), } } @@ -327,10 +316,9 @@ impl KernelScheme for RootScheme { st_gid: 0, st_size: inner.data.len() as u64, ..Default::default() - } + }, })?; Ok(()) } - } diff --git a/src/scheme/serio.rs b/src/scheme/serio.rs index 13979c5d0e..bd240330f4 100644 --- a/src/scheme/serio.rs +++ b/src/scheme/serio.rs @@ -1,15 +1,21 @@ //! PS/2 unfortunately requires a kernel driver to prevent race conditions due //! to how status is utilized -use core::str; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::{ + str, + sync::atomic::{AtomicUsize, Ordering}, +}; use spin::RwLock; -use crate::event; -use crate::scheme::*; -use crate::sync::WaitQueue; -use crate::syscall::flag::{EventFlags, EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK}; -use crate::syscall::usercopy::UserSliceWo; +use crate::{ + event, + scheme::*, + sync::WaitQueue, + syscall::{ + flag::{EventFlags, EVENT_READ, F_GETFL, F_SETFL, O_ACCMODE, O_NONBLOCK}, + usercopy::UserSliceWo, + }, +}; static NEXT_ID: AtomicUsize = AtomicUsize::new(0); @@ -45,18 +51,19 @@ impl KernelScheme for SerioScheme { return Err(Error::new(EPERM)); } - let index = path - .parse::() - .or(Err(Error::new(ENOENT)))?; + let index = path.parse::().or(Err(Error::new(ENOENT)))?; if index >= INPUT.len() { return Err(Error::new(ENOENT)); } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, Handle { - index, - flags: flags & ! O_ACCMODE - }); + HANDLES.write().insert( + id, + Handle { + index, + flags: flags & !O_ACCMODE, + }, + ); Ok(OpenResult::SchemeLocal(id)) } @@ -67,10 +74,10 @@ impl KernelScheme for SerioScheme { match cmd { F_GETFL => Ok(handle.flags), F_SETFL => { - handle.flags = arg & ! O_ACCMODE; + handle.flags = arg & !O_ACCMODE; Ok(0) - }, - _ => Err(Error::new(EINVAL)) + } + _ => Err(Error::new(EINVAL)), } } else { Err(Error::new(EBADF)) @@ -110,8 +117,11 @@ impl KernelScheme for SerioScheme { *handles.get(&id).ok_or(Error::new(EBADF))? }; - INPUT[handle.index] - .receive_into_user(buf, handle.flags & O_NONBLOCK != O_NONBLOCK, "SerioScheme::read") + INPUT[handle.index].receive_into_user( + buf, + handle.flags & O_NONBLOCK != O_NONBLOCK, + "SerioScheme::read", + ) } fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { diff --git a/src/scheme/sys/block.rs b/src/scheme/sys/block.rs index b02c139661..b792cf02fb 100644 --- a/src/scheme/sys/block.rs +++ b/src/scheme/sys/block.rs @@ -1,9 +1,7 @@ -use alloc::string::String; -use alloc::vec::Vec; +use alloc::{string::String, vec::Vec}; use core::fmt::Write; -use crate::context; -use crate::syscall::error::Result; +use crate::{context, syscall::error::Result}; pub fn resource() -> Result> { let mut string = String::new(); @@ -24,7 +22,7 @@ pub fn resource() -> Result> { let _ = writeln!(string, "{}: {}", id, name); - if ! row.2.is_empty() { + if !row.2.is_empty() { let _ = writeln!(string, " {}", row.2); } } diff --git a/src/scheme/sys/context.rs b/src/scheme/sys/context.rs index d75d42c1d9..4cd3fd7386 100644 --- a/src/scheme/sys/context.rs +++ b/src/scheme/sys/context.rs @@ -1,28 +1,27 @@ -use alloc::string::String; -use alloc::vec::Vec; +use alloc::{string::String, vec::Vec}; -use crate::context; -use crate::paging::PAGE_SIZE; -use crate::syscall::error::Result; +use crate::{context, paging::PAGE_SIZE, syscall::error::Result}; pub fn resource() -> Result> { - let mut string = format!("{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<12}{:<8}{}\n", - "PID", - "PGID", - "PPID", - "SID", - "RUID", - "RGID", - "RNS", - "EUID", - "EGID", - "ENS", - "STAT", - "CPU", - "AFF", - "TIME", - "MEM", - "NAME"); + let mut string = format!( + "{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<12}{:<8}{}\n", + "PID", + "PGID", + "PPID", + "SID", + "RUID", + "RGID", + "RNS", + "EUID", + "EGID", + "ENS", + "STAT", + "CPU", + "AFF", + "TIME", + "MEM", + "NAME" + ); { let contexts = context::contexts(); for (_id, context_lock) in contexts.iter() { @@ -43,12 +42,14 @@ pub fn resource() -> Result> { match context.status { context::Status::Runnable => { stat_string.push('R'); - }, - context::Status::Blocked | context::Status::HardBlocked { .. } => if context.wake.is_some() { - stat_string.push('S'); - } else { - stat_string.push('B'); - }, + } + context::Status::Blocked | context::Status::HardBlocked { .. } => { + if context.wake.is_some() { + stat_string.push('S'); + } else { + stat_string.push('B'); + } + } context::Status::Stopped(_sig) => { stat_string.push('T'); } @@ -65,7 +66,10 @@ pub fn resource() -> Result> { } else { format!("?") }; - let affinity = format!("{:#x}", context.sched_affinity.get() & ((1 << crate::cpu_count()) - 1)); + let affinity = format!( + "{:#x}", + context.sched_affinity.get() & ((1 << crate::cpu_count()) - 1) + ); let cpu_time_s = context.cpu_time / crate::time::NANOS_PER_SEC; let cpu_time_ns = context.cpu_time % crate::time::NANOS_PER_SEC; @@ -100,23 +104,25 @@ pub fn resource() -> Result> { format!("{} B", memory) }; - string.push_str(&format!("{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<12}{:<8}{}\n", - context.id.get(), - context.pgid.get(), - context.ppid.get(), - context.session_id.get(), - context.ruid, - context.rgid, - context.rns.get(), - context.euid, - context.egid, - context.ens.get(), - stat_string, - cpu_string, - affinity, - cpu_time_string, - memory_string, - context.name)); + string.push_str(&format!( + "{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<6}{:<12}{:<8}{}\n", + context.id.get(), + context.pgid.get(), + context.ppid.get(), + context.session_id.get(), + context.ruid, + context.rgid, + context.rns.get(), + context.euid, + context.egid, + context.ens.get(), + stat_string, + cpu_string, + affinity, + cpu_time_string, + memory_string, + context.name + )); } } diff --git a/src/scheme/sys/cpu.rs b/src/scheme/sys/cpu.rs index 01854c818e..f9f2c67267 100644 --- a/src/scheme/sys/cpu.rs +++ b/src/scheme/sys/cpu.rs @@ -1,13 +1,15 @@ use alloc::vec::Vec; -use crate::device::cpu::cpu_info; -use crate::syscall::error::{Error, EIO, Result}; +use crate::{ + device::cpu::cpu_info, + syscall::error::{Error, Result, EIO}, +}; pub fn resource() -> Result> { let mut string = format!("CPUs: {}\n", crate::cpu_count()); match cpu_info(&mut string) { Ok(()) => Ok(string.into_bytes()), - Err(_) => Err(Error::new(EIO)) + Err(_) => Err(Error::new(EIO)), } } diff --git a/src/scheme/sys/exe.rs b/src/scheme/sys/exe.rs index 6e01c7e184..bbf2d35a49 100644 --- a/src/scheme/sys/exe.rs +++ b/src/scheme/sys/exe.rs @@ -1,8 +1,12 @@ use alloc::vec::Vec; -use crate::context; -use crate::syscall::error::Result; +use crate::{context, syscall::error::Result}; pub fn resource() -> Result> { - Ok(context::current()?.read().name.clone().into_owned().into_bytes()) + Ok(context::current()? + .read() + .name + .clone() + .into_owned() + .into_bytes()) } diff --git a/src/scheme/sys/iostat.rs b/src/scheme/sys/iostat.rs index c381a3af6b..557baf1a72 100644 --- a/src/scheme/sys/iostat.rs +++ b/src/scheme/sys/iostat.rs @@ -1,9 +1,6 @@ -use alloc::string::String; -use alloc::vec::Vec; +use crate::{context, scheme, syscall::error::Result}; +use alloc::{string::String, vec::Vec}; use core::fmt::Write; -use crate::context; -use crate::scheme; -use crate::syscall::error::Result; pub fn resource() -> Result> { let mut string = String::new(); @@ -26,7 +23,7 @@ pub fn resource() -> Result> { for (fd, f) in row.2.iter().enumerate() { let file = match *f { None => continue, - Some(ref file) => file.clone() + Some(ref file) => file.clone(), }; let description = file.description.read(); @@ -36,7 +33,14 @@ pub fn resource() -> Result> { match schemes.get(description.scheme) { Some(scheme) => scheme.clone(), None => { - let _ = writeln!(string, " {:>4}: {:>8} {:>8} {:>08X}: no scheme", fd, description.scheme.get(), description.number, description.flags); + let _ = writeln!( + string, + " {:>4}: {:>8} {:>8} {:>08X}: no scheme", + fd, + description.scheme.get(), + description.number, + description.flags + ); continue; } } diff --git a/src/scheme/sys/irq.rs b/src/scheme/sys/irq.rs index 7e4b69c31a..655dc1b4b5 100644 --- a/src/scheme/sys/irq.rs +++ b/src/scheme/sys/irq.rs @@ -1,7 +1,4 @@ -use alloc::{ - string::String, - vec::Vec, -}; +use alloc::{string::String, vec::Vec}; use core::fmt::Write; use crate::syscall::error::Result; diff --git a/src/scheme/sys/log.rs b/src/scheme/sys/log.rs index c8b8efd4bb..709e4a117e 100644 --- a/src/scheme/sys/log.rs +++ b/src/scheme/sys/log.rs @@ -1,7 +1,6 @@ use alloc::vec::Vec; -use crate::log::LOG; -use crate::syscall::error::Result; +use crate::{log::LOG, syscall::error::Result}; pub fn resource() -> Result> { let mut vec = Vec::new(); diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs index eff0051165..e1f8d879c8 100644 --- a/src/scheme/sys/mod.rs +++ b/src/scheme/sys/mod.rs @@ -1,16 +1,21 @@ -use alloc::collections::BTreeMap; -use alloc::vec::Vec; -use core::str; -use core::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{collections::BTreeMap, vec::Vec}; +use core::{ + str, + sync::atomic::{AtomicUsize, Ordering}, +}; use spin::RwLock; -use crate::syscall::data::Stat; -use crate::syscall::error::{Error, EBADF, ENOENT, Result}; -use crate::syscall::flag::{MODE_DIR, MODE_FILE}; -use crate::arch::interrupt; -use crate::syscall::usercopy::UserSliceWo; +use crate::{ + arch::interrupt, + syscall::{ + data::Stat, + error::{Error, Result, EBADF, ENOENT}, + flag::{MODE_DIR, MODE_FILE}, + usercopy::UserSliceWo, + }, +}; -use super::{KernelScheme, CallerCtx, OpenResult, calc_seek_offset}; +use super::{calc_seek_offset, CallerCtx, KernelScheme, OpenResult}; mod block; mod context; @@ -28,7 +33,7 @@ struct Handle { path: &'static str, data: Vec, mode: u16, - seek: usize + seek: usize, } type SysFn = fn() -> Result>; @@ -70,32 +75,38 @@ impl KernelScheme for SysScheme { if path.is_empty() { let mut data = Vec::new(); for entry in FILES.iter() { - if ! data.is_empty() { + if !data.is_empty() { data.push(b'\n'); } data.extend_from_slice(entry.0.as_bytes()); } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - HANDLES.write().insert(id, Handle { - path: "", - data, - mode: MODE_DIR | 0o444, - seek: 0 - }); - return Ok(OpenResult::SchemeLocal(id)) + HANDLES.write().insert( + id, + Handle { + path: "", + data, + mode: MODE_DIR | 0o444, + seek: 0, + }, + ); + return Ok(OpenResult::SchemeLocal(id)); } else { //Have to iterate to get the path without allocation for entry in FILES.iter() { if &entry.0 == &path { let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); let data = entry.1()?; - HANDLES.write().insert(id, Handle { - path: entry.0, - data, - mode: MODE_FILE | 0o444, - seek: 0 - }); + HANDLES.write().insert( + id, + Handle { + path: entry.0, + data, + mode: MODE_FILE | 0o444, + seek: 0, + }, + ); return Ok(OpenResult::SchemeLocal(id)); } } @@ -118,7 +129,11 @@ impl KernelScheme for SysScheme { } fn close(&self, id: usize) -> Result<()> { - HANDLES.write().remove(&id).ok_or(Error::new(EBADF)).and(Ok(())) + HANDLES + .write() + .remove(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(())) } fn kfpath(&self, id: usize, buf: UserSliceWo) -> Result { let handles = HANDLES.read(); @@ -131,7 +146,6 @@ impl KernelScheme for SysScheme { bytes_read += remaining.copy_common_bytes_from_slice(handle.path.as_bytes())?; } - Ok(bytes_read) } fn kread(&self, id: usize, buffer: UserSliceWo) -> Result { diff --git a/src/scheme/sys/scheme.rs b/src/scheme/sys/scheme.rs index e0a367ac51..2c64e61258 100644 --- a/src/scheme/sys/scheme.rs +++ b/src/scheme/sys/scheme.rs @@ -1,8 +1,9 @@ use alloc::vec::Vec; -use crate::context; -use crate::scheme; -use crate::syscall::error::{Error, ESRCH, Result}; +use crate::{ + context, scheme, + syscall::error::{Error, Result, ESRCH}, +}; pub fn resource() -> Result> { let scheme_ns = { diff --git a/src/scheme/sys/scheme_num.rs b/src/scheme/sys/scheme_num.rs index 091284dcfb..9a6dc23bee 100644 --- a/src/scheme/sys/scheme_num.rs +++ b/src/scheme/sys/scheme_num.rs @@ -1,8 +1,9 @@ use alloc::vec::Vec; -use crate::context; -use crate::scheme; -use crate::syscall::error::{Error, ESRCH, Result}; +use crate::{ + context, scheme, + syscall::error::{Error, Result, ESRCH}, +}; pub fn resource() -> Result> { let scheme_ns = { diff --git a/src/scheme/sys/syscall.rs b/src/scheme/sys/syscall.rs index 5f3184f19c..3da88a9c8c 100644 --- a/src/scheme/sys/syscall.rs +++ b/src/scheme/sys/syscall.rs @@ -1,10 +1,7 @@ -use alloc::string::String; -use alloc::vec::Vec; +use alloc::{string::String, vec::Vec}; use core::fmt::Write; -use crate::context; -use crate::syscall; -use crate::syscall::error::Result; +use crate::{context, syscall, syscall::error::Result}; pub fn resource() -> Result> { let mut string = String::new(); @@ -26,7 +23,11 @@ pub fn resource() -> Result> { let _ = writeln!(string, "{}: {}", id, name); if let Some((a, b, c, d, e, f)) = row.2 { - let _ = writeln!(string, " {}", syscall::debug::format_call(a, b, c, d, e, f)); + let _ = writeln!( + string, + " {}", + syscall::debug::format_call(a, b, c, d, e, f) + ); } } } diff --git a/src/scheme/sys/uname.rs b/src/scheme/sys/uname.rs index b82546153f..c4136a0dcb 100644 --- a/src/scheme/sys/uname.rs +++ b/src/scheme/sys/uname.rs @@ -1,9 +1,11 @@ -use alloc::vec::Vec; use crate::syscall::error::Result; +use alloc::vec::Vec; pub fn resource() -> Result> { - Ok(format!("Redox\n\n{}\n\n{}\n", - env!("CARGO_PKG_VERSION"), - env!("TARGET").split('-').next().unwrap()).into_bytes()) + Ok(format!( + "Redox\n\n{}\n\n{}\n", + env!("CARGO_PKG_VERSION"), + env!("TARGET").split('-').next().unwrap() + ) + .into_bytes()) } - diff --git a/src/scheme/time.rs b/src/scheme/time.rs index 645bdad1c2..237c5916e6 100644 --- a/src/scheme/time.rs +++ b/src/scheme/time.rs @@ -1,16 +1,22 @@ use alloc::collections::BTreeMap; -use core::{mem, str}; -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::{ + mem, str, + sync::atomic::{AtomicUsize, Ordering}, +}; use spin::RwLock; -use crate::context::timeout; -use crate::syscall::data::TimeSpec; -use crate::syscall::error::*; -use crate::syscall::flag::{CLOCK_REALTIME, CLOCK_MONOTONIC, EventFlags}; -use crate::syscall::usercopy::{UserSliceWo, UserSliceRo}; -use crate::time; +use crate::{ + context::timeout, + syscall::{ + data::TimeSpec, + error::*, + flag::{EventFlags, CLOCK_MONOTONIC, CLOCK_REALTIME}, + usercopy::{UserSliceRo, UserSliceWo}, + }, + time, +}; -use super::{GlobalSchemes, KernelScheme, CallerCtx, OpenResult}; +use super::{CallerCtx, GlobalSchemes, KernelScheme, OpenResult}; static NEXT_ID: AtomicUsize = AtomicUsize::new(1); // Using BTreeMap as hashbrown doesn't have a const constructor. @@ -25,7 +31,7 @@ impl KernelScheme for TimeScheme { match clock { CLOCK_REALTIME => (), CLOCK_MONOTONIC => (), - _ => return Err(Error::new(ENOENT)) + _ => return Err(Error::new(ENOENT)), } let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); @@ -39,7 +45,11 @@ impl KernelScheme for TimeScheme { } fn fevent(&self, id: usize, _flags: EventFlags) -> Result { - HANDLES.read().get(&id).ok_or(Error::new(EBADF)).and(Ok(EventFlags::empty())) + HANDLES + .read() + .get(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(EventFlags::empty())) } fn fsync(&self, id: usize) -> Result<()> { @@ -48,7 +58,11 @@ impl KernelScheme for TimeScheme { } fn close(&self, id: usize) -> Result<()> { - HANDLES.write().remove(&id).ok_or(Error::new(EBADF)).and(Ok(())) + HANDLES + .write() + .remove(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(())) } fn kread(&self, id: usize, buf: UserSliceWo) -> Result { let clock = *HANDLES.read().get(&id).ok_or(Error::new(EBADF))?; @@ -59,7 +73,7 @@ impl KernelScheme for TimeScheme { let arch_time = match clock { CLOCK_REALTIME => time::realtime(), CLOCK_MONOTONIC => time::monotonic(), - _ => return Err(Error::new(EINVAL)) + _ => return Err(Error::new(EINVAL)), }; let time = TimeSpec { tv_sec: (arch_time / time::NANOS_PER_SEC) as i64, @@ -84,7 +98,7 @@ impl KernelScheme for TimeScheme { timeout::register(GlobalSchemes::Time.scheme_id(), id, clock, time); bytes_written += mem::size_of::(); - }; + } Ok(bytes_written) } @@ -94,5 +108,4 @@ impl KernelScheme for TimeScheme { let scheme_path = format!("time:{}", clock).into_bytes(); buf.copy_common_bytes_from_slice(&scheme_path) } - } diff --git a/src/scheme/user.rs b/src/scheme/user.rs index e52a1063ff..35cb5e519d 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -1,31 +1,47 @@ -use alloc::sync::{Arc, Weak}; -use alloc::boxed::Box; -use alloc::vec::Vec; -use syscall::{SKMSG_FRETURNFD, SKMSG_PROVIDE_MMAP, MAP_FIXED_NOREPLACE, MunmapFlags, SKMSG_FOBTAINFD, FobtainFdFlags, SendFdFlags}; -use core::mem::size_of; -use core::num::NonZeroUsize; -use core::sync::atomic::{AtomicBool, Ordering}; -use core::{mem, usize}; +use alloc::{ + boxed::Box, + sync::{Arc, Weak}, + vec::Vec, +}; +use core::{ + mem, + mem::size_of, + num::NonZeroUsize, + sync::atomic::{AtomicBool, Ordering}, + usize, +}; use hashbrown::hash_map::{Entry, HashMap}; use spin::{Mutex, RwLock}; +use syscall::{ + FobtainFdFlags, MunmapFlags, SendFdFlags, MAP_FIXED_NOREPLACE, SKMSG_FOBTAINFD, + SKMSG_FRETURNFD, SKMSG_PROVIDE_MMAP, +}; -use crate::context::context::HardBlockedReason; -use crate::context::{self, Context, BorrowedHtBuf, Status}; -use crate::context::file::{FileDescription, FileDescriptor}; -use crate::context::memory::{AddrSpace, DANGLING, Grant, GrantFileRef, PageSpan, MmapMode, BorrowedFmapSource}; -use crate::event; -use crate::memory::Frame; -use crate::paging::mapper::InactiveFlusher; -use crate::paging::{PAGE_SIZE, Page, VirtualAddress}; -use crate::scheme::SchemeId; -use crate::sync::WaitQueue; -use crate::syscall::data::{Map, Packet}; -use crate::syscall::error::*; -use crate::syscall::flag::{EventFlags, EVENT_READ, O_NONBLOCK, PROT_READ, PROT_WRITE, MapFlags}; -use crate::syscall::number::*; -use crate::syscall::usercopy::{UserSlice, UserSliceWo, UserSliceRo}; +use crate::{ + context::{ + self, + context::HardBlockedReason, + file::{FileDescription, FileDescriptor}, + memory::{ + AddrSpace, BorrowedFmapSource, Grant, GrantFileRef, MmapMode, PageSpan, DANGLING, + }, + BorrowedHtBuf, Context, Status, + }, + event, + memory::Frame, + paging::{mapper::InactiveFlusher, Page, VirtualAddress, PAGE_SIZE}, + scheme::SchemeId, + sync::WaitQueue, + syscall::{ + data::{Map, Packet}, + error::*, + flag::{EventFlags, MapFlags, EVENT_READ, O_NONBLOCK, PROT_READ, PROT_WRITE}, + number::*, + usercopy::{UserSlice, UserSliceRo, UserSliceWo}, + }, +}; -use super::{FileHandle, OpenResult, KernelScheme, CallerCtx}; +use super::{CallerCtx, FileHandle, KernelScheme, OpenResult}; pub struct UserInner { root_id: SchemeId, @@ -41,7 +57,10 @@ pub struct UserInner { } enum State { - Waiting { context: Weak>, fd: Option>> }, + Waiting { + context: Weak>, + fd: Option>>, + }, Responded(Response), Fmap(Weak>), Placeholder, @@ -59,7 +78,14 @@ const ONE: NonZeroUsize = match NonZeroUsize::new(1) { }; impl UserInner { - pub fn new(root_id: SchemeId, scheme_id: SchemeId, handle_id: usize, name: Box, flags: usize, context: Weak>) -> UserInner { + pub fn new( + root_id: SchemeId, + scheme_id: SchemeId, + handle_id: usize, + name: Box, + flags: usize, + context: Weak>, + ) -> UserInner { UserInner { root_id, handle_id, @@ -109,20 +135,32 @@ impl UserInner { } } - pub fn call_extended(&self, ctx: CallerCtx, fd: Option>>, [a, b, c, d]: [usize; 4]) -> Result { - self.call_extended_inner(fd, Packet { - id: self.next_id(), - pid: ctx.pid, - uid: ctx.uid, - gid: ctx.gid, - a, - b, - c, - d - }) + pub fn call_extended( + &self, + ctx: CallerCtx, + fd: Option>>, + [a, b, c, d]: [usize; 4], + ) -> Result { + self.call_extended_inner( + fd, + Packet { + id: self.next_id(), + pid: ctx.pid, + uid: ctx.uid, + gid: ctx.gid, + a, + b, + c, + d, + }, + ) } - fn call_extended_inner(&self, fd: Option>>, packet: Packet) -> Result { + fn call_extended_inner( + &self, + fd: Option>>, + packet: Packet, + ) -> Result { if self.unmounting.load(Ordering::SeqCst) { return Err(Error::new(ENODEV)); } @@ -134,7 +172,13 @@ impl UserInner { { let mut states = self.states.lock(); current_context.write().block("UserScheme::call"); - states.insert(id, State::Waiting { context: Arc::downgrade(¤t_context), fd }); + states.insert( + id, + State::Waiting { + context: Arc::downgrade(¤t_context), + fd, + }, + ); } self.todo.send(packet); @@ -166,7 +210,7 @@ impl UserInner { o.remove(); return Ok(response); } - } + }, } } } @@ -174,14 +218,20 @@ impl UserInner { /// Map a readable structure to the scheme's userspace and return the /// pointer #[must_use = "copying back to head/tail buffers can fail"] - pub fn capture_user(&self, buf: UserSlice) -> Result> { - UserInner::capture_inner( - &self.context, - buf, - ) + pub fn capture_user( + &self, + buf: UserSlice, + ) -> Result> { + UserInner::capture_inner(&self.context, buf) } pub fn copy_and_capture_tail(&self, buf: &[u8]) -> Result> { - let dst_addr_space = Arc::clone(self.context.upgrade().ok_or(Error::new(ENODEV))?.read().addr_space()?); + let dst_addr_space = Arc::clone( + self.context + .upgrade() + .ok_or(Error::new(ENODEV))? + .read() + .addr_space()?, + ); let mut tail = BorrowedHtBuf::tail()?; let tail_frame = tail.frame(); @@ -191,7 +241,15 @@ impl UserInner { tail.buf_mut()[..buf.len()].copy_from_slice(buf); let is_pinned = true; - let dst_page = dst_addr_space.write().mmap_anywhere(ONE, PROT_READ, |dst_page, flags, mapper, flusher| Ok(Grant::allocated_shared_one_page(tail_frame, dst_page, flags, mapper, flusher, is_pinned)?))?; + let dst_page = dst_addr_space.write().mmap_anywhere( + ONE, + PROT_READ, + |dst_page, flags, mapper, flusher| { + Ok(Grant::allocated_shared_one_page( + tail_frame, dst_page, flags, mapper, flusher, is_pinned, + )?) + }, + )?; Ok(CaptureGuard { destroyed: false, @@ -202,7 +260,10 @@ impl UserInner { src: Some(tail), dst: None, }, - tail: CopyInfo { src: None, dst: None }, + tail: CopyInfo { + src: None, + dst: None, + }, }) } @@ -212,7 +273,10 @@ impl UserInner { /// Capture a buffer owned by userspace, mapping it contiguously onto scheme memory. // TODO: Hypothetical accept_head_leak, accept_tail_leak options might be useful for // libc-controlled buffer pools. - fn capture_inner(context_weak: &Weak>, user_buf: UserSlice) -> Result> { + fn capture_inner( + context_weak: &Weak>, + user_buf: UserSlice, + ) -> Result> { let (mode, map_flags) = match (READ, WRITE) { (true, false) => (Mode::Ro, PROT_READ), (false, true) => (Mode::Wo, PROT_WRITE), @@ -232,13 +296,25 @@ impl UserInner { base: DANGLING, len: 0, space: None, - head: CopyInfo { src: None, dst: None }, - tail: CopyInfo { src: None, dst: None }, + head: CopyInfo { + src: None, + dst: None, + }, + tail: CopyInfo { + src: None, + dst: None, + }, }); } let cur_space_lock = AddrSpace::current()?; - let dst_space_lock = Arc::clone(context_weak.upgrade().ok_or(Error::new(ESRCH))?.read().addr_space()?); + let dst_space_lock = Arc::clone( + context_weak + .upgrade() + .ok_or(Error::new(ESRCH))? + .read() + .addr_space()?, + ); if Arc::ptr_eq(&dst_space_lock, &cur_space_lock) { // Same address space, no need to remap anything! @@ -247,8 +323,14 @@ impl UserInner { base: user_buf.addr(), len: user_buf.len(), space: None, - head: CopyInfo { src: None, dst: None }, - tail: CopyInfo { src: None, dst: None }, + head: CopyInfo { + src: None, + dst: None, + }, + tail: CopyInfo { + src: None, + dst: None, + }, }); } @@ -261,7 +343,10 @@ impl UserInner { let mut dst_space = dst_space_lock.write(); - let free_span = dst_space.grants.find_free(dst_space.mmap_min, page_count).ok_or(Error::new(ENOMEM))?; + let free_span = dst_space + .grants + .find_free(dst_space.mmap_min, page_count) + .ok_or(Error::new(ENOMEM))?; let head = if !head_part_of_buf.is_empty() { // FIXME: Signal context can probably recursively use head/tail. @@ -276,19 +361,30 @@ impl UserInner { array.buf_mut()[offset + len..].fill(0_u8); let slice = &mut array.buf_mut()[offset..][..len]; - let head_part_of_buf = user_buf.limit(len).expect("always smaller than max len"); + let head_part_of_buf = + user_buf.limit(len).expect("always smaller than max len"); - head_part_of_buf.reinterpret_unchecked::().copy_to_slice(slice)?; + head_part_of_buf + .reinterpret_unchecked::() + .copy_to_slice(slice)?; } Mode::Wo => { array.buf_mut().fill(0_u8); } } - dst_space.mmap(Some(free_span.base), ONE, map_flags | MAP_FIXED_NOREPLACE, &mut Vec::new(), move |dst_page, page_flags, mapper, flusher| { - let is_pinned = true; - Ok(Grant::allocated_shared_one_page(frame, dst_page, page_flags, mapper, flusher, is_pinned)?) - })?; + dst_space.mmap( + Some(free_span.base), + ONE, + map_flags | MAP_FIXED_NOREPLACE, + &mut Vec::new(), + move |dst_page, page_flags, mapper, flusher| { + let is_pinned = true; + Ok(Grant::allocated_shared_one_page( + frame, dst_page, page_flags, mapper, flusher, is_pinned, + )?) + }, + )?; let head = CopyInfo { src: Some(array), @@ -302,30 +398,54 @@ impl UserInner { dst: None, } }; - let (first_middle_dst_page, first_middle_src_page) = if !head_part_of_buf.is_empty() { (free_span.base.next(), src_page.next()) } else { (free_span.base, src_page) }; + let (first_middle_dst_page, first_middle_src_page) = if !head_part_of_buf.is_empty() { + (free_span.base.next(), src_page.next()) + } else { + (free_span.base, src_page) + }; let middle_page_count = middle_tail_part_of_buf.len() / PAGE_SIZE; let tail_size = middle_tail_part_of_buf.len() % PAGE_SIZE; - let (_middle_part_of_buf, tail_part_of_buf) = middle_tail_part_of_buf.split_at(middle_page_count * PAGE_SIZE).expect("split must succeed"); + let (_middle_part_of_buf, tail_part_of_buf) = middle_tail_part_of_buf + .split_at(middle_page_count * PAGE_SIZE) + .expect("split must succeed"); if let Some(middle_page_count) = NonZeroUsize::new(middle_page_count) { - dst_space.mmap(Some(first_middle_dst_page), middle_page_count, map_flags | MAP_FIXED_NOREPLACE, &mut Vec::new(), move |dst_page, _, mapper, flusher| { - let eager = true; + dst_space.mmap( + Some(first_middle_dst_page), + middle_page_count, + map_flags | MAP_FIXED_NOREPLACE, + &mut Vec::new(), + move |dst_page, _, mapper, flusher| { + let eager = true; - // It doesn't make sense to allow a context, that has borrowed non-RAM physical - // memory, to DIRECTLY do scheme calls onto that memory. - // - // (TODO: Maybe there are some niche use cases for that, possibly PCI transfer - // BARs, but it doesn't make sense yet.) - let allow_phys = false; + // It doesn't make sense to allow a context, that has borrowed non-RAM physical + // memory, to DIRECTLY do scheme calls onto that memory. + // + // (TODO: Maybe there are some niche use cases for that, possibly PCI transfer + // BARs, but it doesn't make sense yet.) + let allow_phys = false; - // Deny any attempts by the scheme, to unmap these temporary pages. The only way to - // unmap them is to respond to the scheme socket. - let is_pinned_userscheme_borrow = true; + // Deny any attempts by the scheme, to unmap these temporary pages. The only way to + // unmap them is to respond to the scheme socket. + let is_pinned_userscheme_borrow = true; - Ok(Grant::borrow(Arc::clone(&cur_space_lock), &mut *cur_space_lock.write(), first_middle_src_page, dst_page, middle_page_count.get(), map_flags, mapper, flusher, eager, allow_phys, is_pinned_userscheme_borrow)?) - })?; + Ok(Grant::borrow( + Arc::clone(&cur_space_lock), + &mut *cur_space_lock.write(), + first_middle_src_page, + dst_page, + middle_page_count.get(), + map_flags, + mapper, + flusher, + eager, + allow_phys, + is_pinned_userscheme_borrow, + )?) + }, + )?; } let tail = if !tail_part_of_buf.is_empty() { @@ -342,17 +462,27 @@ impl UserInner { to_zero.fill(0_u8); // FIXME: remove reinterpret_unchecked - tail_part_of_buf.reinterpret_unchecked::().copy_to_slice(to_copy)?; + tail_part_of_buf + .reinterpret_unchecked::() + .copy_to_slice(to_copy)?; } Mode::Wo => { array.buf_mut().fill(0_u8); } } - dst_space.mmap(Some(tail_dst_page), ONE, map_flags | MAP_FIXED_NOREPLACE, &mut Vec::new(), move |dst_page, page_flags, mapper, flusher| { - let is_pinned = true; - Ok(Grant::allocated_shared_one_page(frame, dst_page, page_flags, mapper, flusher, is_pinned)?) - })?; + dst_space.mmap( + Some(tail_dst_page), + ONE, + map_flags | MAP_FIXED_NOREPLACE, + &mut Vec::new(), + move |dst_page, page_flags, mapper, flusher| { + let is_pinned = true; + Ok(Grant::allocated_shared_one_page( + frame, dst_page, page_flags, mapper, flusher, is_pinned, + )?) + }, + )?; CopyInfo { src: Some(array), @@ -407,7 +537,13 @@ impl UserInner { Ok(packets_read * size_of::()) } - pub fn request_fmap(&self, id: usize, offset: u64, required_page_count: usize, flags: MapFlags) -> Result<()> { + pub fn request_fmap( + &self, + id: usize, + offset: u64, + required_page_count: usize, + flags: MapFlags, + ) -> Result<()> { log::info!("REQUEST FMAP"); let packet_id = self.next_id(); @@ -432,8 +568,12 @@ impl UserInner { if packet.id == 0 { // TODO: Simplify logic by using SKMSG with packet.id being ignored? match packet.a { - SYS_FEVENT => event::trigger(self.scheme_id, packet.b, EventFlags::from_bits_truncate(packet.c)), - _ => log::warn!("Unknown scheme -> kernel message {}", packet.a) + SYS_FEVENT => event::trigger( + self.scheme_id, + packet.b, + EventFlags::from_bits_truncate(packet.c), + ), + _ => log::warn!("Unknown scheme -> kernel message {}", packet.a), } } else if Error::demux(packet.a) == Err(Error::new(ESKMSG)) { // The reason why the new ESKMSG mechanism was introduced, is that passing packet IDs @@ -442,13 +582,22 @@ impl UserInner { SKMSG_FRETURNFD => { let fd = packet.c; - let desc = context::current()?.read().remove_file(FileHandle::from(fd)).ok_or(Error::new(EINVAL))?.description; + let desc = context::current()? + .read() + .remove_file(FileHandle::from(fd)) + .ok_or(Error::new(EINVAL))? + .description; self.respond(packet.id, Response::Fd(desc))?; } SKMSG_FOBTAINFD => { let flags = FobtainFdFlags::from_bits(packet.d).ok_or(Error::new(EINVAL))?; - let description = match self.states.lock().get_mut(&packet.id).ok_or(Error::new(EINVAL))? { + let description = match self + .states + .lock() + .get_mut(&packet.id) + .ok_or(Error::new(EINVAL))? + { State::Waiting { ref mut fd, .. } => fd.take().ok_or(Error::new(ENOENT))?, _ => return Err(Error::new(ENOENT)), }; @@ -456,9 +605,21 @@ impl UserInner { // FIXME: Description can leak if context::current() fails, or if there is no // additional file table space. if flags.contains(FobtainFdFlags::MANUAL_FD) { - context::current()?.read().insert_file(FileHandle::from(packet.c), FileDescriptor { description, cloexec: true }); + context::current()?.read().insert_file( + FileHandle::from(packet.c), + FileDescriptor { + description, + cloexec: true, + }, + ); } else { - let fd = context::current()?.read().add_file(FileDescriptor { description, cloexec: true }).ok_or(Error::new(EMFILE))?; + let fd = context::current()? + .read() + .add_file(FileDescriptor { + description, + cloexec: true, + }) + .ok_or(Error::new(EMFILE))?; UserSlice::wo(packet.c, size_of::())?.write_usize(fd.get())?; } } @@ -477,38 +638,48 @@ impl UserInner { let page_count = packet.d; - if page_count != 1 { return Err(Error::new(EINVAL)); } + if page_count != 1 { + return Err(Error::new(EINVAL)); + } let context = match self.states.lock().entry(packet.id) { - Entry::Occupied(mut o) => match mem::replace(o.get_mut(), State::Placeholder) { - // invalid state - State::Placeholder => { - return Err(Error::new(EBADFD)); + Entry::Occupied(mut o) => { + match mem::replace(o.get_mut(), State::Placeholder) { + // invalid state + State::Placeholder => { + return Err(Error::new(EBADFD)); + } + // invalid kernel to scheme call + old_state @ (State::Waiting { .. } | State::Responded(_)) => { + *o.get_mut() = old_state; + return Err(Error::new(EINVAL)); + } + State::Fmap(context) => { + o.remove(); + context + } } - // invalid kernel to scheme call - old_state @ (State::Waiting { .. } | State::Responded(_)) => { - *o.get_mut() = old_state; - return Err(Error::new(EINVAL)); - } - State::Fmap(context) => { - o.remove(); - context - } - }, + } Entry::Vacant(_) => return Err(Error::new(EINVAL)), }; let context = context.upgrade().ok_or(Error::new(ESRCH))?; - let (frame, _) = AddrSpace::current()?.read().table.utable.translate(base_addr).ok_or(Error::new(EFAULT))?; + let (frame, _) = AddrSpace::current()? + .read() + .table + .utable + .translate(base_addr) + .ok_or(Error::new(EFAULT))?; let mut context = context.write(); match context.status { - Status::HardBlocked { reason: HardBlockedReason::AwaitingMmap { .. } } => context.status = Status::Runnable, + Status::HardBlocked { + reason: HardBlockedReason::AwaitingMmap { .. }, + } => context.status = Status::Runnable, _ => (), } context.fmap_ret = Some(Frame::containing_address(frame)); - } _ => return Err(Error::new(EINVAL)), } @@ -532,7 +703,9 @@ impl UserInner { } State::Waiting { context, fd } => { - to_close = fd.and_then(|f| Arc::try_unwrap(f).ok()).map(RwLock::into_inner); + to_close = fd + .and_then(|f| Arc::try_unwrap(f).ok()) + .map(RwLock::into_inner); if let Some(context) = context.upgrade() { context.write().unblock(); @@ -541,7 +714,7 @@ impl UserInner { o.remove(); } } - } + }, // invalid state Entry::Vacant(_) => return Err(Error::new(EBADFD)), } @@ -560,7 +733,12 @@ impl UserInner { Ok(()) } - fn fmap_inner(&self, dst_addr_space: Arc>, file: usize, map: &Map) -> Result { + fn fmap_inner( + &self, + dst_addr_space: Arc>, + file: usize, + map: &Map, + ) -> Result { let unaligned_size = map.size; if unaligned_size == 0 { @@ -572,15 +750,19 @@ impl UserInner { if map.address % PAGE_SIZE != 0 { return Err(Error::new(EINVAL)); }; - let dst_base = (map.address != 0).then_some(Page::containing_address(VirtualAddress::new(map.address))); + let dst_base = (map.address != 0) + .then_some(Page::containing_address(VirtualAddress::new(map.address))); if map.offset % PAGE_SIZE != 0 { return Err(Error::new(EINVAL)); } let src_address_space = Arc::clone( - self.context.upgrade().ok_or(Error::new(ENODEV))? - .read().addr_space()? + self.context + .upgrade() + .ok_or(Error::new(ENODEV))? + .read() + .addr_space()?, ); if Arc::ptr_eq(&src_address_space, &dst_addr_space) { return Err(Error::new(EBUSY)); @@ -605,20 +787,23 @@ impl UserInner { (context.id, desc.description) }; - let response = self.call_extended_inner(None, Packet { - id: self.next_id(), - pid: pid.into(), - a: KSMSG_MMAP_PREP, - b: file, - c: unaligned_size, - d: map.flags.bits(), - // The uid and gid can be obtained by the proc scheme anyway, if the pid is provided. - uid: map.offset as u32, - #[cfg(target_pointer_width = "64")] - gid: (map.offset >> 32) as u32, - #[cfg(target_pointer_width = "32")] - gid: 0, - })?; + let response = self.call_extended_inner( + None, + Packet { + id: self.next_id(), + pid: pid.into(), + a: KSMSG_MMAP_PREP, + b: file, + c: unaligned_size, + d: map.flags.bits(), + // The uid and gid can be obtained by the proc scheme anyway, if the pid is provided. + uid: map.offset as u32, + #[cfg(target_pointer_width = "64")] + gid: (map.offset >> 32) as u32, + #[cfg(target_pointer_width = "32")] + gid: 0, + }, + )?; // TODO: I've previously tested that this works, but because the scheme trait all of // Redox's schemes currently rely on doesn't allow one-way messages, there's no current @@ -628,8 +813,7 @@ impl UserInner { let mapping_is_lazy = false; let base_page_opt = match response { - Response::Regular(code) => (!mapping_is_lazy) - .then_some(Error::demux(code)?), + Response::Regular(code) => (!mapping_is_lazy).then_some(Error::demux(code)?), Response::Fd(_) => { log::debug!("Scheme incorrectly returned an fd for fmap."); @@ -661,7 +845,7 @@ impl UserInner { flusher: { flusher = InactiveFlusher::new(); &mut flusher - } + }, } }), None => None, @@ -669,9 +853,22 @@ impl UserInner { let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0"); let mut notify_files = Vec::new(); - let dst_base = dst_addr_space.write().mmap(dst_base, page_count_nz, map.flags, &mut notify_files, |dst_base, flags, mapper, flusher| { - Grant::borrow_fmap(PageSpan::new(dst_base, page_count), flags, file_ref, src, mapper, flusher) - })?; + let dst_base = dst_addr_space.write().mmap( + dst_base, + page_count_nz, + map.flags, + &mut notify_files, + |dst_base, flags, mapper, flusher| { + Grant::borrow_fmap( + PageSpan::new(dst_base, page_count), + flags, + file_ref, + src, + mapper, + flusher, + ) + }, + )?; for map in notify_files { let _ = map.unmap(); @@ -691,8 +888,12 @@ pub struct CaptureGuard { tail: CopyInfo, } impl CaptureGuard { - fn base(&self) -> usize { self.base } - fn len(&self) -> usize { self.len } + fn base(&self) -> usize { + self.base + } + fn len(&self) -> usize { + self.len + } } struct CopyInfo { src: Option, @@ -714,10 +915,20 @@ impl CaptureGuard { let mut result = Ok(()); // TODO: Encode src and dst better using const generics. - if let CopyInfo { src: Some(ref src), dst: Some(ref mut dst) } = self.head { - result = result.and_then(|()| dst.copy_from_slice(&src.buf()[self.base % PAGE_SIZE..][..dst.len()])); + if let CopyInfo { + src: Some(ref src), + dst: Some(ref mut dst), + } = self.head + { + result = result.and_then(|()| { + dst.copy_from_slice(&src.buf()[self.base % PAGE_SIZE..][..dst.len()]) + }); } - if let CopyInfo { src: Some(ref src), dst: Some(ref mut dst) } = self.tail { + if let CopyInfo { + src: Some(ref src), + dst: Some(ref mut dst), + } = self.tail + { result = result.and_then(|()| dst.copy_from_slice(&src.buf()[..dst.len()])); } let Some(space) = self.space.take() else { @@ -727,7 +938,9 @@ impl CaptureGuard { let (first_page, page_count, _offset) = page_range_containing(self.base, self.len); let unpin = true; - space.write().munmap(PageSpan::new(first_page, page_count), unpin)?; + space + .write() + .munmap(PageSpan::new(first_page, page_count), unpin)?; result } @@ -751,7 +964,7 @@ fn page_range_containing(base: usize, size: usize) -> (Page, usize, usize) { /// `UserInner` has to be wrapped #[derive(Clone)] pub struct UserScheme { - pub(crate) inner: Weak + pub(crate) inner: Weak, } impl UserScheme { @@ -818,7 +1031,9 @@ impl KernelScheme for UserScheme { fn fevent(&self, file: usize, flags: EventFlags) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - inner.call(SYS_FEVENT, file, flags.bits(), 0).map(EventFlags::from_bits_truncate) + inner + .call(SYS_FEVENT, file, flags.bits(), 0) + .map(EventFlags::from_bits_truncate) } /* @@ -946,7 +1161,13 @@ impl KernelScheme for UserScheme { address.release()?; result.map(|_| ()) } - fn kfmap(&self, file: usize, addr_space: &Arc>, map: &Map, _consume: bool) -> Result { + fn kfmap( + &self, + file: usize, + addr_space: &Arc>, + map: &Map, + _consume: bool, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; inner.fmap_inner(Arc::clone(addr_space), file, map) @@ -954,31 +1175,44 @@ impl KernelScheme for UserScheme { fn kfunmap(&self, number: usize, offset: usize, size: usize, flags: MunmapFlags) -> Result<()> { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let res = inner.call_extended(CallerCtx { - pid: context::context_id().into(), - uid: offset as u32, - #[cfg(target_pointer_width = "64")] - gid: (offset >> 32) as u32, + let res = inner.call_extended( + CallerCtx { + pid: context::context_id().into(), + uid: offset as u32, + #[cfg(target_pointer_width = "64")] + gid: (offset >> 32) as u32, - // TODO: saturating_shr? - #[cfg(not(target_pointer_width = "64"))] - gid: 0, - - }, None, [KSMSG_MUNMAP, number, size, flags.bits()])?; + // TODO: saturating_shr? + #[cfg(not(target_pointer_width = "64"))] + gid: 0, + }, + None, + [KSMSG_MUNMAP, number, size, flags.bits()], + )?; match res { Response::Regular(_) => Ok(()), Response::Fd(_) => Err(Error::new(EIO)), } } - fn ksendfd(&self, number: usize, desc: Arc>, flags: SendFdFlags, arg: u64) -> Result { + fn ksendfd( + &self, + number: usize, + desc: Arc>, + flags: SendFdFlags, + arg: u64, + ) -> Result { let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - let res = inner.call_extended(CallerCtx { - pid: context::context_id().into(), - uid: arg as u32, - gid: (arg >> 32) as u32, - }, Some(desc), [SYS_SENDFD, number, flags.bits(), 0])?; + let res = inner.call_extended( + CallerCtx { + pid: context::context_id().into(), + uid: arg as u32, + gid: (arg >> 32) as u32, + }, + Some(desc), + [SYS_SENDFD, number, flags.bits(), 0], + )?; match res { Response::Regular(res) => Ok(res), diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 833925b17c..f4ece62255 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,7 +1,5 @@ -pub use self::wait_condition::WaitCondition; -pub use self::wait_queue::WaitQueue; -pub use self::wait_map::WaitMap; +pub use self::{wait_condition::WaitCondition, wait_map::WaitMap, wait_queue::WaitQueue}; pub mod wait_condition; -pub mod wait_queue; pub mod wait_map; +pub mod wait_queue; diff --git a/src/sync/wait_condition.rs b/src/sync/wait_condition.rs index b4f0e5cefd..002560ad88 100644 --- a/src/sync/wait_condition.rs +++ b/src/sync/wait_condition.rs @@ -1,18 +1,17 @@ -use alloc::sync::Arc; -use alloc::vec::Vec; +use alloc::{sync::Arc, vec::Vec}; use spin::{Mutex, MutexGuard, RwLock}; use crate::context::{self, Context}; #[derive(Debug)] pub struct WaitCondition { - contexts: Mutex>>> + contexts: Mutex>>>, } impl WaitCondition { pub const fn new() -> WaitCondition { WaitCondition { - contexts: Mutex::new(Vec::new()) + contexts: Mutex::new(Vec::new()), } } @@ -57,7 +56,9 @@ impl WaitCondition { drop(guard); } - unsafe { context::switch(); } + unsafe { + context::switch(); + } let mut waited = true; @@ -86,7 +87,7 @@ impl WaitCondition { } impl Drop for WaitCondition { - fn drop(&mut self){ + fn drop(&mut self) { unsafe { self.notify_signal() }; } } diff --git a/src/sync/wait_map.rs b/src/sync/wait_map.rs index b21acf8520..9f15d0bbd5 100644 --- a/src/sync/wait_map.rs +++ b/src/sync/wait_map.rs @@ -8,14 +8,17 @@ use crate::sync::WaitCondition; pub struct WaitMap { // Using BTreeMap as this depends on .keys() providing elements in sorted order. pub inner: Mutex>, - pub condition: WaitCondition + pub condition: WaitCondition, } -impl WaitMap where K: Clone + Ord { +impl WaitMap +where + K: Clone + Ord, +{ pub fn new() -> WaitMap { WaitMap { inner: Mutex::new(BTreeMap::new()), - condition: WaitCondition::new() + condition: WaitCondition::new(), } } diff --git a/src/sync/wait_queue.rs b/src/sync/wait_queue.rs index 83a98265cc..8e626cb4b3 100644 --- a/src/sync/wait_queue.rs +++ b/src/sync/wait_queue.rs @@ -2,9 +2,13 @@ use alloc::collections::VecDeque; use spin::Mutex; use syscall::{EAGAIN, EINTR}; -use crate::sync::WaitCondition; -use crate::syscall::usercopy::UserSliceWo; -use crate::syscall::error::{Error, EINVAL, Result}; +use crate::{ + sync::WaitCondition, + syscall::{ + error::{Error, Result, EINVAL}, + usercopy::UserSliceWo, + }, +}; #[derive(Debug)] pub struct WaitQueue { @@ -16,11 +20,16 @@ impl WaitQueue { pub const fn new() -> WaitQueue { WaitQueue { inner: Mutex::new(VecDeque::new()), - condition: WaitCondition::new() + condition: WaitCondition::new(), } } - pub fn receive_into_user(&self, buf: UserSliceWo, block: bool, reason: &'static str) -> Result { + pub fn receive_into_user( + &self, + buf: UserSliceWo, + block: bool, + reason: &'static str, + ) -> Result { loop { let mut inner = self.inner.lock(); @@ -41,8 +50,18 @@ impl WaitQueue { } let (s1, s2) = inner.as_slices(); - let s1_bytes = unsafe { core::slice::from_raw_parts(s1.as_ptr().cast::(), s1.len() * core::mem::size_of::()) }; - let s2_bytes = unsafe { core::slice::from_raw_parts(s2.as_ptr().cast::(), s2.len() * core::mem::size_of::()) }; + let s1_bytes = unsafe { + core::slice::from_raw_parts( + s1.as_ptr().cast::(), + s1.len() * core::mem::size_of::(), + ) + }; + let s2_bytes = unsafe { + core::slice::from_raw_parts( + s2.as_ptr().cast::(), + s2.len() * core::mem::size_of::(), + ) + }; let mut bytes_copied = buf.copy_common_bytes_from_slice(s1_bytes)?; diff --git a/src/syscall/debug.rs b/src/syscall/debug.rs index 020758ec85..d3dac3857a 100644 --- a/src/syscall/debug.rs +++ b/src/syscall/debug.rs @@ -1,15 +1,17 @@ +use alloc::{string::String, vec::Vec}; use core::{ascii, mem}; -use alloc::string::String; -use alloc::vec::Vec; -use super::data::{Map, Stat, TimeSpec}; -use super::{flag::*, copy_path_to_buf}; -use super::number::*; -use super::usercopy::UserSlice; +use super::{ + copy_path_to_buf, + data::{Map, Stat, TimeSpec}, + flag::*, + number::*, + usercopy::UserSlice, +}; use crate::syscall::error::Result; -struct ByteStr<'a>(&'a[u8]); +struct ByteStr<'a>(&'a [u8]); impl<'a> ::core::fmt::Debug for ByteStr<'a> { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { @@ -29,7 +31,7 @@ fn debug_path(ptr: usize, len: usize) -> Result { } fn debug_buf(ptr: usize, len: usize) -> Result> { UserSlice::ro(ptr, len).and_then(|user| { - let mut buf = vec! [0_u8; 4096]; + let mut buf = vec![0_u8; 4096]; let count = user.copy_common_bytes_to_slice(&mut buf)?; buf.truncate(count); Ok(buf) @@ -55,9 +57,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - "unlink({:?})", debug_path(b, c).as_ref().map(|p| ByteStr(p.as_bytes())), ), - SYS_CLOSE => format!( - "close({})", b - ), + SYS_CLOSE => format!("close({})", b), SYS_DUP => format!( "dup({}, {:?})", b, @@ -69,26 +69,9 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - c, debug_buf(d, e).as_ref().map(|b| ByteStr(&*b)), ), - SYS_SENDFD => format!( - "sendfd({}, {}, {:#0x} {:#0x} {:#0x})", - b, - c, - d, - e, - f, - ), - SYS_READ => format!( - "read({}, {:#X}, {})", - b, - c, - d - ), - SYS_WRITE => format!( - "write({}, {:#X}, {})", - b, - c, - d - ), + SYS_SENDFD => format!("sendfd({}, {}, {:#0x} {:#0x} {:#0x})", b, c, d, e, f,), + SYS_READ => format!("read({}, {:#X}, {})", b, c, d), + SYS_WRITE => format!("write({}, {:#X}, {})", b, c, d), SYS_LSEEK => format!( "lseek({}, {}, {} ({}))", b, @@ -97,21 +80,12 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - SEEK_SET => "SEEK_SET", SEEK_CUR => "SEEK_CUR", SEEK_END => "SEEK_END", - _ => "UNKNOWN" + _ => "UNKNOWN", }, d ), - SYS_FCHMOD => format!( - "fchmod({}, {:#o})", - b, - c - ), - SYS_FCHOWN => format!( - "fchown({}, {}, {})", - b, - c, - d - ), + SYS_FCHMOD => format!("fchmod({}, {:#o})", b, c), + SYS_FCHOWN => format!("fchown({}, {}, {})", b, c, d), SYS_FCNTL => format!( "fcntl({}, {} ({}), {:#X})", b, @@ -121,7 +95,7 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - F_SETFD => "F_SETFD", F_SETFL => "F_SETFL", F_GETFL => "F_GETFL", - _ => "UNKNOWN" + _ => "UNKNOWN", }, c, d @@ -131,47 +105,22 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - b, UserSlice::ro(c, d).and_then(|buf| unsafe { buf.read_exact::() }), ), - SYS_FUNMAP => format!( - "funmap({:#X}, {:#X})", - b, - c, - ), - SYS_FPATH => format!( - "fpath({}, {:#X}, {})", - b, - c, - d - ), - SYS_FRENAME => format!( - "frename({}, {:?})", - b, - debug_path(c, d), - ), + SYS_FUNMAP => format!("funmap({:#X}, {:#X})", b, c,), + SYS_FPATH => format!("fpath({}, {:#X}, {})", b, c, d), + SYS_FRENAME => format!("frename({}, {:?})", b, debug_path(c, d),), SYS_FSTAT => format!( "fstat({}, {:?})", b, UserSlice::ro(c, d).and_then(|buf| unsafe { buf.read_exact::() }), ), - SYS_FSTATVFS => format!( - "fstatvfs({}, {:#X}, {})", - b, - c, - d - ), - SYS_FSYNC => format!( - "fsync({})", - b - ), - SYS_FTRUNCATE => format!( - "ftruncate({}, {})", - b, - c - ), + SYS_FSTATVFS => format!("fstatvfs({}, {:#X}, {})", b, c, d), + SYS_FSYNC => format!("fsync({})", b), + SYS_FTRUNCATE => format!("ftruncate({}, {})", b, c), SYS_FUTIMENS => format!( "futimens({}, {:?})", b, UserSlice::ro(c, d).and_then(|buf| { - let mut times = vec! [unsafe { buf.read_exact::()? }]; + let mut times = vec![unsafe { buf.read_exact::()? }]; // One or two timespecs if let Some(second) = buf.advance(mem::size_of::()) { @@ -181,15 +130,10 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - }), ), - SYS_CLOCK_GETTIME => format!( - "clock_gettime({}, {:?})", - b, - unsafe { read_struct::(c) } - ), - SYS_EXIT => format!( - "exit({})", - b - ), + SYS_CLOCK_GETTIME => format!("clock_gettime({}, {:?})", b, unsafe { + read_struct::(c) + }), + SYS_EXIT => format!("exit({})", b), SYS_FUTEX => format!( "futex({:#X} [{:?}], {}, {}, {}, {})", b, @@ -208,23 +152,10 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - SYS_GETPID => format!("getpid()"), SYS_GETPPID => format!("getppid()"), SYS_GETUID => format!("getuid()"), - SYS_IOPL => format!( - "iopl({})", - b - ), - SYS_KILL => format!( - "kill({}, {})", - b, - c - ), + SYS_IOPL => format!("iopl({})", b), + SYS_KILL => format!("kill({}, {})", b, c), SYS_SIGRETURN => format!("sigreturn()"), - SYS_SIGACTION => format!( - "sigaction({}, {:#X}, {:#X}, {:#X})", - b, - c, - d, - e - ), + SYS_SIGACTION => format!("sigaction({}, {:#X}, {:#X}, {:#X})", b, c, d, e), SYS_SIGPROCMASK => format!( "sigprocmask({}, {:?}, {:?})", b, @@ -240,56 +171,23 @@ pub fn format_call(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) - b as *const u8, c, ), - SYS_MPROTECT => format!( - "mprotect({:#X}, {}, {:?})", - b, - c, - MapFlags::from_bits(d) - ), + SYS_MPROTECT => format!("mprotect({:#X}, {}, {:?})", b, c, MapFlags::from_bits(d)), SYS_NANOSLEEP => format!( "nanosleep({:?}, ({}, {}))", unsafe { read_struct::(b) }, c, d ), - SYS_VIRTTOPHYS => format!( - "virttophys({:#X})", - b - ), - SYS_SETREGID => format!( - "setregid({}, {})", - b, - c - ), - SYS_SETRENS => format!( - "setrens({}, {})", - b, - c - ), - SYS_SETREUID => format!( - "setreuid({}, {})", - b, - c - ), - SYS_UMASK => format!( - "umask({:#o}", - b - ), - SYS_WAITPID => format!( - "waitpid({}, {:#X}, {:?})", - b, - c, - WaitFlags::from_bits(d) - ), + SYS_VIRTTOPHYS => format!("virttophys({:#X})", b), + SYS_SETREGID => format!("setregid({}, {})", b, c), + SYS_SETRENS => format!("setrens({}, {})", b, c), + SYS_SETREUID => format!("setreuid({}, {})", b, c), + SYS_UMASK => format!("umask({:#o}", b), + SYS_WAITPID => format!("waitpid({}, {:#X}, {:?})", b, c, WaitFlags::from_bits(d)), SYS_YIELD => format!("yield()"), _ => format!( "UNKNOWN{} {:#X}({:#X}, {:#X}, {:#X}, {:#X}, {:#X})", - a, a, - b, - c, - d, - e, - f - ) + a, a, b, c, d, e, f + ), } } diff --git a/src/syscall/driver.rs b/src/syscall/driver.rs index 9baeb9c08e..60e17a8272 100644 --- a/src/syscall/driver.rs +++ b/src/syscall/driver.rs @@ -1,9 +1,11 @@ use alloc::sync::Arc; -use crate::interrupt::InterruptStack; -use crate::paging::VirtualAddress; -use crate::context; -use crate::syscall::error::{Error, EFAULT, EINVAL, EPERM, ESRCH, Result}; +use crate::{ + context, + interrupt::InterruptStack, + paging::VirtualAddress, + syscall::error::{Error, Result, EFAULT, EINVAL, EPERM, ESRCH}, +}; fn enforce_root() -> Result<()> { let contexts = context::contexts(); let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; @@ -52,8 +54,12 @@ pub fn virttophys(virtual_address: usize) -> Result { let addr_space = Arc::clone(context::current()?.read().addr_space()?); let addr_space = addr_space.read(); - match addr_space.table.utable.translate(VirtualAddress::new(virtual_address)) { + match addr_space + .table + .utable + .translate(VirtualAddress::new(virtual_address)) + { Some((physical_address, _)) => Ok(physical_address.data()), - None => Err(Error::new(EFAULT)) + None => Err(Error::new(EFAULT)), } } diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index 278b2c072d..23339c463f 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -1,34 +1,55 @@ //! Filesystem syscalls -use alloc::sync::Arc; -use alloc::vec::Vec; +use alloc::{sync::Arc, vec::Vec}; use spin::RwLock; -use crate::context::file::{FileDescriptor, FileDescription}; -use crate::context; -use crate::context::memory::{PageSpan, AddrSpace}; -use crate::paging::{PAGE_SIZE, VirtualAddress, Page}; -use crate::scheme::{self, FileHandle, OpenResult, KernelScheme, SchemeId, CallerCtx}; -use crate::syscall::data::Stat; -use crate::syscall::error::*; -use crate::syscall::flag::*; +use crate::{ + context, + context::{ + file::{FileDescription, FileDescriptor}, + memory::{AddrSpace, PageSpan}, + }, + paging::{Page, VirtualAddress, PAGE_SIZE}, + scheme::{self, CallerCtx, FileHandle, KernelScheme, OpenResult, SchemeId}, + syscall::{data::Stat, error::*, flag::*}, +}; -use super::usercopy::{UserSlice, UserSliceWo, UserSliceRo}; +use super::usercopy::{UserSlice, UserSliceRo, UserSliceWo}; -pub fn file_op_generic(fd: FileHandle, op: impl FnOnce(&dyn KernelScheme, &CallerCtx, usize) -> Result) -> Result { +pub fn file_op_generic( + fd: FileHandle, + op: impl FnOnce(&dyn KernelScheme, &CallerCtx, usize) -> Result, +) -> Result { file_op_generic_ext(fd, |s, _, ctx, no| op(s, ctx, no)) } -pub fn file_op_generic_ext(fd: FileHandle, op: impl FnOnce(&dyn KernelScheme, SchemeId, &CallerCtx, usize) -> Result) -> Result { +pub fn file_op_generic_ext( + fd: FileHandle, + op: impl FnOnce(&dyn KernelScheme, SchemeId, &CallerCtx, usize) -> Result, +) -> Result { let (ctx, file) = match context::current()?.read() { - ref context => (CallerCtx { pid: context.id.into(), uid: context.euid, gid: context.egid }, context.get_file(fd).ok_or(Error::new(EBADF))?), + ref context => ( + CallerCtx { + pid: context.id.into(), + uid: context.euid, + gid: context.egid, + }, + context.get_file(fd).ok_or(Error::new(EBADF))?, + ), }; - let FileDescription { scheme: scheme_id, number, .. } = *file.description.read(); + let FileDescription { + scheme: scheme_id, + number, + .. + } = *file.description.read(); - let scheme = scheme::schemes().get(scheme_id).ok_or(Error::new(EBADF))?.clone(); + let scheme = scheme::schemes() + .get(scheme_id) + .ok_or(Error::new(EBADF))? + .clone(); op(&*scheme, scheme_id, &ctx, number) } pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result { - let mut path_buf = vec! [0_u8; max_len]; + let mut path_buf = vec![0_u8; max_len]; if raw_path.len() > path_buf.len() { return Err(Error::new(ENAMETOOLONG)); } @@ -43,7 +64,13 @@ const PATH_MAX: usize = PAGE_SIZE; /// Open syscall pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { let (pid, uid, gid, scheme_ns, umask) = match context::current()?.read() { - ref context => (context.id.into(), context.euid, context.egid, context.ens, context.umask), + ref context => ( + context.id.into(), + context.euid, + context.egid, + context.ens, + context.umask, + ), }; let flags = (flags & (!0o777)) | ((flags & 0o777) & (!(umask & 0o777))); @@ -63,7 +90,9 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { let description = { let (scheme_id, scheme) = { let schemes = scheme::schemes(); - let (scheme_id, scheme) = schemes.get_name(scheme_ns, scheme_name).ok_or(Error::new(ENODEV))?; + let (scheme_id, scheme) = schemes + .get_name(scheme_ns, scheme_name) + .ok_or(Error::new(ENODEV))?; (scheme_id, scheme.clone()) }; @@ -79,10 +108,13 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result { }; //drop(path_buf); - context::current()?.read().add_file(FileDescriptor { - description, - cloexec: flags & O_CLOEXEC == O_CLOEXEC, - }).ok_or(Error::new(EMFILE)) + context::current()? + .read() + .add_file(FileDescriptor { + description, + cloexec: flags & O_CLOEXEC == O_CLOEXEC, + }) + .ok_or(Error::new(EMFILE)) } /// rmdir syscall @@ -103,7 +135,9 @@ pub fn rmdir(raw_path: UserSliceRo) -> Result<()> { let scheme = { let schemes = scheme::schemes(); - let (_scheme_id, scheme) = schemes.get_name(scheme_ns, scheme_name).ok_or(Error::new(ENODEV))?; + let (_scheme_id, scheme) = schemes + .get_name(scheme_ns, scheme_name) + .ok_or(Error::new(ENODEV))?; scheme.clone() }; scheme.rmdir(reference, caller_ctx) @@ -126,7 +160,9 @@ pub fn unlink(raw_path: UserSliceRo) -> Result<()> { let scheme = { let schemes = scheme::schemes(); - let (_scheme_id, scheme) = schemes.get_name(scheme_ns, scheme_name).ok_or(Error::new(ENODEV))?; + let (_scheme_id, scheme) = schemes + .get_name(scheme_ns, scheme_name) + .ok_or(Error::new(ENODEV))?; scheme.clone() }; scheme.unlink(reference, caller_ctx) @@ -146,7 +182,10 @@ pub fn close(fd: FileHandle) -> Result<()> { fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result { let (file, caller_ctx) = match context::current()?.read() { - ref context => (context.get_file(fd).ok_or(Error::new(EBADF))?, context.caller_ctx()), + ref context => ( + context.get_file(fd).ok_or(Error::new(EBADF))?, + context.caller_ctx(), + ), }; if user_buf.is_empty() { @@ -159,7 +198,8 @@ fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result Result Result { let new_file = duplicate_file(fd, buf)?; - context::current()?.read().add_file(new_file).ok_or(Error::new(EMFILE)) + context::current()? + .read() + .add_file(new_file) + .ok_or(Error::new(EMFILE)) } /// Duplicate file descriptor, replacing another @@ -199,7 +242,9 @@ pub fn dup2(fd: FileHandle, new_fd: FileHandle, buf: UserSliceRo) -> Result Result { @@ -211,12 +256,27 @@ pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> // TODO: Ensure deadlocks can't happen - let (scheme, number) = match current.get_file(socket).ok_or(Error::new(EBADF))?.description.read() { + let (scheme, number) = match current + .get_file(socket) + .ok_or(Error::new(EBADF))? + .description + .read() + { ref desc => (desc.scheme, desc.number), }; - let scheme = scheme::schemes().get(scheme).ok_or(Error::new(ENODEV))?.clone(); + let scheme = scheme::schemes() + .get(scheme) + .ok_or(Error::new(ENODEV))? + .clone(); - (scheme, number, current.remove_file(fd).ok_or(Error::new(EBADF))?.description) + ( + scheme, + number, + current + .remove_file(fd) + .ok_or(Error::new(EBADF))? + .description, + ) }; // Inform the scheme whether there are still references to the file description to be sent, @@ -249,7 +309,8 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { // Communicate fcntl with scheme if cmd != F_DUPFD && cmd != F_GETFD && cmd != F_SETFD { let scheme = scheme::schemes() - .get(description.scheme).ok_or(Error::new(EBADF))? + .get(description.scheme) + .ok_or(Error::new(EBADF))? .clone(); scheme.fcntl(description.number, cmd, arg)?; @@ -265,7 +326,8 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; let context = context_lock.read(); - return context.add_file_min(new_file, arg) + return context + .add_file_min(new_file, arg) .ok_or(Error::new(EMFILE)) .map(FileHandle::into); } @@ -283,32 +345,36 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result { } else { Ok(0) } - }, + } F_SETFD => { file.cloexec = arg & O_CLOEXEC == O_CLOEXEC; Ok(0) - }, - F_GETFL => { - Ok(description.flags) - }, + } + F_GETFL => Ok(description.flags), F_SETFL => { - let new_flags = (description.flags & O_ACCMODE) | (arg & ! O_ACCMODE); + let new_flags = (description.flags & O_ACCMODE) | (arg & !O_ACCMODE); drop(description); file.description.write().flags = new_flags; Ok(0) - }, - _ => { - Err(Error::new(EINVAL)) } + _ => Err(Error::new(EINVAL)), }, - None => Err(Error::new(EBADF)) + None => Err(Error::new(EBADF)), } } } pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { let (file, caller_ctx, scheme_ns) = match context::current()?.read() { - ref context => (context.get_file(fd).ok_or(Error::new(EBADF))?, CallerCtx { uid: context.euid, gid: context.egid, pid: context.id.get() }, context.ens), + ref context => ( + context.get_file(fd).ok_or(Error::new(EBADF))?, + CallerCtx { + uid: context.euid, + gid: context.egid, + pid: context.id.get(), + }, + context.ens, + ), }; /* @@ -323,7 +389,9 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> { let (scheme_id, scheme) = { let schemes = scheme::schemes(); - let (scheme_id, scheme) = schemes.get_name(scheme_ns, scheme_name).ok_or(Error::new(ENODEV))?; + let (scheme_id, scheme) = schemes + .get_name(scheme_ns, scheme_name) + .ok_or(Error::new(ENODEV))?; (scheme_id, scheme.clone()) }; @@ -344,8 +412,15 @@ pub fn fstat(fd: FileHandle, user_buf: UserSliceWo) -> Result<()> { // TODO: Ensure only the kernel can access the stat when st_dev is set, or use another API // for retrieving the scheme ID from a file descriptor. // TODO: Less hacky method. - let st_dev = scheme_id.get().try_into().map_err(|_| Error::new(EOVERFLOW))?; - user_buf.advance(core::mem::offset_of!(Stat, st_dev)).and_then(|b| b.limit(8)).ok_or(Error::new(EIO))?.copy_from_slice(&u64::to_ne_bytes(st_dev))?; + let st_dev = scheme_id + .get() + .try_into() + .map_err(|_| Error::new(EOVERFLOW))?; + user_buf + .advance(core::mem::offset_of!(Stat, st_dev)) + .and_then(|b| b.limit(8)) + .ok_or(Error::new(EIO))? + .copy_from_slice(&u64::to_ne_bytes(st_dev))?; Ok(()) }) @@ -360,11 +435,16 @@ pub fn funmap(virtual_address: usize, length: usize) -> Result { let length_aligned = length.next_multiple_of(PAGE_SIZE); if length != length_aligned { - log::warn!("funmap passed length {:#x} instead of {:#x}", length, length_aligned); + log::warn!( + "funmap passed length {:#x} instead of {:#x}", + length, + length_aligned + ); } let addr_space = Arc::clone(context::current()?.read().addr_space()?); - let span = PageSpan::validate_nonempty(VirtualAddress::new(virtual_address), length_aligned).ok_or(Error::new(EINVAL))?; + let span = PageSpan::validate_nonempty(VirtualAddress::new(virtual_address), length_aligned) + .ok_or(Error::new(EINVAL))?; let unpin = false; let notify = addr_space.write().munmap(span, unpin)?; @@ -375,8 +455,18 @@ pub fn funmap(virtual_address: usize, length: usize) -> Result { Ok(0) } -pub fn mremap(old_address: usize, old_size: usize, new_address: usize, new_size: usize, flags: usize) -> Result { - if old_address % PAGE_SIZE != 0 || old_size % PAGE_SIZE != 0 || new_address % PAGE_SIZE != 0 || new_size % PAGE_SIZE != 0 { +pub fn mremap( + old_address: usize, + old_size: usize, + new_address: usize, + new_size: usize, + flags: usize, +) -> Result { + if old_address % PAGE_SIZE != 0 + || old_size % PAGE_SIZE != 0 + || new_address % PAGE_SIZE != 0 + || new_size % PAGE_SIZE != 0 + { return Err(Error::new(EINVAL)); } if old_size == 0 || new_size == 0 { @@ -387,7 +477,8 @@ pub fn mremap(old_address: usize, old_size: usize, new_address: usize, new_size: let new_base = Page::containing_address(VirtualAddress::new(new_address)); let mremap_flags = MremapFlags::from_bits_truncate(flags); - let prot_flags = MapFlags::from_bits_truncate(flags) & (MapFlags::PROT_READ | MapFlags::PROT_WRITE | MapFlags::PROT_EXEC); + let prot_flags = MapFlags::from_bits_truncate(flags) + & (MapFlags::PROT_READ | MapFlags::PROT_WRITE | MapFlags::PROT_EXEC); let map_flags = if mremap_flags.contains(MremapFlags::FIXED_REPLACE) { MapFlags::MAP_FIXED @@ -404,7 +495,15 @@ pub fn mremap(old_address: usize, old_size: usize, new_address: usize, new_size: let mut guard = addr_space.write(); - let base = AddrSpace::r#move(&mut *guard, None, src_span, requested_dst_base, new_page_count, map_flags, &mut Vec::new())?; + let base = AddrSpace::r#move( + &mut *guard, + None, + src_span, + requested_dst_base, + new_page_count, + map_flags, + &mut Vec::new(), + )?; Ok(base.start_address().data()) } diff --git a/src/syscall/futex.rs b/src/syscall/futex.rs index 5d7f190fa4..40310fff2a 100644 --- a/src/syscall/futex.rs +++ b/src/syscall/futex.rs @@ -2,20 +2,23 @@ //! Futex or Fast Userspace Mutex is "a method for waiting until a certain condition becomes true." //! //! For more information about futexes, please read [this](https://eli.thegreenplace.net/2018/basics-of-futexes/) blog post, and the [futex(2)](http://man7.org/linux/man-pages/man2/futex.2.html) man page -use alloc::collections::VecDeque; -use alloc::sync::Arc; -use rmm::Arch; +use alloc::{collections::VecDeque, sync::Arc}; use core::sync::atomic::{AtomicU32, Ordering}; +use rmm::Arch; use spin::RwLock; -use crate::context::{self, memory::AddrSpace, Context}; -use crate::memory::PhysicalAddress; -use crate::paging::{Page, VirtualAddress}; -use crate::time; +use crate::{ + context::{self, memory::AddrSpace, Context}, + memory::PhysicalAddress, + paging::{Page, VirtualAddress}, + time, +}; -use crate::syscall::data::TimeSpec; -use crate::syscall::error::{Error, Result, EAGAIN, EFAULT, EINVAL, ESRCH}; -use crate::syscall::flag::{FUTEX_REQUEUE, FUTEX_WAIT, FUTEX_WAIT64, FUTEX_WAKE}; +use crate::syscall::{ + data::TimeSpec, + error::{Error, Result, EAGAIN, EFAULT, EINVAL, ESRCH}, + flag::{FUTEX_REQUEUE, FUTEX_WAIT, FUTEX_WAIT64, FUTEX_WAKE}, +}; use super::usercopy::UserSlice; @@ -58,8 +61,10 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, addr2: usize) -> R match op { // TODO: FUTEX_WAIT_MULTIPLE? FUTEX_WAIT | FUTEX_WAIT64 => { - let timeout_opt = UserSlice::ro(val2, core::mem::size_of::())?.none_if_null() - .map(|buf| unsafe { buf.read_exact::() }).transpose()?; + let timeout_opt = UserSlice::ro(val2, core::mem::size_of::())? + .none_if_null() + .map(|buf| unsafe { buf.read_exact::() }) + .transpose()?; { let mut futexes = FUTEXES.write(); @@ -75,7 +80,8 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, addr2: usize) -> R // On systems where virtual memory is not abundant, we might instead add an // atomic usercopy function. - let accessible_addr = unsafe { crate::paging::RmmA::phys_to_virt(target_physaddr) }.data(); + let accessible_addr = + unsafe { crate::paging::RmmA::phys_to_virt(target_physaddr) }.data(); ( u64::from(unsafe { @@ -93,7 +99,9 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, addr2: usize) -> R return Err(Error::new(EINVAL)); } ( - u64::from(unsafe { (*(addr as *const AtomicU64)).load(Ordering::SeqCst) }), + u64::from(unsafe { + (*(addr as *const AtomicU64)).load(Ordering::SeqCst) + }), val as u64, ) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 36a7fa158d..c32b636b3b 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -7,35 +7,26 @@ extern crate syscall; use syscall::{EventFlags, EOVERFLOW}; pub use self::syscall::{ - FloatRegisters, - IntRegisters, - EnvRegisters, - data, - error, - flag, - io, - number, - ptrace_event, + data, error, flag, io, number, ptrace_event, EnvRegisters, FloatRegisters, IntRegisters, }; -pub use self::driver::*; -pub use self::fs::*; -pub use self::futex::futex; -pub use self::privilege::*; -pub use self::process::*; -pub use self::time::*; -pub use self::usercopy::validate_region; +pub use self::{ + driver::*, fs::*, futex::futex, privilege::*, process::*, time::*, usercopy::validate_region, +}; -use self::data::{Map, SigAction, TimeSpec}; -use self::error::{Error, Result, ENOSYS}; -use self::flag::{MapFlags, WaitFlags}; -use self::number::*; +use self::{ + data::{Map, SigAction, TimeSpec}, + error::{Error, Result, ENOSYS}, + flag::{MapFlags, WaitFlags}, + number::*, +}; -use crate::context::ContextId; -use crate::context::memory::AddrSpace; -use crate::interrupt::InterruptStack; -use crate::scheme::{FileHandle, SchemeNamespace, memory::MemoryScheme}; -use crate::syscall::usercopy::UserSlice; +use crate::{ + context::{memory::AddrSpace, ContextId}, + interrupt::InterruptStack, + scheme::{memory::MemoryScheme, FileHandle, SchemeNamespace}, + syscall::usercopy::UserSlice, +}; /// Debug pub mod debug; @@ -63,79 +54,128 @@ pub mod usercopy; /// This function is the syscall handler of the kernel, it is composed of an inner function that returns a `Result`. After the inner function runs, the syscall /// function calls [`Error::mux`] on it. -pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, stack: &mut InterruptStack) -> usize { +pub fn syscall( + a: usize, + b: usize, + c: usize, + d: usize, + e: usize, + f: usize, + stack: &mut InterruptStack, +) -> usize { #[inline(always)] - fn inner(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, stack: &mut InterruptStack) -> Result { + fn inner( + a: usize, + b: usize, + c: usize, + d: usize, + e: usize, + f: usize, + stack: &mut InterruptStack, + ) -> Result { //SYS_* is declared in kernel/syscall/src/number.rs match a & SYS_CLASS { SYS_CLASS_FILE => { let fd = FileHandle::from(b); match a & SYS_ARG { SYS_ARG_SLICE => match a { - SYS_WRITE => file_op_generic(fd, |scheme, _, number| scheme.kwrite(number, UserSlice::ro(c, d)?)), + SYS_WRITE => file_op_generic(fd, |scheme, _, number| { + scheme.kwrite(number, UserSlice::ro(c, d)?) + }), SYS_FMAP => { let addrspace = AddrSpace::current()?; let map = unsafe { UserSlice::ro(c, d)?.read_exact::()? }; if b == !0 { MemoryScheme::fmap_anonymous(&addrspace, &map, false) } else { - file_op_generic(fd, |scheme, _, number| scheme.kfmap(number, &addrspace, &map, false)) + file_op_generic(fd, |scheme, _, number| { + scheme.kfmap(number, &addrspace, &map, false) + }) } - }, + } // SYS_FMAP_OLD is ignored - SYS_FUTIMENS => file_op_generic(fd, |scheme, _, number| scheme.kfutimens(number, UserSlice::ro(c, d)?)), + SYS_FUTIMENS => file_op_generic(fd, |scheme, _, number| { + scheme.kfutimens(number, UserSlice::ro(c, d)?) + }), _ => return Err(Error::new(ENOSYS)), - } + }, SYS_ARG_MSLICE => match a { - SYS_READ => file_op_generic(fd, |scheme, _, number| scheme.kread(number, UserSlice::wo(c, d)?)), - SYS_FPATH => file_op_generic(fd, |scheme, _, number| scheme.kfpath(number, UserSlice::wo(c, d)?)), + SYS_READ => file_op_generic(fd, |scheme, _, number| { + scheme.kread(number, UserSlice::wo(c, d)?) + }), + SYS_FPATH => file_op_generic(fd, |scheme, _, number| { + scheme.kfpath(number, UserSlice::wo(c, d)?) + }), SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?).map(|()| 0), - SYS_FSTATVFS => file_op_generic(fd, |scheme, _, number| scheme.kfstatvfs(number, UserSlice::wo(c, d)?).map(|()| 0)), + SYS_FSTATVFS => file_op_generic(fd, |scheme, _, number| { + scheme.kfstatvfs(number, UserSlice::wo(c, d)?).map(|()| 0) + }), _ => return Err(Error::new(ENOSYS)), }, _ => match a { SYS_DUP => dup(fd, UserSlice::ro(c, d)?).map(FileHandle::into), - SYS_DUP2 => dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?).map(FileHandle::into), + SYS_DUP2 => dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?) + .map(FileHandle::into), #[cfg(target_pointer_width = "32")] - SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64 | ((f as u64) << 32)), + SYS_SENDFD => { + sendfd(fd, FileHandle::from(c), d, e as u64 | ((f as u64) << 32)) + } #[cfg(target_pointer_width = "64")] SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64), - SYS_LSEEK => file_op_generic(fd, |scheme, _, number| scheme.seek(number, c as isize, d)), - SYS_FCHMOD => file_op_generic(fd, |scheme, _, number| scheme.fchmod(number, c as u16).map(|()| 0)), - SYS_FCHOWN => file_op_generic(fd, |scheme, _, number| scheme.fchown(number, c as u32, d as u32).map(|()| 0)), + SYS_LSEEK => file_op_generic(fd, |scheme, _, number| { + scheme.seek(number, c as isize, d) + }), + SYS_FCHMOD => file_op_generic(fd, |scheme, _, number| { + scheme.fchmod(number, c as u16).map(|()| 0) + }), + SYS_FCHOWN => file_op_generic(fd, |scheme, _, number| { + scheme.fchown(number, c as u32, d as u32).map(|()| 0) + }), SYS_FCNTL => fcntl(fd, c, d), - SYS_FEVENT => file_op_generic(fd, |scheme, _, number| Ok(scheme.fevent(number, EventFlags::from_bits_truncate(c))?.bits())), + SYS_FEVENT => file_op_generic(fd, |scheme, _, number| { + Ok(scheme + .fevent(number, EventFlags::from_bits_truncate(c))? + .bits()) + }), SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?).map(|()| 0), SYS_FUNMAP => funmap(b, c), - SYS_FSYNC => file_op_generic(fd, |scheme, _, number| scheme.fsync(number).map(|()| 0)), + SYS_FSYNC => file_op_generic(fd, |scheme, _, number| { + scheme.fsync(number).map(|()| 0) + }), // TODO: 64-bit lengths on 32-bit platforms - SYS_FTRUNCATE => file_op_generic(fd, |scheme, _, number| scheme.ftruncate(number, c).map(|()| 0)), + SYS_FTRUNCATE => file_op_generic(fd, |scheme, _, number| { + scheme.ftruncate(number, c).map(|()| 0) + }), SYS_CLOSE => close(fd).map(|()| 0), _ => return Err(Error::new(ENOSYS)), - } + }, } - }, + } SYS_CLASS_PATH => match a { SYS_OPEN => open(UserSlice::ro(b, c)?, d).map(FileHandle::into), SYS_RMDIR => rmdir(UserSlice::ro(b, c)?).map(|()| 0), SYS_UNLINK => unlink(UserSlice::ro(b, c)?).map(|()| 0), - _ => Err(Error::new(ENOSYS)) + _ => Err(Error::new(ENOSYS)), }, _ => match a { SYS_YIELD => sched_yield().map(|()| 0), SYS_NANOSLEEP => nanosleep( UserSlice::ro(b, core::mem::size_of::())?, UserSlice::wo(c, core::mem::size_of::())?.none_if_null(), - ).map(|()| 0), - SYS_CLOCK_GETTIME => clock_gettime(b, UserSlice::wo(c, core::mem::size_of::())?).map(|()| 0), + ) + .map(|()| 0), + SYS_CLOCK_GETTIME => { + clock_gettime(b, UserSlice::wo(c, core::mem::size_of::())?) + .map(|()| 0) + } SYS_FUTEX => futex(b, c, d, e, f), SYS_GETPID => getpid().map(ContextId::into), SYS_GETPGID => getpgid(ContextId::from(b)).map(ContextId::into), @@ -143,7 +183,16 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, stack SYS_EXIT => exit((b & 0xFF) << 8), SYS_KILL => kill(ContextId::from(b), c), - SYS_WAITPID => waitpid(ContextId::from(b), if c == 0 { None } else { Some(UserSlice::wo(c, core::mem::size_of::())?) }, WaitFlags::from_bits_truncate(d)).map(ContextId::into), + SYS_WAITPID => waitpid( + ContextId::from(b), + if c == 0 { + None + } else { + Some(UserSlice::wo(c, core::mem::size_of::())?) + }, + WaitFlags::from_bits_truncate(d), + ) + .map(ContextId::into), SYS_IOPL => iopl(b, stack), SYS_GETEGID => getegid(), SYS_GETENS => getens(), @@ -152,7 +201,11 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, stack SYS_GETNS => getns(), SYS_GETUID => getuid(), SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)), - SYS_MKNS => mkns(UserSlice::ro(b, c.checked_mul(core::mem::size_of::<[usize; 2]>()).ok_or(Error::new(EOVERFLOW))?)?), + SYS_MKNS => mkns(UserSlice::ro( + b, + c.checked_mul(core::mem::size_of::<[usize; 2]>()) + .ok_or(Error::new(EOVERFLOW))?, + )?), SYS_SETPGID => setpgid(ContextId::from(b), ContextId::from(c)), SYS_SETREUID => setreuid(b as u32, c as u32), SYS_SETRENS => setrens(SchemeNamespace::from(b), SchemeNamespace::from(c)), @@ -162,20 +215,22 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, stack UserSlice::ro(c, core::mem::size_of::())?.none_if_null(), UserSlice::wo(d, core::mem::size_of::())?.none_if_null(), e, - ).map(|()| 0), + ) + .map(|()| 0), SYS_SIGPROCMASK => sigprocmask( b, UserSlice::ro(c, 16)?.none_if_null(), UserSlice::wo(d, 16)?.none_if_null(), - ).map(|()| 0), + ) + .map(|()| 0), SYS_SIGRETURN => sigreturn(), SYS_UMASK => umask(b), SYS_VIRTTOPHYS => virttophys(b), SYS_MREMAP => mremap(b, c, d, e, f), - _ => Err(Error::new(ENOSYS)) - } + _ => Err(Error::new(ENOSYS)), + }, } } @@ -258,7 +313,7 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, stack match result { Ok(ref ok) => { print!("Ok({} ({:#X}))", ok, ok); - }, + } Err(ref err) => { print!("Err({} ({:#X}))", err, err.errno); } diff --git a/src/syscall/privilege.rs b/src/syscall/privilege.rs index 98f632623b..b12d239edb 100644 --- a/src/syscall/privilege.rs +++ b/src/syscall/privilege.rs @@ -1,11 +1,15 @@ use alloc::vec::Vec; -use crate::context; -use crate::scheme::{self, SchemeNamespace}; -use crate::syscall::error::*; +use crate::{ + context, + scheme::{self, SchemeNamespace}, + syscall::error::*, +}; -use super::copy_path_to_buf; -use super::usercopy::{UserSlice, UserSliceRo}; +use super::{ + copy_path_to_buf, + usercopy::{UserSlice, UserSliceRo}, +}; pub fn getegid() -> Result { let contexts = context::contexts(); @@ -61,7 +65,9 @@ pub fn mkns(mut user_buf: UserSliceRo) -> Result { let mut names = Vec::with_capacity(user_buf.len() / core::mem::size_of::<[usize; 2]>()); - while let Some((current_name_ptr_buf, next_part)) = user_buf.split_at(core::mem::size_of::<[usize; 2]>()) { + while let Some((current_name_ptr_buf, next_part)) = + user_buf.split_at(core::mem::size_of::<[usize; 2]>()) + { let mut iter = current_name_ptr_buf.usizes(); let ptr = iter.next().ok_or(Error::new(EINVAL))??; let len = iter.next().ok_or(Error::new(EINVAL))??; @@ -85,41 +91,39 @@ pub fn setregid(rgid: u32, egid: u32) -> Result { let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; let mut context = context_lock.write(); - let setrgid = - if context.euid == 0 { - // Allow changing RGID if root - true - } else if rgid == context.egid { - // Allow changing RGID if used for EGID - true - } else if rgid == context.rgid { - // Allow changing RGID if used for RGID - true - } else if rgid as i32 == -1 { - // Ignore RGID if -1 is passed - false - } else { - // Not permitted otherwise - return Err(Error::new(EPERM)); - }; + let setrgid = if context.euid == 0 { + // Allow changing RGID if root + true + } else if rgid == context.egid { + // Allow changing RGID if used for EGID + true + } else if rgid == context.rgid { + // Allow changing RGID if used for RGID + true + } else if rgid as i32 == -1 { + // Ignore RGID if -1 is passed + false + } else { + // Not permitted otherwise + return Err(Error::new(EPERM)); + }; - let setegid = - if context.euid == 0 { - // Allow changing EGID if root - true - } else if egid == context.egid { - // Allow changing EGID if used for EGID - true - } else if egid == context.rgid { - // Allow changing EGID if used for RGID - true - } else if egid as i32 == -1 { - // Ignore EGID if -1 is passed - false - } else { - // Not permitted otherwise - return Err(Error::new(EPERM)); - }; + let setegid = if context.euid == 0 { + // Allow changing EGID if root + true + } else if egid == context.egid { + // Allow changing EGID if used for EGID + true + } else if egid == context.rgid { + // Allow changing EGID if used for RGID + true + } else if egid as i32 == -1 { + // Ignore EGID if -1 is passed + false + } else { + // Not permitted otherwise + return Err(Error::new(EPERM)); + }; if setrgid { context.rgid = rgid; @@ -137,53 +141,51 @@ pub fn setrens(rns: SchemeNamespace, ens: SchemeNamespace) -> Result { let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; let mut context = context_lock.write(); - let setrns = - if rns.get() as isize == -1 { - // Ignore RNS if -1 is passed - false - } else if rns.get() == 0 { - // Allow entering capability mode - true - } else if context.rns.get() == 0 { - // Do not allow leaving capability mode - return Err(Error::new(EPERM)); - } else if context.euid == 0 { - // Allow setting RNS if root - true - } else if rns == context.ens { - // Allow setting RNS if used for ENS - true - } else if rns == context.rns { - // Allow setting RNS if used for RNS - true - } else { - // Not permitted otherwise - return Err(Error::new(EPERM)); - }; + let setrns = if rns.get() as isize == -1 { + // Ignore RNS if -1 is passed + false + } else if rns.get() == 0 { + // Allow entering capability mode + true + } else if context.rns.get() == 0 { + // Do not allow leaving capability mode + return Err(Error::new(EPERM)); + } else if context.euid == 0 { + // Allow setting RNS if root + true + } else if rns == context.ens { + // Allow setting RNS if used for ENS + true + } else if rns == context.rns { + // Allow setting RNS if used for RNS + true + } else { + // Not permitted otherwise + return Err(Error::new(EPERM)); + }; - let setens = - if ens.get() as isize == -1 { - // Ignore ENS if -1 is passed - false - } else if ens.get() == 0 { - // Allow entering capability mode - true - } else if context.ens.get() == 0 { - // Do not allow leaving capability mode - return Err(Error::new(EPERM)); - } else if context.euid == 0 { - // Allow setting ENS if root - true - } else if ens == context.ens { - // Allow setting ENS if used for ENS - true - } else if ens == context.rns { - // Allow setting ENS if used for RNS - true - } else { - // Not permitted otherwise - return Err(Error::new(EPERM)); - }; + let setens = if ens.get() as isize == -1 { + // Ignore ENS if -1 is passed + false + } else if ens.get() == 0 { + // Allow entering capability mode + true + } else if context.ens.get() == 0 { + // Do not allow leaving capability mode + return Err(Error::new(EPERM)); + } else if context.euid == 0 { + // Allow setting ENS if root + true + } else if ens == context.ens { + // Allow setting ENS if used for ENS + true + } else if ens == context.rns { + // Allow setting ENS if used for RNS + true + } else { + // Not permitted otherwise + return Err(Error::new(EPERM)); + }; if setrns { assert_ne!(rns.get() as isize, -1); @@ -203,41 +205,39 @@ pub fn setreuid(ruid: u32, euid: u32) -> Result { let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; let mut context = context_lock.write(); - let setruid = - if context.euid == 0 { - // Allow setting RUID if root - true - } else if ruid == context.euid { - // Allow setting RUID if used for EUID - true - } else if ruid == context.ruid { - // Allow setting RUID if used for RUID - true - } else if ruid as i32 == -1 { - // Ignore RUID if -1 is passed - false - } else { - // Not permitted otherwise - return Err(Error::new(EPERM)); - }; + let setruid = if context.euid == 0 { + // Allow setting RUID if root + true + } else if ruid == context.euid { + // Allow setting RUID if used for EUID + true + } else if ruid == context.ruid { + // Allow setting RUID if used for RUID + true + } else if ruid as i32 == -1 { + // Ignore RUID if -1 is passed + false + } else { + // Not permitted otherwise + return Err(Error::new(EPERM)); + }; - let seteuid = - if context.euid == 0 { - // Allow setting EUID if root - true - } else if euid == context.euid { - // Allow setting EUID if used for EUID - true - } else if euid == context.ruid { - // Allow setting EUID if used for RUID - true - } else if euid as i32 == -1 { - // Ignore EUID if -1 is passed - false - } else { - // Not permitted otherwise - return Err(Error::new(EPERM)); - }; + let seteuid = if context.euid == 0 { + // Allow setting EUID if root + true + } else if euid == context.euid { + // Allow setting EUID if used for EUID + true + } else if euid == context.ruid { + // Allow setting EUID if used for RUID + true + } else if euid as i32 == -1 { + // Ignore EUID if -1 is passed + false + } else { + // Not permitted otherwise + return Err(Error::new(EPERM)); + }; if setruid { context.ruid = ruid; diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 76c219d296..35b9576b84 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -1,32 +1,37 @@ -use alloc::{ - sync::Arc, - vec::Vec, -}; +use alloc::{sync::Arc, vec::Vec}; use core::mem; use spin::RwLock; -use crate::context::memory::PageSpan; -use crate::context::{ContextId, memory::AddrSpace, WaitpidKey}; +use crate::context::{ + memory::{AddrSpace, PageSpan}, + ContextId, WaitpidKey, +}; -use crate::Bootstrap; -use crate::context; -use crate::interrupt; -use crate::paging::mapper::PageFlushAll; -use crate::paging::{Page, PageFlags, VirtualAddress, PAGE_SIZE}; -use crate::ptrace; -use crate::start::usermode; -use crate::syscall::data::SigAction; -use crate::syscall::error::*; -use crate::syscall::flag::{wifcontinued, wifstopped, MapFlags, - PTRACE_STOP_EXIT, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, - SIGCONT, SIGTERM, WaitFlags, WCONTINUED, WNOHANG, WUNTRACED}; -use crate::syscall::ptrace_event; +use crate::{ + context, interrupt, + paging::{mapper::PageFlushAll, Page, PageFlags, VirtualAddress, PAGE_SIZE}, + ptrace, + start::usermode, + syscall::{ + data::SigAction, + error::*, + flag::{ + wifcontinued, wifstopped, MapFlags, WaitFlags, PTRACE_STOP_EXIT, SIGCONT, SIGTERM, + SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, WCONTINUED, WNOHANG, WUNTRACED, + }, + ptrace_event, + }, + Bootstrap, +}; -use super::usercopy::{UserSliceWo, UserSliceRo}; +use super::usercopy::{UserSliceRo, UserSliceWo}; pub fn exit(status: usize) -> ! { - ptrace::breakpoint_callback(PTRACE_STOP_EXIT, Some(ptrace_event!(PTRACE_STOP_EXIT, status))); + ptrace::breakpoint_callback( + PTRACE_STOP_EXIT, + Some(ptrace_event!(PTRACE_STOP_EXIT, status)), + ); { let context_lock = context::current().expect("exit failed to find context"); @@ -36,8 +41,10 @@ pub fn exit(status: usize) -> ! { let pid = { let mut context = context_lock.write(); - close_files = Arc::try_unwrap(mem::take(&mut context.files)).map_or_else(|_| Vec::new(), RwLock::into_inner); - addrspace_opt = mem::take(&mut context.addr_space).and_then(|a| Arc::try_unwrap(a).ok()); + close_files = Arc::try_unwrap(mem::take(&mut context.files)) + .map_or_else(|_| Vec::new(), RwLock::into_inner); + addrspace_opt = + mem::take(&mut context.addr_space).and_then(|a| Arc::try_unwrap(a).ok()); drop(context.syscall_head.take()); drop(context.syscall_tail.take()); context.id @@ -47,15 +54,19 @@ pub fn exit(status: usize) -> ! { if pid == ContextId::from(1) { println!("Main kernel thread exited with status {:X}", status); - extern { + extern "C" { fn kreset() -> !; fn kstop() -> !; } if status == SIGTERM { - unsafe { kreset(); } + unsafe { + kreset(); + } } else { - unsafe { kstop(); } + unsafe { + kstop(); + } } } @@ -101,10 +112,13 @@ pub fn exit(status: usize) -> ! { waitpid.send(c_pid, c_status); } - waitpid.send(WaitpidKey { - pid: Some(pid), - pgid: Some(pgid) - }, (pid, status)); + waitpid.send( + WaitpidKey { + pid: Some(pid), + pgid: Some(pgid), + }, + (pid, status), + ); } } @@ -158,10 +172,7 @@ pub fn kill(pid: ContextId, sig: usize) -> Result { let contexts = context::contexts(); let send = |context: &mut context::Context| -> bool { - if euid == 0 - || euid == context.ruid - || ruid == context.ruid - { + if euid == 0 || euid == context.ruid || ruid == context.ruid { // If sig = 0, test that process exists and can be // signalled, but don't send any signal. if sig != 0 { @@ -231,7 +242,9 @@ pub fn kill(pid: ContextId, sig: usize) -> Result { Err(Error::new(EPERM)) } else { // Switch to ensure delivery to self - unsafe { context::switch(); } + unsafe { + context::switch(); + } Ok(0) } @@ -243,9 +256,13 @@ pub fn kill(pid: ContextId, sig: usize) -> Result { pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result { // println!("mprotect {:#X}, {}, {:#X}", address, size, flags); - let span = PageSpan::validate_nonempty(VirtualAddress::new(address), size).ok_or(Error::new(EINVAL))?; + let span = PageSpan::validate_nonempty(VirtualAddress::new(address), size) + .ok_or(Error::new(EINVAL))?; - AddrSpace::current()?.write().mprotect(span, flags).map(|()| 0) + AddrSpace::current()? + .write() + .mprotect(span, flags) + .map(|()| 0) } pub fn setpgid(pid: ContextId, pgid: ContextId) -> Result { @@ -276,7 +293,12 @@ pub fn setpgid(pid: ContextId, pgid: ContextId) -> Result { } } -pub fn sigaction(sig: usize, act_opt: Option, oldact_opt: Option, restorer: usize) -> Result<()> { +pub fn sigaction( + sig: usize, + act_opt: Option, + oldact_opt: Option, + restorer: usize, +) -> Result<()> { if sig == 0 || sig > 0x7F { return Err(Error::new(EINVAL)); } @@ -296,7 +318,11 @@ pub fn sigaction(sig: usize, act_opt: Option, oldact_opt: Option, oldmask_opt: Option) -> Result<()> { +pub fn sigprocmask( + how: usize, + mask_opt: Option, + oldmask_opt: Option, +) -> Result<()> { { let contexts = context::contexts(); let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; @@ -306,26 +332,24 @@ pub fn sigprocmask(how: usize, mask_opt: Option, oldmask_opt: Optio if let Some(oldmask) = oldmask_opt { // TODO: sigprocmask should be u64 - let (lo_dst, hi_dst) = oldmask.split_at(core::mem::size_of::()).ok_or(Error::new(EINVAL))?; + let (lo_dst, hi_dst) = oldmask + .split_at(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))?; lo_dst.write_u64(old_lo)?; hi_dst.write_u64(old_hi)?; } if let Some(mask) = mask_opt { - let (lo_src, hi_src) = mask.split_at(core::mem::size_of::()).ok_or(Error::new(EINVAL))?; + let (lo_src, hi_src) = mask + .split_at(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))?; let lo_arg = lo_src.read_u64()?; let hi_arg = hi_src.read_u64()?; context.sigmask = match how { - SIG_BLOCK => { - [old_lo | lo_arg, old_hi | hi_arg] - }, - SIG_UNBLOCK => { - [old_lo & !lo_arg, old_hi & !hi_arg] - }, - SIG_SETMASK => { - [lo_arg, hi_arg] - }, + SIG_BLOCK => [old_lo | lo_arg, old_hi | hi_arg], + SIG_UNBLOCK => [old_lo & !lo_arg, old_hi & !hi_arg], + SIG_SETMASK => [lo_arg, hi_arg], _ => { return Err(Error::new(EINVAL)); } @@ -382,19 +406,28 @@ fn reap(pid: ContextId) -> Result { } let _ = context::contexts_mut() - .remove(pid).ok_or(Error::new(ESRCH))?; + .remove(pid) + .ok_or(Error::new(ESRCH))?; Ok(pid) } -pub fn waitpid(pid: ContextId, status_ptr: Option, flags: WaitFlags) -> Result { +pub fn waitpid( + pid: ContextId, + status_ptr: Option, + flags: WaitFlags, +) -> Result { let (ppid, waitpid) = { let contexts = context::contexts(); let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; let context = context_lock.read(); (context.id, Arc::clone(&context.waitpid)) }; - let write_status = |value| status_ptr.map(|ptr| ptr.write_usize(value)).unwrap_or(Ok(())); + let write_status = |value| { + status_ptr + .map(|ptr| ptr.write_usize(value)) + .unwrap_or(Ok(())) + }; let grim_reaper = |w_pid: ContextId, status: usize| -> Option> { if wifcontinued(status) { @@ -429,7 +462,7 @@ pub fn waitpid(pid: ContextId, status_ptr: Option, flags: WaitFlags } } - if ! found { + if !found { return Err(Error::new(ECHILD)); } } @@ -460,7 +493,7 @@ pub fn waitpid(pid: ContextId, status_ptr: Option, flags: WaitFlags } } - if ! found { + if !found { return Err(Error::new(ECHILD)); } } @@ -468,17 +501,20 @@ pub fn waitpid(pid: ContextId, status_ptr: Option, flags: WaitFlags if flags & WNOHANG == WNOHANG { if let Some((w_pid, status)) = waitpid.receive_nonblock(&WaitpidKey { pid: None, - pgid: Some(pgid) + pgid: Some(pgid), }) { grim_reaper(w_pid, status) } else { Some(Ok(ContextId::from(0))) } } else { - let (w_pid, status) = waitpid.receive(&WaitpidKey { - pid: None, - pgid: Some(pgid) - }, "waitpid pgid"); + let (w_pid, status) = waitpid.receive( + &WaitpidKey { + pid: None, + pgid: Some(pgid), + }, + "waitpid pgid", + ); grim_reaper(w_pid, status) } } else { @@ -487,7 +523,12 @@ pub fn waitpid(pid: ContextId, status_ptr: Option, flags: WaitFlags let context_lock = contexts.get(pid).ok_or(Error::new(ECHILD))?; let mut context = context_lock.write(); if context.ppid != ppid { - println!("TODO: Hack for rustc - changing ppid of {} from {} to {}", context.id.get(), context.ppid.get(), ppid.get()); + println!( + "TODO: Hack for rustc - changing ppid of {} from {} to {}", + context.id.get(), + context.ppid.get(), + ppid.get() + ); context.ppid = ppid; //return Err(Error::new(ECHILD)); Some(context.status.clone()) @@ -499,23 +540,26 @@ pub fn waitpid(pid: ContextId, status_ptr: Option, flags: WaitFlags if let Some(context::Status::Exited(status)) = hack_status { let _ = waitpid.receive_nonblock(&WaitpidKey { pid: Some(pid), - pgid: None + pgid: None, }); grim_reaper(pid, status) } else if flags & WNOHANG == WNOHANG { if let Some((w_pid, status)) = waitpid.receive_nonblock(&WaitpidKey { pid: Some(pid), - pgid: None + pgid: None, }) { grim_reaper(w_pid, status) } else { Some(Ok(ContextId::from(0))) } } else { - let (w_pid, status) = waitpid.receive(&WaitpidKey { - pid: Some(pid), - pgid: None - }, "waitpid pid"); + let (w_pid, status) = waitpid.receive( + &WaitpidKey { + pid: Some(pid), + pgid: None, + }, + "waitpid pid", + ); grim_reaper(w_pid, status) } }; @@ -530,10 +574,14 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) -> ! { assert_ne!(bootstrap.page_count, 0); { - let addr_space = Arc::clone(context::contexts().current() - .expect("expected a context to exist when executing init") - .read().addr_space() - .expect("expected bootstrap context to have an address space")); + let addr_space = Arc::clone( + context::contexts() + .current() + .expect("expected a context to exist when executing init") + .read() + .addr_space() + .expect("expected bootstrap context to have an address space"), + ); // TODO: Use AddrSpace::mmap. let mut addr_space = addr_space.write(); @@ -541,17 +589,19 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) -> ! { // TODO: Mark as owned and then support reclaiming the memory to the allocator if // deallocated? - addr_space.grants.insert(context::memory::Grant::zeroed( - PageSpan::new( - Page::containing_address(VirtualAddress::new(0)), - bootstrap.page_count, - ), - PageFlags::new().user(true).write(true).execute(true), - &mut addr_space.table.utable, - PageFlushAll::new(), - false, // is_shared - ).expect("failed to physmap bootstrap memory")); - + addr_space.grants.insert( + context::memory::Grant::zeroed( + PageSpan::new( + Page::containing_address(VirtualAddress::new(0)), + bootstrap.page_count, + ), + PageFlags::new().user(true).write(true).execute(true), + &mut addr_space.table.utable, + PageFlushAll::new(), + false, // is_shared + ) + .expect("failed to physmap bootstrap memory"), + ); } // TODO: Not all arches do linear mapping UserSliceWo::new(0, bootstrap.page_count * PAGE_SIZE) diff --git a/src/syscall/time.rs b/src/syscall/time.rs index 25a001984a..ffbfffed03 100644 --- a/src/syscall/time.rs +++ b/src/syscall/time.rs @@ -1,8 +1,12 @@ -use crate::time; -use crate::context; -use crate::syscall::data::TimeSpec; -use crate::syscall::error::*; -use crate::syscall::flag::{CLOCK_REALTIME, CLOCK_MONOTONIC}; +use crate::{ + context, + syscall::{ + data::TimeSpec, + error::*, + flag::{CLOCK_MONOTONIC, CLOCK_REALTIME}, + }, + time, +}; use super::usercopy::{UserSliceRo, UserSliceWo}; @@ -10,7 +14,7 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo) -> Result<()> { let arch_time = match clock { CLOCK_REALTIME => time::realtime(), CLOCK_MONOTONIC => time::monotonic(), - _ => return Err(Error::new(EINVAL)) + _ => return Err(Error::new(EINVAL)), }; buf.copy_exactly(&TimeSpec { @@ -38,7 +42,9 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option) -> Resu //TODO: Find out wake reason loop { - unsafe { context::switch(); } + unsafe { + context::switch(); + } let contexts = context::contexts(); let context_lock = contexts.current().ok_or(Error::new(ESRCH))?; @@ -71,6 +77,8 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option) -> Resu } pub fn sched_yield() -> Result<()> { - unsafe { context::switch(); } + unsafe { + context::switch(); + } Ok(()) } diff --git a/src/syscall/usercopy.rs b/src/syscall/usercopy.rs index 0d00f570b2..2b4001f08e 100644 --- a/src/syscall/usercopy.rs +++ b/src/syscall/usercopy.rs @@ -1,9 +1,11 @@ -use crate::paging::{Page, VirtualAddress}; -use crate::memory::PAGE_SIZE; +use crate::{ + memory::PAGE_SIZE, + paging::{Page, VirtualAddress}, +}; -use crate::arch::{arch_copy_to_user, arch_copy_from_user}; +use crate::arch::{arch_copy_from_user, arch_copy_to_user}; -use crate::syscall::error::{Error, EINVAL, EFAULT, Result}; +use crate::syscall::error::{Error, Result, EFAULT, EINVAL}; #[derive(Clone, Copy)] pub struct UserSlice { @@ -16,10 +18,7 @@ pub type UserSliceRw = UserSlice; impl UserSlice { pub fn empty() -> Self { - Self { - base: 0, - len: 0, - } + Self { base: 0, len: 0 } } pub fn len(&self) -> usize { self.len @@ -42,7 +41,16 @@ impl UserSlice { if idx > self.len { return None; } - Some((Self { base: self.base, len: idx }, Self { base: self.base + idx, len: self.len - idx })) + Some(( + Self { + base: self.base, + len: idx, + }, + Self { + base: self.base + idx, + len: self.len - idx, + }, + )) } pub fn advance(self, by: usize) -> Option { Some(self.split_at(by)?.1) @@ -51,25 +59,35 @@ impl UserSlice { Some(self.split_at(to)?.0) } pub fn none_if_null(self) -> Option { - if self.addr() == 0 { None } else { Some(self) } + if self.addr() == 0 { + None + } else { + Some(self) + } } /// Not unsafe, because user memory is not covered by the memory model that decides if /// something is UB, but it can break logic invariants - pub fn reinterpret_unchecked(self) -> UserSlice { + pub fn reinterpret_unchecked( + self, + ) -> UserSlice { UserSlice { base: self.base, len: self.len, } } pub fn in_variable_chunks(self, chunk_size: usize) -> impl Iterator { - (0..self.len()).step_by(chunk_size).map(move |i| self.advance(i).expect("already limited by length, must succeed")) + (0..self.len()).step_by(chunk_size).map(move |i| { + self.advance(i) + .expect("already limited by length, must succeed") + }) } pub fn in_exact_chunks(self, chunk_size: usize) -> impl Iterator { - (0..self.len().div_floor(chunk_size)) - .map(move |i| { - self.advance(i * chunk_size).expect("already limited by length, must succeed") - .limit(chunk_size).expect("length is aligned") - }) + (0..self.len().div_floor(chunk_size)).map(move |i| { + self.advance(i * chunk_size) + .expect("already limited by length, must succeed") + .limit(chunk_size) + .expect("length is aligned") + }) } } impl UserSlice { @@ -88,35 +106,51 @@ impl UserSlice { } pub unsafe fn read_exact(self) -> Result { let mut t: T = core::mem::zeroed(); - let slice = unsafe { core::slice::from_raw_parts_mut((&mut t as *mut T).cast::(), core::mem::size_of::()) }; + let slice = unsafe { + core::slice::from_raw_parts_mut( + (&mut t as *mut T).cast::(), + core::mem::size_of::(), + ) + }; - self.limit(core::mem::size_of::()).ok_or(Error::new(EINVAL))?.copy_to_slice(slice)?; + self.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_to_slice(slice)?; Ok(t) } pub fn copy_common_bytes_to_slice(self, slice: &mut [u8]) -> Result { let min = core::cmp::min(self.len(), slice.len()); - self.limit(min).expect("min(len, x) is always <= len").copy_to_slice(&mut slice[..min])?; + self.limit(min) + .expect("min(len, x) is always <= len") + .copy_to_slice(&mut slice[..min])?; Ok(min) } // TODO: Merge int IO functions? pub fn read_usize(self) -> Result { let mut ret = 0_usize.to_ne_bytes(); - self.limit(core::mem::size_of::()).ok_or(Error::new(EINVAL))?.copy_to_slice(&mut ret)?; + self.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_to_slice(&mut ret)?; Ok(usize::from_ne_bytes(ret)) } pub fn read_u32(self) -> Result { let mut ret = 0_u32.to_ne_bytes(); - self.limit(4).ok_or(Error::new(EINVAL))?.copy_to_slice(&mut ret)?; + self.limit(4) + .ok_or(Error::new(EINVAL))? + .copy_to_slice(&mut ret)?; Ok(u32::from_ne_bytes(ret)) } pub fn read_u64(self) -> Result { let mut ret = 0_u64.to_ne_bytes(); - self.limit(8).ok_or(Error::new(EINVAL))?.copy_to_slice(&mut ret)?; + self.limit(8) + .ok_or(Error::new(EINVAL))? + .copy_to_slice(&mut ret)?; Ok(u64::from_ne_bytes(ret)) } pub fn usizes(self) -> impl Iterator> { - self.in_exact_chunks(core::mem::size_of::()).map(Self::read_usize) + self.in_exact_chunks(core::mem::size_of::()) + .map(Self::read_usize) } } impl UserSlice { @@ -135,23 +169,33 @@ impl UserSlice { } pub fn copy_common_bytes_from_slice(self, slice: &[u8]) -> Result { let min = core::cmp::min(self.len(), slice.len()); - self.limit(min).expect("min(len, x) is always <= len").copy_from_slice(&slice[..min])?; + self.limit(min) + .expect("min(len, x) is always <= len") + .copy_from_slice(&slice[..min])?; Ok(min) } pub fn copy_exactly(self, slice: &[u8]) -> Result<()> { - self.limit(slice.len()).ok_or(Error::new(EINVAL))?.copy_from_slice(slice)?; + self.limit(slice.len()) + .ok_or(Error::new(EINVAL))? + .copy_from_slice(slice)?; Ok(()) } pub fn write_usize(self, word: usize) -> Result<()> { - self.limit(core::mem::size_of::()).ok_or(Error::new(EINVAL))?.copy_from_slice(&word.to_ne_bytes())?; + self.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_from_slice(&word.to_ne_bytes())?; Ok(()) } pub fn write_u32(self, int: u32) -> Result<()> { - self.limit(core::mem::size_of::()).ok_or(Error::new(EINVAL))?.copy_from_slice(&int.to_ne_bytes())?; + self.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_from_slice(&int.to_ne_bytes())?; Ok(()) } pub fn write_u64(self, int: u64) -> Result<()> { - self.limit(core::mem::size_of::()).ok_or(Error::new(EINVAL))?.copy_from_slice(&int.to_ne_bytes())?; + self.limit(core::mem::size_of::()) + .ok_or(Error::new(EINVAL))? + .copy_from_slice(&int.to_ne_bytes())?; Ok(()) } } @@ -173,7 +217,8 @@ impl UserSliceRw { } fn is_kernel_mem(slice: &[u8]) -> bool { - (slice.as_ptr() as usize) >= crate::USER_END_OFFSET && (slice.as_ptr() as usize).checked_add(slice.len()).is_some() + (slice.as_ptr() as usize) >= crate::USER_END_OFFSET + && (slice.as_ptr() as usize).checked_add(slice.len()).is_some() } /// Convert `[addr, addr+size)` into `(page, page_count)`. @@ -192,5 +237,8 @@ pub fn validate_region(address: usize, size: usize) -> Result<(Page, usize)> { if address.saturating_add(size) > crate::USER_END_OFFSET { return Err(Error::new(EFAULT)); } - Ok((Page::containing_address(VirtualAddress::new(address)), size / PAGE_SIZE)) + Ok(( + Page::containing_address(VirtualAddress::new(address)), + size / PAGE_SIZE, + )) }