vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cargo fix --edition & rustfmt

Browse Source 
Or to be precise:
RUST_TARGET_PATH=$(pwd)/targets cargo fix --edition \
--target targets/x86_64-unknown-kernel.json \
--target targets/i686-unknown-kernel.json \
--target targets/aarch64-unknown-kernel.json \
--target targets/riscv64-unknown-kernel.json \
-Zbuild-std=core,alloc --allow-dirty --bin kernel
cargo fmt
This commit is contained in:
bjorn3
2025-09-10 16:00:28 +02:00
parent ff48830c0b
commit cea93f7647
90 changed files with 4080 additions and 3571 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -2,7 +2,7 @@
name = "kernel"
version = "0.5.12"
build = "build.rs"
edition = "2021"
edition = "2024"
[build-dependencies]
cc = "1.0"
src/acpi/hpet.rs
+31 -21
View File
@@ -63,16 +63,17 @@ impl Hpet {
#[cfg(target_arch = "x86")]
impl Hpet {
pub unsafe fn map(&self) {
use crate::{
memory::{Frame, KernelMapper},
paging::{entry::EntryFlags, Page, VirtualAddress},
};
use rmm::PageFlags;
unsafe {
use crate::{
memory::{Frame, KernelMapper},
paging::{entry::EntryFlags, Page, VirtualAddress},
};
use rmm::PageFlags;
let frame = Frame::containing(PhysicalAddress::new(self.base_address.address as usize));
let page = Page::containing_address(VirtualAddress::new(crate::HPET_OFFSET));
let frame = Frame::containing(PhysicalAddress::new(self.base_address.address as usize));
let page = Page::containing_address(VirtualAddress::new(crate::HPET_OFFSET));
KernelMapper::lock()
KernelMapper::lock()
.get_mut()
.expect(
"KernelMapper locked re-entrant while mapping memory for GenericAddressStructure",
@@ -86,36 +87,45 @@ impl Hpet {
)
.expect("failed to map memory for GenericAddressStructure")
.flush();
}
}
pub unsafe fn read_u64(&self, offset: usize) -> u64 {
read_volatile((crate::HPET_OFFSET + offset) as *const u64)
unsafe { read_volatile((crate::HPET_OFFSET + offset) as *const u64) }
}
pub unsafe fn write_u64(&mut self, offset: usize, value: u64) {
write_volatile((crate::HPET_OFFSET + offset) as *mut u64, value);
unsafe {
write_volatile((crate::HPET_OFFSET + offset) as *mut u64, value);
}
}
}
#[cfg(not(target_arch = "x86"))]
impl Hpet {
pub unsafe fn map(&self) {
map_device_memory(
PhysicalAddress::new(self.base_address.address as usize),
PAGE_SIZE,
);
unsafe {
map_device_memory(
PhysicalAddress::new(self.base_address.address as usize),
PAGE_SIZE,
);
}
}
pub unsafe fn read_u64(&self, offset: usize) -> u64 {
read_volatile(
(self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *const u64,
)
unsafe {
read_volatile(
(self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *const u64,
)
}
}
pub unsafe fn write_u64(&mut self, offset: usize, value: u64) {
write_volatile(
(self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *mut u64,
value,
);
unsafe {
write_volatile(
(self.base_address.address as usize + offset + crate::PHYS_OFFSET) as *mut u64,
value,
);
}
}
}
src/acpi/mod.rs
+79 -75
View File
@@ -28,17 +28,19 @@ mod spcr;
mod xsdt;
unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::paging::PageMapper) {
let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data()));
let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data()));
unsafe {
let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data()));
let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data()));
for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE {
let (_, flush) = mapper
.map_linearly(
base.add(page_idx * crate::memory::PAGE_SIZE),
PageFlags::new(),
)
.expect("failed to linearly map SDT");
flush.flush();
for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE {
let (_, flush) = mapper
.map_linearly(
base.add(page_idx * crate::memory::PAGE_SIZE),
PageFlags::new(),
)
.expect("failed to linearly map SDT");
flush.flush();
}
}
}
@@ -93,83 +95,85 @@ pub static RXSDT_ENUM: Once<RxsdtEnum> = Once::new();
/// Parse the ACPI tables to gather CPU, interrupt, and timer information
pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
{
let mut sdt_ptrs = SDT_POINTERS.write();
*sdt_ptrs = Some(HashMap::new());
}
// Search for RSDP
let rsdp_opt = RSDP::get_rsdp(&mut KernelMapper::lock(), already_supplied_rsdp);
if let Some(rsdp) = rsdp_opt {
info!("SDT address: {:#x}", rsdp.sdt_address());
let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock());
for &c in rxsdt.signature.iter() {
print!("{}", c as char);
unsafe {
{
let mut sdt_ptrs = SDT_POINTERS.write();
*sdt_ptrs = Some(HashMap::new());
}
println!(":");
let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) {
let mut initialized = false;
// Search for RSDP
let rsdp_opt = RSDP::get_rsdp(&mut KernelMapper::lock(), already_supplied_rsdp);
let rsdt = RXSDT_ENUM.call_once(|| {
initialized = true;
if let Some(rsdp) = rsdp_opt {
info!("SDT address: {:#x}", rsdp.sdt_address());
let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock());
RxsdtEnum::Rsdt(rsdt)
});
for &c in rxsdt.signature.iter() {
print!("{}", c as char);
}
println!(":");
if !initialized {
log::error!("RXSDT_ENUM already initialized");
let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) {
let mut initialized = false;
let rsdt = RXSDT_ENUM.call_once(|| {
initialized = true;
RxsdtEnum::Rsdt(rsdt)
});
if !initialized {
log::error!("RXSDT_ENUM already initialized");
}
rsdt
} else if let Some(xsdt) = Xsdt::new(rxsdt) {
let mut initialized = false;
let xsdt = RXSDT_ENUM.call_once(|| {
initialized = true;
RxsdtEnum::Xsdt(xsdt)
});
if !initialized {
log::error!("RXSDT_ENUM already initialized");
}
xsdt
} else {
println!("UNKNOWN RSDT OR XSDT SIGNATURE");
return;
};
// TODO: Don't touch ACPI tables in kernel?
for sdt in rxsdt.iter() {
get_sdt(sdt, &mut KernelMapper::lock());
}
rsdt
} else if let Some(xsdt) = Xsdt::new(rxsdt) {
let mut initialized = false;
for sdt_address in rxsdt.iter() {
let sdt = &*((sdt_address + crate::PHYS_OFFSET) as *const Sdt);
let xsdt = RXSDT_ENUM.call_once(|| {
initialized = true;
RxsdtEnum::Xsdt(xsdt)
});
if !initialized {
log::error!("RXSDT_ENUM already initialized");
let signature = get_sdt_signature(sdt);
if let Some(ref mut ptrs) = *(SDT_POINTERS.write()) {
ptrs.insert(signature, sdt);
}
}
xsdt
//TODO: support this on any arch
#[cfg(target_arch = "aarch64")]
spcr::Spcr::init();
// TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface
// to initialize enumerated processors to userspace?
Madt::init();
// TODO: Let userspace setup HPET, and then provide an interface to specify which timer to
// use?
Hpet::init();
#[cfg(target_arch = "aarch64")]
gtdt::Gtdt::init();
} else {
println!("UNKNOWN RSDT OR XSDT SIGNATURE");
return;
};
// TODO: Don't touch ACPI tables in kernel?
for sdt in rxsdt.iter() {
get_sdt(sdt, &mut KernelMapper::lock());
println!("NO RSDP FOUND");
}
for sdt_address in rxsdt.iter() {
let sdt = &*((sdt_address + crate::PHYS_OFFSET) as *const Sdt);
let signature = get_sdt_signature(sdt);
if let Some(ref mut ptrs) = *(SDT_POINTERS.write()) {
ptrs.insert(signature, sdt);
}
}
//TODO: support this on any arch
#[cfg(target_arch = "aarch64")]
spcr::Spcr::init();
// TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface
// to initialize enumerated processors to userspace?
Madt::init();
// TODO: Let userspace setup HPET, and then provide an interface to specify which timer to
// use?
Hpet::init();
#[cfg(target_arch = "aarch64")]
gtdt::Gtdt::init();
} else {
println!("NO RSDP FOUND");
}
}
src/allocator/linked_list.rs
+28 -21
View File
@@ -12,38 +12,45 @@ pub struct Allocator;
impl Allocator {
pub unsafe fn init(offset: usize, size: usize) {
*HEAP.lock() = Some(Heap::new(offset, size));
unsafe {
*HEAP.lock() = Some(Heap::new(offset, size));
}
}
}
unsafe impl GlobalAlloc for Allocator {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
while let Some(ref mut heap) = *HEAP.lock() {
match heap.allocate_first_fit(layout) {
Err(()) => {
let size = heap.size();
super::map_heap(
&mut KernelMapper::lock(),
crate::KERNEL_HEAP_OFFSET + size,
crate::KERNEL_HEAP_SIZE,
);
heap.extend(crate::KERNEL_HEAP_SIZE);
}
other => {
return other
.ok()
.map_or(ptr::null_mut(), |allocation| allocation.as_ptr())
unsafe {
while let Some(ref mut heap) = *HEAP.lock() {
match heap.allocate_first_fit(layout) {
Err(()) => {
let size = heap.size();
super::map_heap(
&mut KernelMapper::lock(),
crate::KERNEL_HEAP_OFFSET + size,
crate::KERNEL_HEAP_SIZE,
);
heap.extend(crate::KERNEL_HEAP_SIZE);
}
other => {
return other
.ok()
.map_or(ptr::null_mut(), |allocation| allocation.as_ptr())
}
}
}
panic!("__rust_allocate: heap not initialized");
}
panic!("__rust_allocate: heap not initialized");
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
if let Some(ref mut heap) = *HEAP.lock() {
heap.deallocate(NonNull::new_unchecked(ptr), layout)
} else {
panic!("__rust_deallocate: heap not initialized");
unsafe {
match *HEAP.lock() {
Some(ref mut heap) => heap.deallocate(NonNull::new_unchecked(ptr), layout),
_ => {
panic!("__rust_deallocate: heap not initialized");
}
}
}
}
}
src/allocator/mod.rs
+28 -24
View File
@@ -17,35 +17,39 @@ mod linked_list;
mod slab;
unsafe fn map_heap(mapper: &mut KernelMapper, offset: usize, size: usize) {
let mapper = mapper
.get_mut()
.expect("failed to obtain exclusive access to KernelMapper while extending heap");
let mut flush_all = PageFlushAll::new();
unsafe {
let mapper = mapper
.get_mut()
.expect("failed to obtain exclusive access to KernelMapper while extending heap");
let mut flush_all = PageFlushAll::new();
let heap_start_page = Page::containing_address(VirtualAddress::new(offset));
let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1));
for page in Page::range_inclusive(heap_start_page, heap_end_page) {
let result = mapper
.map(
page.start_address(),
PageFlags::new()
.write(true)
.global(cfg!(not(feature = "pti"))),
)
.expect("failed to map kernel heap");
flush_all.consume(result);
let heap_start_page = Page::containing_address(VirtualAddress::new(offset));
let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1));
for page in Page::range_inclusive(heap_start_page, heap_end_page) {
let result = mapper
.map(
page.start_address(),
PageFlags::new()
.write(true)
.global(cfg!(not(feature = "pti"))),
)
.expect("failed to map kernel heap");
flush_all.consume(result);
}
flush_all.flush();
}
flush_all.flush();
}
pub unsafe fn init() {
let offset = crate::KERNEL_HEAP_OFFSET;
let size = crate::KERNEL_HEAP_SIZE;
unsafe {
let offset = crate::KERNEL_HEAP_OFFSET;
let size = crate::KERNEL_HEAP_SIZE;
// Map heap pages
map_heap(&mut KernelMapper::lock(), offset, size);
// Map heap pages
map_heap(&mut KernelMapper::lock(), offset, size);
// Initialize global heap
Allocator::init(offset, size);
// Initialize global heap
Allocator::init(offset, size);
}
}
src/arch/aarch64/device/cpu/registers/control_regs.rs
+102 -54
View File
@@ -13,129 +13,177 @@ bitflags! {
}
pub unsafe fn ttbr0_el1() -> u64 {
let ret: u64;
asm!("mrs {}, ttbr0_el1", out(reg) ret);
ret
unsafe {
let ret: u64;
asm!("mrs {}, ttbr0_el1", out(reg) ret);
ret
}
}
pub unsafe fn ttbr0_el1_write(val: u64) {
asm!("msr ttbr0_el1, {}", in(reg) val);
unsafe {
asm!("msr ttbr0_el1, {}", in(reg) val);
}
}
pub unsafe fn ttbr1_el1() -> u64 {
let ret: u64;
asm!("mrs {}, ttbr1_el1", out(reg) ret);
ret
unsafe {
let ret: u64;
asm!("mrs {}, ttbr1_el1", out(reg) ret);
ret
}
}
pub unsafe fn ttbr1_el1_write(val: u64) {
asm!("msr ttbr1_el1, {}", in(reg) val);
unsafe {
asm!("msr ttbr1_el1, {}", in(reg) val);
}
}
pub unsafe fn mair_el1() -> MairEl1 {
let ret: u64;
asm!("mrs {}, mair_el1", out(reg) ret);
MairEl1::from_bits_truncate(ret)
unsafe {
let ret: u64;
asm!("mrs {}, mair_el1", out(reg) ret);
MairEl1::from_bits_truncate(ret)
}
}
pub unsafe fn mair_el1_write(val: MairEl1) {
asm!("msr mair_el1, {}", in(reg) val.bits());
unsafe {
asm!("msr mair_el1, {}", in(reg) val.bits());
}
}
pub unsafe fn tpidr_el0() -> u64 {
let ret: u64;
asm!("mrs {}, tpidr_el0", out(reg) ret);
ret
unsafe {
let ret: u64;
asm!("mrs {}, tpidr_el0", out(reg) ret);
ret
}
}
pub unsafe fn tpidr_el0_write(val: u64) {
asm!("msr tpidr_el0, {}", in(reg) val);
unsafe {
asm!("msr tpidr_el0, {}", in(reg) val);
}
}
pub unsafe fn tpidr_el1() -> u64 {
let ret: u64;
asm!("mrs {}, tpidr_el1", out(reg) ret);
ret
unsafe {
let ret: u64;
asm!("mrs {}, tpidr_el1", out(reg) ret);
ret
}
}
pub unsafe fn tpidr_el1_write(val: u64) {
asm!("msr tpidr_el1, {}", in(reg) val);
unsafe {
asm!("msr tpidr_el1, {}", in(reg) val);
}
}
pub unsafe fn tpidrro_el0() -> u64 {
let ret: u64;
asm!("mrs {}, tpidrro_el0", out(reg) ret);
ret
unsafe {
let ret: u64;
asm!("mrs {}, tpidrro_el0", out(reg) ret);
ret
}
}
pub unsafe fn tpidrro_el0_write(val: u64) {
asm!("msr tpidrro_el0, {}", in(reg) val);
unsafe {
asm!("msr tpidrro_el0, {}", in(reg) val);
}
}
pub unsafe fn esr_el1() -> u32 {
let ret: u32;
asm!("mrs {0:w}, esr_el1", out(reg) ret);
ret
unsafe {
let ret: u32;
asm!("mrs {0:w}, esr_el1", out(reg) ret);
ret
}
}
pub unsafe fn vhe_present() -> bool {
let mut mmfr1: u64;
asm!("mrs {}, id_aa64mmfr1_el1", out(reg) mmfr1);
unsafe {
let mut mmfr1: u64;
asm!("mrs {}, id_aa64mmfr1_el1", out(reg) mmfr1);
// The VHE (Virtualization Host Extensions) field is in bits [7:4].
let vhe_field = (mmfr1 >> 4) & 0b1111;
// The VHE (Virtualization Host Extensions) field is in bits [7:4].
let vhe_field = (mmfr1 >> 4) & 0b1111;
vhe_field != 0
vhe_field != 0
}
}
pub unsafe fn cntfrq_el0() -> u32 {
let ret: usize;
asm!("mrs {}, cntfrq_el0", out(reg) ret);
ret as u32
unsafe {
let ret: usize;
asm!("mrs {}, cntfrq_el0", out(reg) ret);
ret as u32
}
}
pub unsafe fn ptmr_ctrl() -> u32 {
let ret: usize;
asm!("mrs {}, cntp_ctl_el0", out(reg) ret);
ret as u32
unsafe {
let ret: usize;
asm!("mrs {}, cntp_ctl_el0", out(reg) ret);
ret as u32
}
}
pub unsafe fn ptmr_ctrl_write(val: u32) {
asm!("msr cntp_ctl_el0, {}", in(reg) val as usize);
unsafe {
asm!("msr cntp_ctl_el0, {}", in(reg) val as usize);
}
}
pub unsafe fn ptmr_tval() -> u32 {
let ret: usize;
asm!("mrs {0}, cntp_tval_el0", out(reg) ret);
ret as u32
unsafe {
let ret: usize;
asm!("mrs {0}, cntp_tval_el0", out(reg) ret);
ret as u32
}
}
pub unsafe fn ptmr_tval_write(val: u32) {
asm!("msr cntp_tval_el0, {}", in(reg) val as usize);
unsafe {
asm!("msr cntp_tval_el0, {}", in(reg) val as usize);
}
}
pub unsafe fn vtmr_ctrl() -> u32 {
let ret: usize;
asm!("mrs {}, cntv_ctl_el0", out(reg) ret);
ret as u32
unsafe {
let ret: usize;
asm!("mrs {}, cntv_ctl_el0", out(reg) ret);
ret as u32
}
}
pub unsafe fn vtmr_ctrl_write(val: u32) {
asm!("msr cntv_ctl_el0, {}", in(reg) val as usize);
unsafe {
asm!("msr cntv_ctl_el0, {}", in(reg) val as usize);
}
}
pub unsafe fn vtmr_tval() -> u32 {
let ret: usize;
asm!("mrs {0}, cntv_tval_el0", out(reg) ret);
ret as u32
unsafe {
let ret: usize;
asm!("mrs {0}, cntv_tval_el0", out(reg) ret);
ret as u32
}
}
pub unsafe fn vtmr_tval_write(val: u32) {
asm!("msr cntv_tval_el0, {}", in(reg) val as usize);
unsafe {
asm!("msr cntv_tval_el0, {}", in(reg) val as usize);
}
}
pub unsafe fn midr() -> u32 {
let ret: usize;
asm!("mrs {}, midr_el1", out(reg) ret);
ret as u32
unsafe {
let ret: usize;
asm!("mrs {}, midr_el1", out(reg) ret);
ret as u32
}
}
src/arch/aarch64/device/generic_timer.rs
+18 -16
View File
@@ -24,22 +24,24 @@ bitflags! {
}
pub unsafe fn init(fdt: &Fdt) {
let mut timer = GenericTimer::new();
timer.init();
if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) {
let irq = get_interrupt(fdt, &node, 1).unwrap();
debug!("irq = {:?}", irq);
if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
//PHYS_NONSECURE_PPI only
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(irq)
.unwrap();
info!("generic_timer virq = {}", virq);
register_irq(virq as u32, Box::new(timer));
IRQ_CHIP.irq_enable(virq as u32);
} else {
error!("Failed to find irq parent for generic timer");
unsafe {
let mut timer = GenericTimer::new();
timer.init();
if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) {
let irq = get_interrupt(fdt, &node, 1).unwrap();
debug!("irq = {:?}", irq);
if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
//PHYS_NONSECURE_PPI only
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(irq)
.unwrap();
info!("generic_timer virq = {}", virq);
register_irq(virq as u32, Box::new(timer));
IRQ_CHIP.irq_enable(virq as u32);
} else {
error!("Failed to find irq parent for generic timer");
}
}
}
}
src/arch/aarch64/device/irqchip/gic.rs
+77 -57
View File
@@ -158,73 +158,83 @@ pub struct GicDistIf {
impl GicDistIf {
pub unsafe fn init(&mut self, addr: usize) {
self.address = addr;
unsafe {
self.address = addr;
// Disable IRQ Distribution
self.write(GICD_CTLR, 0);
// Disable IRQ Distribution
self.write(GICD_CTLR, 0);
let typer = self.read(GICD_TYPER);
self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1;
self.nirqs = ((typer & 0x1f) + 1) * 32;
info!(
"gic: Distributor supports {:?} CPUs and {:?} IRQs",
self.ncpus, self.nirqs
);
let typer = self.read(GICD_TYPER);
self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1;
self.nirqs = ((typer & 0x1f) + 1) * 32;
info!(
"gic: Distributor supports {:?} CPUs and {:?} IRQs",
self.ncpus, self.nirqs
);
// Set all SPIs to level triggered
for irq in (32..self.nirqs).step_by(16) {
self.write(GICD_ICFGR + ((irq / 16) * 4), 0);
}
// Set all SPIs to level triggered
for irq in (32..self.nirqs).step_by(16) {
self.write(GICD_ICFGR + ((irq / 16) * 4), 0);
}
// Disable all SPIs
for irq in (32..self.nirqs).step_by(32) {
self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff);
}
// Disable all SPIs
for irq in (32..self.nirqs).step_by(32) {
self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff);
}
// Affine all SPIs to CPU0 and set priorities for all IRQs
for irq in 0..self.nirqs {
if irq > 31 {
let ext_offset = GICD_ITARGETSR + (4 * (irq / 4));
// Affine all SPIs to CPU0 and set priorities for all IRQs
for irq in 0..self.nirqs {
if irq > 31 {
let ext_offset = GICD_ITARGETSR + (4 * (irq / 4));
let int_offset = irq % 4;
let mut val = self.read(ext_offset);
val |= 0b0000_0001 << (8 * int_offset);
self.write(ext_offset, val);
}
let ext_offset = GICD_IPRIORITY + (4 * (irq / 4));
let int_offset = irq % 4;
let mut val = self.read(ext_offset);
val |= 0b0000_0001 << (8 * int_offset);
val |= 0b0000_0000 << (8 * int_offset);
self.write(ext_offset, val);
}
let ext_offset = GICD_IPRIORITY + (4 * (irq / 4));
let int_offset = irq % 4;
let mut val = self.read(ext_offset);
val |= 0b0000_0000 << (8 * int_offset);
self.write(ext_offset, val);
// Enable IRQ group 0 and group 1 non-secure distribution
self.write(GICD_CTLR, 0x3);
}
// Enable IRQ group 0 and group 1 non-secure distribution
self.write(GICD_CTLR, 0x3);
}
pub unsafe fn irq_enable(&mut self, irq: u32) {
let offset = GICD_ISENABLER + (4 * (irq / 32));
let shift = 1 << (irq % 32);
let mut val = self.read(offset);
val |= shift;
self.write(offset, val);
unsafe {
let offset = GICD_ISENABLER + (4 * (irq / 32));
let shift = 1 << (irq % 32);
let mut val = self.read(offset);
val |= shift;
self.write(offset, val);
}
}
pub unsafe fn irq_disable(&mut self, irq: u32) {
let offset = GICD_ICENABLER + (4 * (irq / 32));
let shift = 1 << (irq % 32);
let mut val = self.read(offset);
val |= shift;
self.write(offset, val);
unsafe {
let offset = GICD_ICENABLER + (4 * (irq / 32));
let shift = 1 << (irq % 32);
let mut val = self.read(offset);
val |= shift;
self.write(offset, val);
}
}
unsafe fn read(&self, reg: u32) -> u32 {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
unsafe {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
}
}
unsafe fn write(&mut self, reg: u32, value: u32) {
write_volatile((self.address + reg as usize) as *mut u32, value);
unsafe {
write_volatile((self.address + reg as usize) as *mut u32, value);
}
}
}
@@ -235,32 +245,42 @@ pub struct GicCpuIf {
impl GicCpuIf {
pub unsafe fn init(&mut self, addr: usize) {
self.address = addr;
unsafe {
self.address = addr;
// Enable CPU0's GIC interface
self.write(GICC_CTLR, 1);
// Set CPU0's Interrupt Priority Mask
self.write(GICC_PMR, 0xff);
// Enable CPU0's GIC interface
self.write(GICC_CTLR, 1);
// Set CPU0's Interrupt Priority Mask
self.write(GICC_PMR, 0xff);
}
}
unsafe fn irq_ack(&mut self) -> u32 {
let irq = self.read(GICC_IAR) & 0x1ff;
if irq == 1023 {
panic!("irq_ack: got ID 1023!!!");
unsafe {
let irq = self.read(GICC_IAR) & 0x1ff;
if irq == 1023 {
panic!("irq_ack: got ID 1023!!!");
}
irq
}
irq
}
unsafe fn irq_eoi(&mut self, irq: u32) {
self.write(GICC_EOIR, irq);
unsafe {
self.write(GICC_EOIR, irq);
}
}
unsafe fn read(&self, reg: u32) -> u32 {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
unsafe {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
}
}
unsafe fn write(&mut self, reg: u32, value: u32) {
write_volatile((self.address + reg as usize) as *mut u32, value);
unsafe {
write_volatile((self.address + reg as usize) as *mut u32, value);
}
}
}
src/arch/aarch64/device/irqchip/gicv3.rs
+32 -26
View File
@@ -149,39 +149,45 @@ pub struct GicV3CpuIf;
impl GicV3CpuIf {
pub unsafe fn init(&mut self) {
// Enable system register access
{
let value = 1_usize;
asm!("msr icc_sre_el1, {}", in(reg) value);
}
// Set control register
{
let value = 0_usize;
asm!("msr icc_ctlr_el1, {}", in(reg) value);
}
// Enable non-secure group 1
{
let value = 1_usize;
asm!("msr icc_igrpen1_el1, {}", in(reg) value);
}
// Set CPU0's Interrupt Priority Mask
{
let value = 0xFF_usize;
asm!("msr icc_pmr_el1, {}", in(reg) value);
unsafe {
// Enable system register access
{
let value = 1_usize;
asm!("msr icc_sre_el1, {}", in(reg) value);
}
// Set control register
{
let value = 0_usize;
asm!("msr icc_ctlr_el1, {}", in(reg) value);
}
// Enable non-secure group 1
{
let value = 1_usize;
asm!("msr icc_igrpen1_el1, {}", in(reg) value);
}
// Set CPU0's Interrupt Priority Mask
{
let value = 0xFF_usize;
asm!("msr icc_pmr_el1, {}", in(reg) value);
}
}
}
unsafe fn irq_ack(&mut self) -> u32 {
let mut irq: usize;
asm!("mrs {}, icc_iar1_el1", out(reg) irq);
irq &= 0x1ff;
if irq == 1023 {
panic!("irq_ack: got ID 1023!!!");
unsafe {
let mut irq: usize;
asm!("mrs {}, icc_iar1_el1", out(reg) irq);
irq &= 0x1ff;
if irq == 1023 {
panic!("irq_ack: got ID 1023!!!");
}
irq as u32
}
irq as u32
}
unsafe fn irq_eoi(&mut self, irq: u32) {
asm!("msr icc_eoir1_el1, {}", in(reg) irq as usize);
unsafe {
asm!("msr icc_eoir1_el1, {}", in(reg) irq as usize);
}
}
}
src/arch/aarch64/device/irqchip/irq_bcm2835.rs
+37 -29
View File
@@ -72,47 +72,55 @@ impl Bcm2835ArmInterruptController {
}
}
unsafe fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize, Option<usize>)> {
//assert address_cells == 0x1, size_cells == 0x1
let mem = node.reg().unwrap().nth(0).unwrap();
let base = get_mmio_address(fdt, node, &mem).unwrap();
let size = mem.size.unwrap() as u32;
let mut ret_virq = None;
unsafe {
//assert address_cells == 0x1, size_cells == 0x1
let mem = node.reg().unwrap().nth(0).unwrap();
let base = get_mmio_address(fdt, node, &mem).unwrap();
let size = mem.size.unwrap() as u32;
let mut ret_virq = None;
if let Some(interrupt_parent) = node.property("interrupt-parent") {
let phandle = interrupt_parent.as_usize().unwrap() as u32;
let irq = get_interrupt(fdt, node, 0).unwrap();
let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap();
//PHYS_NONSECURE_PPI only
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(irq)
.unwrap();
info!(
"register bcm2835arm_ctrl as ic_idx {}'s child virq = {}",
ic_idx, virq
);
ret_virq = Some(virq);
if let Some(interrupt_parent) = node.property("interrupt-parent") {
let phandle = interrupt_parent.as_usize().unwrap() as u32;
let irq = get_interrupt(fdt, node, 0).unwrap();
let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap();
//PHYS_NONSECURE_PPI only
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(irq)
.unwrap();
info!(
"register bcm2835arm_ctrl as ic_idx {}'s child virq = {}",
ic_idx, virq
);
ret_virq = Some(virq);
}
Ok((base as usize, size as usize, ret_virq))
}
Ok((base as usize, size as usize, ret_virq))
}
unsafe fn init(&mut self) {
debug!("IRQ BCM2835 INIT");
//disable all interrupt
self.write(DISABLE_0, 0xffff_ffff);
self.write(DISABLE_1, 0xffff_ffff);
self.write(DISABLE_2, 0xffff_ffff);
unsafe {
debug!("IRQ BCM2835 INIT");
//disable all interrupt
self.write(DISABLE_0, 0xffff_ffff);
self.write(DISABLE_1, 0xffff_ffff);
self.write(DISABLE_2, 0xffff_ffff);
debug!("IRQ BCM2835 END");
debug!("IRQ BCM2835 END");
}
}
unsafe fn read(&self, reg: u32) -> u32 {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
unsafe {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
}
}
unsafe fn write(&mut self, reg: u32, value: u32) {
write_volatile((self.address + reg as usize) as *mut u32, value);
unsafe {
write_volatile((self.address + reg as usize) as *mut u32, value);
}
}
}
src/arch/aarch64/device/irqchip/irq_bcm2836.rs
+17 -11
View File
@@ -89,24 +89,30 @@ impl Bcm2836ArmInterruptController {
}
unsafe fn init(&mut self) {
debug!("IRQ BCM2836 INIT");
//init local timer freq
self.write(LOCAL_CONTROL, 0x0);
self.write(LOCAL_PRESCALER, 0x8000_0000);
unsafe {
debug!("IRQ BCM2836 INIT");
//init local timer freq
self.write(LOCAL_CONTROL, 0x0);
self.write(LOCAL_PRESCALER, 0x8000_0000);
//routing all irq to core
self.write(LOCAL_GPU_ROUTING, self.active_cpu);
debug!("routing all irq to core {}", self.active_cpu);
debug!("IRQ BCM2836 END");
//routing all irq to core
self.write(LOCAL_GPU_ROUTING, self.active_cpu);
debug!("routing all irq to core {}", self.active_cpu);
debug!("IRQ BCM2836 END");
}
}
unsafe fn read(&self, reg: u32) -> u32 {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
unsafe {
let val = read_volatile((self.address + reg as usize) as *const u32);
val
}
}
unsafe fn write(&mut self, reg: u32, value: u32) {
write_volatile((self.address + reg as usize) as *mut u32, value);
unsafe {
write_volatile((self.address + reg as usize) as *mut u32, value);
}
}
}
src/arch/aarch64/device/mod.rs
+30 -26
View File
@@ -16,36 +16,40 @@ pub static ROOT_IC_IDX: AtomicUsize = AtomicUsize::new(0);
pub static ROOT_IC_IDX_IS_SET: AtomicUsize = AtomicUsize::new(0);
unsafe fn init_root_ic(fdt: &Fdt) {
let is_set = ROOT_IC_IDX_IS_SET.load(Ordering::Relaxed);
if is_set != 0 {
let ic_idx = ROOT_IC_IDX.load(Ordering::Relaxed);
info!("Already selected {} as root ic", ic_idx);
return;
}
unsafe {
let is_set = ROOT_IC_IDX_IS_SET.load(Ordering::Relaxed);
if is_set != 0 {
let ic_idx = ROOT_IC_IDX.load(Ordering::Relaxed);
info!("Already selected {} as root ic", ic_idx);
return;
}
let root_irqc_phandle = fdt
.root()
.property("interrupt-parent")
.unwrap()
.as_usize()
.unwrap();
let ic_idx = IRQ_CHIP
.phandle_to_ic_idx(root_irqc_phandle as u32)
.unwrap();
info!("select {} as root ic", ic_idx);
ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed);
let root_irqc_phandle = fdt
.root()
.property("interrupt-parent")
.unwrap()
.as_usize()
.unwrap();
let ic_idx = IRQ_CHIP
.phandle_to_ic_idx(root_irqc_phandle as u32)
.unwrap();
info!("select {} as root ic", ic_idx);
ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed);
}
}
pub unsafe fn init_devicetree(fdt: &Fdt) {
info!("IRQCHIP INIT");
crate::dtb::irqchip::init(&fdt);
init_root_ic(&fdt);
info!("GIT INIT");
generic_timer::init(fdt);
info!("SERIAL INIT");
serial::init(fdt);
info!("RTC INIT");
rtc::init(fdt);
unsafe {
info!("IRQCHIP INIT");
crate::dtb::irqchip::init(&fdt);
init_root_ic(&fdt);
info!("GIT INIT");
generic_timer::init(fdt);
info!("SERIAL INIT");
serial::init(fdt);
info!("RTC INIT");
rtc::init(fdt);
}
}
#[derive(Default)]
src/arch/aarch64/device/rtc.rs
+1 -1
View File
@@ -30,7 +30,7 @@ struct Pl031rtc {
impl Pl031rtc {
unsafe fn read(&self, reg: usize) -> u32 {
read_volatile((crate::PHYS_OFFSET + self.phys + reg) as *const u32)
unsafe { read_volatile((crate::PHYS_OFFSET + self.phys + reg) as *const u32) }
}
pub fn time(&mut self) -> u64 {
src/arch/aarch64/device/serial.rs
+54 -50
View File
@@ -76,63 +76,67 @@ impl InterruptHandler for Com1Irq {
}
pub unsafe fn init_early(dtb: &Fdt) {
if COM1.lock().is_some() {
// Hardcoded UART
return;
}
unsafe {
if COM1.lock().is_some() {
// Hardcoded UART
return;
}
if let Some((phys, size, skip_init, cts, compatible)) = diag_uart_range(dtb) {
let virt = crate::PHYS_OFFSET + phys;
let serial_opt = if compatible.contains("arm,pl011") {
let mut serial_port = uart_pl011::SerialPort::new(virt, cts);
if !skip_init {
serial_port.init(false);
}
Some(SerialKind::Pl011(serial_port))
} else if compatible.contains("ns16550a") || compatible.contains("snps,dw-apb-uart") {
//TODO: get actual register size from device tree
let serial_port = uart_16550::SerialPort::<Mmio<u32>>::new(virt);
if !skip_init {
serial_port.init();
}
Some(SerialKind::Ns16550u32(serial_port))
} else {
None
};
match serial_opt {
Some(serial) => {
*COM1.lock() = Some(serial);
info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size);
}
None => {
log::warn!(
"UART {:?} at {:#X} size {:#X}: no driver found",
compatible,
virt,
size
);
if let Some((phys, size, skip_init, cts, compatible)) = diag_uart_range(dtb) {
let virt = crate::PHYS_OFFSET + phys;
let serial_opt = if compatible.contains("arm,pl011") {
let mut serial_port = uart_pl011::SerialPort::new(virt, cts);
if !skip_init {
serial_port.init(false);
}
Some(SerialKind::Pl011(serial_port))
} else if compatible.contains("ns16550a") || compatible.contains("snps,dw-apb-uart") {
//TODO: get actual register size from device tree
let serial_port = uart_16550::SerialPort::<Mmio<u32>>::new(virt);
if !skip_init {
serial_port.init();
}
Some(SerialKind::Ns16550u32(serial_port))
} else {
None
};
match serial_opt {
Some(serial) => {
*COM1.lock() = Some(serial);
info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size);
}
None => {
log::warn!(
"UART {:?} at {:#X} size {:#X}: no driver found",
compatible,
virt,
size
);
}
}
}
}
}
pub unsafe fn init(fdt: &Fdt) {
//TODO: find actual serial device, not just any PL011
if let Some(node) = fdt.find_compatible(&["arm,pl011"]) {
let irq = get_interrupt(fdt, &node, 0).unwrap();
if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(irq)
.unwrap();
info!("serial_port virq = {}", virq);
register_irq(virq as u32, Box::new(Com1Irq {}));
IRQ_CHIP.irq_enable(virq as u32);
} else {
error!("serial port irq parent not found");
unsafe {
//TODO: find actual serial device, not just any PL011
if let Some(node) = fdt.find_compatible(&["arm,pl011"]) {
let irq = get_interrupt(fdt, &node, 0).unwrap();
if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(irq)
.unwrap();
info!("serial_port virq = {}", virq);
register_irq(virq as u32, Box::new(Com1Irq {}));
IRQ_CHIP.irq_enable(virq as u32);
} else {
error!("serial port irq parent not found");
}
}
if let Some(ref mut serial_port) = *COM1.lock() {
serial_port.enable_irq();
}
}
if let Some(ref mut serial_port) = *COM1.lock() {
serial_port.enable_irq();
}
}
src/arch/aarch64/interrupt/exception.rs
+99 -85
View File
@@ -26,9 +26,11 @@ fn iss(esr: usize) -> u32 {
}
unsafe fn far_el1() -> usize {
let ret: usize;
core::arch::asm!("mrs {}, far_el1", out(reg) ret);
ret
unsafe {
let ret: usize;
core::arch::asm!("mrs {}, far_el1", out(reg) ret);
ret
}
}
unsafe fn instr_data_abort_inner(
@@ -37,48 +39,56 @@ unsafe fn instr_data_abort_inner(
instr_not_data: bool,
_from: &str,
) -> bool {
let iss = iss(stack.iret.esr_el1);
let fsc = iss & 0x3F;
//dbg!(fsc);
unsafe {
let iss = iss(stack.iret.esr_el1);
let fsc = iss & 0x3F;
//dbg!(fsc);
let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111;
//let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111;
let write_not_read_if_data = iss & (1 << 6) != 0;
let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111;
//let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111;
let write_not_read_if_data = iss & (1 << 6) != 0;
let mut flags = GenericPfFlags::empty();
flags.set(GenericPfFlags::PRESENT, !was_translation_fault);
let mut flags = GenericPfFlags::empty();
flags.set(GenericPfFlags::PRESENT, !was_translation_fault);
// TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64
// doesn't appear to require that flag to be set if the read alone would trigger a fault.
flags.set(
GenericPfFlags::INVOLVED_WRITE,
write_not_read_if_data && !instr_not_data,
);
flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data);
flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user);
// TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64
// doesn't appear to require that flag to be set if the read alone would trigger a fault.
flags.set(
GenericPfFlags::INVOLVED_WRITE,
write_not_read_if_data && !instr_not_data,
);
flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data);
flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user);
let faulting_addr = VirtualAddress::new(far_el1());
//dbg!(faulting_addr, flags, from);
let faulting_addr = VirtualAddress::new(far_el1());
//dbg!(faulting_addr, flags, from);
crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok()
crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok()
}
}
unsafe fn cntfrq_el0() -> usize {
let ret: usize;
core::arch::asm!("mrs {}, cntfrq_el0", out(reg) ret);
ret
unsafe {
let ret: usize;
core::arch::asm!("mrs {}, cntfrq_el0", out(reg) ret);
ret
}
}
unsafe fn cntpct_el0() -> usize {
let ret: usize;
core::arch::asm!("mrs {}, cntpct_el0", out(reg) ret);
ret
unsafe {
let ret: usize;
core::arch::asm!("mrs {}, cntpct_el0", out(reg) ret);
ret
}
}
unsafe fn cntvct_el0() -> usize {
let ret: usize;
core::arch::asm!("mrs {}, cntvct_el0", out(reg) ret);
ret
unsafe {
let ret: usize;
core::arch::asm!("mrs {}, cntvct_el0", out(reg) ret);
ret
}
}
unsafe fn instr_trapped_msr_mrs_inner(
@@ -87,52 +97,54 @@ unsafe fn instr_trapped_msr_mrs_inner(
_instr_not_data: bool,
_from: &str,
) -> bool {
let iss = iss(stack.iret.esr_el1);
// let res0 = (iss & 0x1C0_0000) >> 22;
let op0 = (iss & 0x030_0000) >> 20;
let op2 = (iss & 0x00e_0000) >> 17;
let op1 = (iss & 0x001_c000) >> 14;
let crn = (iss & 0x000_3c00) >> 10;
let rt = (iss & 0x000_03e0) >> 5;
let crm = (iss & 0x000_001e) >> 1;
let dir = iss & 0x000_0001;
unsafe {
let iss = iss(stack.iret.esr_el1);
// let res0 = (iss & 0x1C0_0000) >> 22;
let op0 = (iss & 0x030_0000) >> 20;
let op2 = (iss & 0x00e_0000) >> 17;
let op1 = (iss & 0x001_c000) >> 14;
let crn = (iss & 0x000_3c00) >> 10;
let rt = (iss & 0x000_03e0) >> 5;
let crm = (iss & 0x000_001e) >> 1;
let dir = iss & 0x000_0001;
/*
print!("iss=0x{:x}, res0=0b{:03b}, op0=0b{:02b}\n
op2=0b{:03b}, op1=0b{:03b}, crn=0b{:04b}\n
rt=0b{:05b}, crm=0b{:04b}, dir=0b{:b}\n",
iss, res0, op0, op2, op1, crn, rt, crm, dir);
*/
/*
print!("iss=0x{:x}, res0=0b{:03b}, op0=0b{:02b}\n
op2=0b{:03b}, op1=0b{:03b}, crn=0b{:04b}\n
rt=0b{:05b}, crm=0b{:04b}, dir=0b{:b}\n",
iss, res0, op0, op2, op1, crn, rt, crm, dir);
*/
match (op0, op1, crn, crm, op2, dir) {
//MRS <Xt>, CNTFRQ_EL0
(0b11, 0b011, 0b1110, 0b0000, 0b000, 0b1) => {
let reg_val = cntfrq_el0();
stack.store_reg(rt as usize, reg_val);
//skip faulting instruction, A64 instructions are always 32-bits
stack.iret.elr_el1 += 4;
return true;
match (op0, op1, crn, crm, op2, dir) {
//MRS <Xt>, CNTFRQ_EL0
(0b11, 0b011, 0b1110, 0b0000, 0b000, 0b1) => {
let reg_val = cntfrq_el0();
stack.store_reg(rt as usize, reg_val);
//skip faulting instruction, A64 instructions are always 32-bits
stack.iret.elr_el1 += 4;
return true;
}
//MRS <Xt>, CNTPCT_EL0
(0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => {
let reg_val = cntpct_el0();
stack.store_reg(rt as usize, reg_val);
//skip faulting instruction, A64 instructions are always 32-bits
stack.iret.elr_el1 += 4;
return true;
}
//MRS <Xt>, CNTVCT_EL0
(0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => {
let reg_val = cntvct_el0();
stack.store_reg(rt as usize, reg_val);
//skip faulting instruction, A64 instructions are always 32-bits
stack.iret.elr_el1 += 4;
return true;
}
_ => {}
}
//MRS <Xt>, CNTPCT_EL0
(0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => {
let reg_val = cntpct_el0();
stack.store_reg(rt as usize, reg_val);
//skip faulting instruction, A64 instructions are always 32-bits
stack.iret.elr_el1 += 4;
return true;
}
//MRS <Xt>, CNTVCT_EL0
(0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => {
let reg_val = cntvct_el0();
stack.store_reg(rt as usize, reg_val);
//skip faulting instruction, A64 instructions are always 32-bits
stack.iret.elr_el1 += 4;
return true;
}
_ => {}
false
}
false
}
exception_stack!(synchronous_exception_at_el1_with_spx, |stack| {
@@ -155,19 +167,21 @@ exception_stack!(synchronous_exception_at_el1_with_spx, |stack| {
}
});
unsafe fn pf_inner(stack: &mut InterruptStack, ty: u8, from: &str) -> bool {
match ty {
// "Data Abort taken from a lower Exception level"
0b100100 => instr_data_abort_inner(stack, true, false, from),
// "Data Abort taken without a change in Exception level"
0b100101 => instr_data_abort_inner(stack, false, false, from),
// "Instruction Abort taken from a lower Exception level"
0b100000 => instr_data_abort_inner(stack, true, true, from),
// "Instruction Abort taken without a change in Exception level"
0b100001 => instr_data_abort_inner(stack, false, true, from),
// "Trapped MSR, MRS or System instruction execution in AArch64 state"
0b011000 => instr_trapped_msr_mrs_inner(stack, true, true, from),
unsafe {
match ty {
// "Data Abort taken from a lower Exception level"
0b100100 => instr_data_abort_inner(stack, true, false, from),
// "Data Abort taken without a change in Exception level"
0b100101 => instr_data_abort_inner(stack, false, false, from),
// "Instruction Abort taken from a lower Exception level"
0b100000 => instr_data_abort_inner(stack, true, true, from),
// "Instruction Abort taken without a change in Exception level"
0b100001 => instr_data_abort_inner(stack, false, true, from),
// "Trapped MSR, MRS or System instruction execution in AArch64 state"
0b011000 => instr_trapped_msr_mrs_inner(stack, true, true, from),
_ => return false,
_ => return false,
}
}
}
src/arch/aarch64/interrupt/handler.rs
+16 -14
View File
@@ -267,7 +267,7 @@ impl InterruptStack {
#[macro_export]
macro_rules! aarch64_asm {
($($strings:expr,)+) => {
($($strings:expr_2021,)+) => {
core::arch::global_asm!(concat!(
$($strings),+,
));
@@ -376,11 +376,11 @@ macro_rules! pop_special {
macro_rules! exception_stack {
($name:ident, |$stack:ident| $code:block) => {
#[naked]
#[no_mangle]
pub unsafe extern "C" fn $name(stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
#[unsafe(no_mangle)]
pub unsafe extern "C" fn $name(stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { unsafe {
unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) { unsafe {
$code
}
}}
core::arch::naked_asm!(concat!(
// Backup all userspace registers to stack
push_preserved!(),
@@ -399,17 +399,19 @@ macro_rules! exception_stack {
"eret\n",
), sym inner);
}
}}
};
}
#[naked]
pub unsafe extern "C" fn enter_usermode() -> ! {
core::arch::naked_asm!(concat!(
"blr x28\n",
// Restore all userspace registers
pop_special!(),
pop_scratch!(),
pop_preserved!(),
"eret\n",
));
unsafe {
core::arch::naked_asm!(concat!(
"blr x28\n",
// Restore all userspace registers
pop_special!(),
pop_scratch!(),
pop_preserved!(),
"eret\n",
));
}
}
src/arch/aarch64/interrupt/irq.rs
+11 -7
View File
@@ -4,9 +4,11 @@ use core::sync::atomic::Ordering;
// use crate::percpu::PercpuBlock;
unsafe fn irq_ack() -> (u32, Option<usize>) {
let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic;
let irq = ic.irq_ack();
(irq, ic.irq_to_virq(irq))
unsafe {
let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic;
let irq = ic.irq_ack();
(irq, ic.irq_to_virq(irq))
}
}
exception_stack!(irq_at_el0, |_stack| {
@@ -33,11 +35,13 @@ exception_stack!(irq_at_el1, |_stack| {
//TODO
pub unsafe fn trigger(irq: u32) {
// FIXME add_irq accepts a u8 as irq number
// PercpuBlock::current().stats.add_irq(irq);
unsafe {
// FIXME add_irq accepts a u8 as irq number
// PercpuBlock::current().stats.add_irq(irq);
irq_trigger(irq.try_into().unwrap());
IRQ_CHIP.irq_eoi(irq);
irq_trigger(irq.try_into().unwrap());
IRQ_CHIP.irq_eoi(irq);
}
}
/*
src/arch/aarch64/interrupt/mod.rs
+21 -11
View File
@@ -15,7 +15,9 @@ pub use self::handler::InterruptStack;
/// Clear interrupts
#[inline(always)]
pub unsafe fn disable() {
asm!("msr daifset, #2");
unsafe {
asm!("msr daifset, #2");
}
}
/// Set interrupts and halt
@@ -23,8 +25,10 @@ pub unsafe fn disable() {
/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
#[inline(always)]
pub unsafe fn enable_and_halt() {
asm!("msr daifclr, #2");
asm!("wfi");
unsafe {
asm!("msr daifclr, #2");
asm!("wfi");
}
}
/// Set interrupts and nop
@@ -32,14 +36,18 @@ pub unsafe fn enable_and_halt() {
/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
#[inline(always)]
pub unsafe fn enable_and_nop() {
asm!("msr daifclr, #2");
asm!("nop");
unsafe {
asm!("msr daifclr, #2");
asm!("nop");
}
}
/// Halt instruction
#[inline(always)]
pub unsafe fn halt() {
asm!("wfi");
unsafe {
asm!("wfi");
}
}
/// Pause instruction
@@ -51,12 +59,14 @@ pub fn pause() {
#[inline(always)]
pub unsafe fn init() {
// Setup interrupt handlers
asm!(
"
unsafe {
// Setup interrupt handlers
asm!(
"
ldr {tmp}, =exception_vector_base
msr vbar_el1, {tmp}
",
tmp = out(reg) _,
);
tmp = out(reg) _,
);
}
}
src/arch/aarch64/interrupt/syscall.rs
+2 -2
View File
@@ -1,7 +1,7 @@
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn do_exception_unhandled() {}
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn do_exception_synchronous() {}
#[allow(dead_code)]
src/arch/aarch64/interrupt/trace.rs
+17 -13
View File
@@ -8,21 +8,25 @@ pub struct StackTrace {
impl StackTrace {
#[inline(always)]
pub unsafe fn start() -> Option<Self> {
let fp: usize;
asm!("mov {}, fp", out(reg) fp);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(StackTrace {
fp,
pc_ptr: pc_ptr as *const usize,
})
unsafe {
let fp: usize;
asm!("mov {}, fp", out(reg) fp);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(StackTrace {
fp,
pc_ptr: pc_ptr as *const usize,
})
}
}
pub unsafe fn next(self) -> Option<Self> {
let fp = *(self.fp as *const usize);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(StackTrace {
fp: fp,
pc_ptr: pc_ptr as *const usize,
})
unsafe {
let fp = *(self.fp as *const usize);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(StackTrace {
fp: fp,
pc_ptr: pc_ptr as *const usize,
})
}
}
}
src/arch/aarch64/macros.rs
+2 -2
View File
@@ -11,6 +11,6 @@ macro_rules! print {
#[macro_export]
macro_rules! println {
() => (print!("\n"));
($fmt:expr) => (print!(concat!($fmt, "\n")));
($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
($fmt:expr_2021) => (print!(concat!($fmt, "\n")));
($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
}
src/arch/aarch64/misc.rs
+6 -4
View File
@@ -12,10 +12,12 @@ impl PercpuBlock {
#[cold]
pub unsafe fn init(cpu_id: LogicalCpuId) {
let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
let virt = RmmA::phys_to_virt(frame.base()).data() as *mut PercpuBlock;
unsafe {
let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
let virt = RmmA::phys_to_virt(frame.base()).data() as *mut PercpuBlock;
virt.write(PercpuBlock::init(cpu_id));
virt.write(PercpuBlock::init(cpu_id));
crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64);
crate::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64);
}
}
src/arch/aarch64/mod.rs
+7 -5
View File
@@ -40,11 +40,12 @@ pub use ::rmm::AArch64Arch as CurrentRmmArch;
pub use arch_copy_to_user as arch_copy_from_user;
#[naked]
#[link_section = ".usercopy-fns"]
#[unsafe(link_section = ".usercopy-fns")]
pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
// x0, x1, x2
core::arch::naked_asm!(
"
unsafe {
// x0, x1, x2
core::arch::naked_asm!(
"
mov x4, x0
mov x0, 0
2:
@@ -62,7 +63,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
3:
ret
"
);
);
}
}
pub const KFX_SIZE: usize = 1024;
src/arch/aarch64/paging/mod.rs
+10 -6
View File
@@ -18,19 +18,23 @@ pub const PAGE_MASK: usize = RmmA::PAGE_OFFSET_MASK;
/// Setup Memory Access Indirection Register
#[cold]
unsafe fn init_mair() {
let mut val: control_regs::MairEl1 = control_regs::mair_el1();
unsafe {
let mut val: control_regs::MairEl1 = control_regs::mair_el1();
val.insert(control_regs::MairEl1::DEVICE_MEMORY);
val.insert(control_regs::MairEl1::NORMAL_UNCACHED_MEMORY);
val.insert(control_regs::MairEl1::NORMAL_WRITEBACK_MEMORY);
val.insert(control_regs::MairEl1::DEVICE_MEMORY);
val.insert(control_regs::MairEl1::NORMAL_UNCACHED_MEMORY);
val.insert(control_regs::MairEl1::NORMAL_WRITEBACK_MEMORY);
control_regs::mair_el1_write(val);
control_regs::mair_el1_write(val);
}
}
/// Initialize MAIR
#[cold]
pub unsafe fn init() {
init_mair();
unsafe {
init_mair();
}
}
/// Page
src/arch/aarch64/start.rs
+164 -162
View File
@@ -52,189 +52,191 @@ pub struct KernelArgs {
}
/// The entry to Rust, all things must be initialized
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! {
let bootstrap = {
let args = args_ptr.read();
unsafe {
let bootstrap = {
let args = args_ptr.read();
// BSS should already be zero
{
assert_eq!(BSS_TEST_ZERO, 0);
assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
}
// BSS should already be zero
{
assert_eq!(BSS_TEST_ZERO, 0);
assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
}
KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst);
KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst);
KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst);
KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst);
// Convert env to slice
let env = slice::from_raw_parts(
(crate::PHYS_OFFSET + args.env_base) as *const u8,
args.env_size,
);
// Convert env to slice
let env = slice::from_raw_parts(
(crate::PHYS_OFFSET + args.env_base) as *const u8,
args.env_size,
);
// Set up graphical debug
#[cfg(feature = "graphical_debug")]
graphical_debug::init(env);
// Set up graphical debug
#[cfg(feature = "graphical_debug")]
graphical_debug::init(env);
// Get hardware descriptor data
//TODO: use env {DTB,RSDT}_{BASE,SIZE}?
let hwdesc_data = if args.hwdesc_base != 0 {
Some(unsafe {
slice::from_raw_parts(
(crate::PHYS_OFFSET + args.hwdesc_base) as *const u8,
args.hwdesc_size,
)
})
} else {
None
};
let dtb_res = hwdesc_data
.ok_or(fdt::FdtError::BadPtr)
.and_then(|data| Fdt::new(data));
let rsdp_opt = hwdesc_data.and_then(|data| {
if data.starts_with(b"RSD PTR ") {
Some(data.as_ptr())
// Get hardware descriptor data
//TODO: use env {DTB,RSDT}_{BASE,SIZE}?
let hwdesc_data = if args.hwdesc_base != 0 {
Some(unsafe {
slice::from_raw_parts(
(crate::PHYS_OFFSET + args.hwdesc_base) as *const u8,
args.hwdesc_size,
)
})
} else {
None
};
let dtb_res = hwdesc_data
.ok_or(fdt::FdtError::BadPtr)
.and_then(|data| Fdt::new(data));
let rsdp_opt = hwdesc_data.and_then(|data| {
if data.starts_with(b"RSD PTR ") {
Some(data.as_ptr())
} else {
None
}
});
// Try to find serial port prior to logging
if let Ok(dtb) = &dtb_res {
device::serial::init_early(dtb);
}
});
// Try to find serial port prior to logging
if let Ok(dtb) = &dtb_res {
device::serial::init_early(dtb);
}
// Initialize logger
crate::log::init_logger(|r| {
use core::fmt::Write;
let _ = write!(
crate::debug::Writer::new(),
"{}:{} -- {}\n",
r.target(),
r.level(),
r.args()
);
});
log::set_max_level(::log::LevelFilter::Debug);
// Initialize logger
crate::log::init_logger(|r| {
use core::fmt::Write;
let _ = write!(
crate::debug::Writer::new(),
"{}:{} -- {}\n",
r.target(),
r.level(),
r.args()
info!("Redox OS starting...");
info!(
"Kernel: {:X}:{:X}",
{ args.kernel_base },
args.kernel_base + args.kernel_size
);
info!(
"Stack: {:X}:{:X}",
{ args.stack_base },
args.stack_base + args.stack_size
);
info!(
"Env: {:X}:{:X}",
{ args.env_base },
args.env_base + args.env_size
);
info!(
"HWDESC: {:X}:{:X}",
{ args.hwdesc_base },
args.hwdesc_base + args.hwdesc_size
);
info!(
"Areas: {:X}:{:X}",
{ args.areas_base },
args.areas_base + args.areas_size
);
info!(
"Bootstrap: {:X}:{:X}",
{ args.bootstrap_base },
args.bootstrap_base + args.bootstrap_size
);
});
log::set_max_level(::log::LevelFilter::Debug);
info!("Redox OS starting...");
info!(
"Kernel: {:X}:{:X}",
{ args.kernel_base },
args.kernel_base + args.kernel_size
);
info!(
"Stack: {:X}:{:X}",
{ args.stack_base },
args.stack_base + args.stack_size
);
info!(
"Env: {:X}:{:X}",
{ args.env_base },
args.env_base + args.env_size
);
info!(
"HWDESC: {:X}:{:X}",
{ args.hwdesc_base },
args.hwdesc_base + args.hwdesc_size
);
info!(
"Areas: {:X}:{:X}",
{ args.areas_base },
args.areas_base + args.areas_size
);
info!(
"Bootstrap: {:X}:{:X}",
{ args.bootstrap_base },
args.bootstrap_base + args.bootstrap_size
);
interrupt::init();
interrupt::init();
// Initialize RMM
register_bootloader_areas(args.areas_base, args.areas_size);
if let Ok(dtb) = &dtb_res {
register_dev_memory_ranges(dtb);
}
register_memory_region(
args.kernel_base,
args.kernel_size,
BootloaderMemoryKind::Kernel,
);
register_memory_region(
args.stack_base,
args.stack_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.env_base,
args.env_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.hwdesc_base,
args.hwdesc_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.bootstrap_base,
args.bootstrap_size,
BootloaderMemoryKind::IdentityMap,
);
crate::startup::memory::init(None, None);
// Initialize paging
paging::init();
crate::misc::init(crate::cpu_set::LogicalCpuId::new(0));
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
AP_READY.store(false, Ordering::SeqCst);
BSP_READY.store(false, Ordering::SeqCst);
// Setup kernel heap
allocator::init();
// Set up double buffer for graphical debug now that heap is available
#[cfg(feature = "graphical_debug")]
graphical_debug::init_heap();
// Activate memory logging
crate::log::init();
// Initialize devices
match dtb_res {
Ok(dtb) => {
dtb::init(hwdesc_data.map(|slice| (slice.as_ptr() as usize, slice.len())));
device::init_devicetree(&dtb);
// Initialize RMM
register_bootloader_areas(args.areas_base, args.areas_size);
if let Ok(dtb) = &dtb_res {
register_dev_memory_ranges(dtb);
}
Err(err) => {
dtb::init(None);
log::warn!("failed to parse DTB: {}", err);
#[cfg(feature = "acpi")]
{
crate::acpi::init(rsdp_opt);
register_memory_region(
args.kernel_base,
args.kernel_size,
BootloaderMemoryKind::Kernel,
);
register_memory_region(
args.stack_base,
args.stack_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.env_base,
args.env_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.hwdesc_base,
args.hwdesc_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.bootstrap_base,
args.bootstrap_size,
BootloaderMemoryKind::IdentityMap,
);
crate::startup::memory::init(None, None);
// Initialize paging
paging::init();
crate::misc::init(crate::cpu_set::LogicalCpuId::new(0));
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
AP_READY.store(false, Ordering::SeqCst);
BSP_READY.store(false, Ordering::SeqCst);
// Setup kernel heap
allocator::init();
// Set up double buffer for graphical debug now that heap is available
#[cfg(feature = "graphical_debug")]
graphical_debug::init_heap();
// Activate memory logging
crate::log::init();
// Initialize devices
match dtb_res {
Ok(dtb) => {
dtb::init(hwdesc_data.map(|slice| (slice.as_ptr() as usize, slice.len())));
device::init_devicetree(&dtb);
}
Err(err) => {
dtb::init(None);
log::warn!("failed to parse DTB: {}", err);
#[cfg(feature = "acpi")]
{
crate::acpi::init(rsdp_opt);
}
}
}
}
BSP_READY.store(true, Ordering::SeqCst);
BSP_READY.store(true, Ordering::SeqCst);
crate::Bootstrap {
base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new(
args.bootstrap_base,
)),
page_count: args.bootstrap_size / crate::memory::PAGE_SIZE,
env,
}
};
crate::Bootstrap {
base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new(
args.bootstrap_base,
)),
page_count: args.bootstrap_size / crate::memory::PAGE_SIZE,
env,
}
};
crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap);
crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap);
}
}
#[repr(C, packed)]
src/arch/aarch64/stop.rs
+20 -14
View File
@@ -1,26 +1,32 @@
use core::arch::asm;
pub unsafe fn kreset() -> ! {
println!("kreset");
unsafe {
println!("kreset");
asm!("hvc #0",
in("x0") 0x8400_0009_usize,
options(noreturn),
)
asm!("hvc #0",
in("x0") 0x8400_0009_usize,
options(noreturn),
)
}
}
pub unsafe fn emergency_reset() -> ! {
asm!("hvc #0",
in("x0") 0x8400_0009_usize,
options(noreturn),
)
unsafe {
asm!("hvc #0",
in("x0") 0x8400_0009_usize,
options(noreturn),
)
}
}
pub unsafe fn kstop() -> ! {
println!("kstop");
unsafe {
println!("kstop");
asm!("hvc #0",
in("x0") 0x8400_0008_usize,
options(noreturn),
)
asm!("hvc #0",
in("x0") 0x8400_0008_usize,
options(noreturn),
)
}
}
src/arch/riscv64/device/irqchip/hlic.rs
+32 -24
View File
@@ -25,31 +25,39 @@ fn acknowledge(interrupt: usize) {
}
pub unsafe fn interrupt(hart: usize, interrupt: usize) {
assert!(
hart < CPU_INTERRUPT_HANDLERS.len(),
"Unexpected hart in interrupt routine"
);
acknowledge(interrupt);
let ic_idx = CPU_INTERRUPT_HANDLERS[hart].unwrap_or_else(|| {
panic!(
"No hlic connected to hart {} yet interrupt {} occurred",
hart, interrupt
)
});
let virq = IRQ_CHIP
.irq_to_virq(ic_idx, interrupt as u32)
.unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt));
if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler {
handler.irq_handler(virq as u32);
} else if let Some(ic_idx) = IRQ_CHIP.irq_desc[virq].basic.child_ic_idx {
IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_handler(virq as u32);
} else {
panic!(
"Unconnected interrupt {} occurred on hlic connected to hart {}",
interrupt, hart
unsafe {
assert!(
hart < CPU_INTERRUPT_HANDLERS.len(),
"Unexpected hart in interrupt routine"
);
acknowledge(interrupt);
let ic_idx = CPU_INTERRUPT_HANDLERS[hart].unwrap_or_else(|| {
panic!(
"No hlic connected to hart {} yet interrupt {} occurred",
hart, interrupt
)
});
let virq = IRQ_CHIP
.irq_to_virq(ic_idx, interrupt as u32)
.unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt));
match &mut IRQ_CHIP.irq_desc[virq].handler {
Some(handler) => {
handler.irq_handler(virq as u32);
}
_ => match IRQ_CHIP.irq_desc[virq].basic.child_ic_idx {
Some(ic_idx) => {
IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_handler(virq as u32);
}
_ => {
panic!(
"Unconnected interrupt {} occurred on hlic connected to hart {}",
interrupt, hart
);
}
},
}
}
}
src/arch/riscv64/device/mod.rs
+14 -10
View File
@@ -69,14 +69,16 @@ unsafe fn init_intc(cpu: &FdtNode) {
}
pub unsafe fn init() {
let data = DTB_BINARY.get().unwrap();
let fdt = Fdt::new(data).unwrap();
unsafe {
let data = DTB_BINARY.get().unwrap();
let fdt = Fdt::new(data).unwrap();
crate::dtb::irqchip::init(&fdt);
crate::dtb::irqchip::init(&fdt);
let cpu = fdt.find_node(format!("/cpus/cpu@{}", 0).as_str()).unwrap();
init_intc(&cpu);
init_time(&fdt);
let cpu = fdt.find_node(format!("/cpus/cpu@{}", 0).as_str()).unwrap();
init_intc(&cpu);
init_time(&fdt);
}
}
fn init_time(fdt: &Fdt) {
@@ -90,11 +92,13 @@ fn init_time(fdt: &Fdt) {
}
pub unsafe fn init_noncore() {
let data = DTB_BINARY.get().unwrap();
let fdt = Fdt::new(data).unwrap();
unsafe {
let data = DTB_BINARY.get().unwrap();
let fdt = Fdt::new(data).unwrap();
init_clint(&fdt);
serial::init(&fdt);
init_clint(&fdt);
serial::init(&fdt);
}
}
#[derive(Default)]
src/arch/riscv64/device/serial.rs
+55 -52
View File
@@ -65,65 +65,68 @@ impl InterruptHandler for Com1Irq {
}
pub unsafe fn init_early(dtb: &Fdt) {
if COM1.lock().is_some() {
// Hardcoded UART
return;
}
unsafe {
if COM1.lock().is_some() {
// Hardcoded UART
return;
}
if let Some((phys, size, skip_init, _cts, compatible)) = diag_uart_range(dtb) {
let virt = crate::PHYS_OFFSET + phys;
let serial_opt =
if compatible.contains("ns16550a") {
//TODO: get actual register size from device tree
let serial_port = uart_16550::SerialPort::<Mmio<u8>>::new(virt);
if !skip_init {
serial_port.init();
}
Some(SerialPort::Ns16550u8(serial_port))
} else if compatible.contains("snps,dw-apb-uart") {
//TODO: get actual register size from device tree
let serial_port = uart_16550::SerialPort::<Mmio<u32>>::new(virt);
if !skip_init {
serial_port.init();
}
Some(SerialPort::Ns16550u32(serial_port))
} else {
None
};
match serial_opt {
Some(serial) => {
*COM1.lock() = Some(serial);
info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size);
}
None => {
log::warn!(
"UART {:?} at {:#X} size {:#X}: no driver found",
compatible,
virt,
size
);
if let Some((phys, size, skip_init, _cts, compatible)) = diag_uart_range(dtb) {
let virt = crate::PHYS_OFFSET + phys;
let serial_opt = if compatible.contains("ns16550a") {
//TODO: get actual register size from device tree
let serial_port = uart_16550::SerialPort::<Mmio<u8>>::new(virt);
if !skip_init {
serial_port.init();
}
Some(SerialPort::Ns16550u8(serial_port))
} else if compatible.contains("snps,dw-apb-uart") {
//TODO: get actual register size from device tree
let serial_port = uart_16550::SerialPort::<Mmio<u32>>::new(virt);
if !skip_init {
serial_port.init();
}
Some(SerialPort::Ns16550u32(serial_port))
} else {
None
};
match serial_opt {
Some(serial) => {
*COM1.lock() = Some(serial);
info!("UART {:?} at {:#X} size {:#X}", compatible, virt, size);
}
None => {
log::warn!(
"UART {:?} at {:#X} size {:#X}: no driver found",
compatible,
virt,
size
);
}
}
}
}
}
pub unsafe fn init(fdt: &Fdt) -> Option<()> {
if let Some(node) = fdt.find_compatible(&["ns16550a", "snps,dw-apb-uart"]) {
let intr = get_interrupt(fdt, &node, 0).unwrap();
let interrupt_parent = interrupt_parent(fdt, &node)?;
let phandle = interrupt_parent.property("phandle")?.as_usize()? as u32;
let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle)?;
unsafe {
if let Some(node) = fdt.find_compatible(&["ns16550a", "snps,dw-apb-uart"]) {
let intr = get_interrupt(fdt, &node, 0).unwrap();
let interrupt_parent = interrupt_parent(fdt, &node)?;
let phandle = interrupt_parent.property("phandle")?.as_usize()? as u32;
let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle)?;
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(intr)
.unwrap();
info!("serial_port virq = {}", virq);
register_irq(virq as u32, Box::new(Com1Irq {}));
IRQ_CHIP.irq_enable(virq as u32);
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
.ic
.irq_xlate(intr)
.unwrap();
info!("serial_port virq = {}", virq);
register_irq(virq as u32, Box::new(Com1Irq {}));
IRQ_CHIP.irq_enable(virq as u32);
}
if let Some(ref mut _serial_port) = *COM1.lock() {
// serial_port.enable_irq(); // FIXME receive int is enabled by default in 16550. Disable by default?
}
Some(())
}
if let Some(ref mut _serial_port) = *COM1.lock() {
// serial_port.enable_irq(); // FIXME receive int is enabled by default in 16550. Disable by default?
}
Some(())
}
src/arch/riscv64/interrupt/exception.rs
+112 -99
View File
@@ -86,129 +86,142 @@ global_asm!(concat!(
);
unsafe fn exception_handler_inner(regs: &mut InterruptStack) {
let scause: usize;
let sstatus: usize;
core::arch::asm!(
"csrr t0, scause",
"csrr t1, sstatus",
lateout("t0") scause,
lateout("t1") sstatus,
options(nostack)
);
unsafe {
let scause: usize;
let sstatus: usize;
core::arch::asm!(
"csrr t0, scause",
"csrr t1, sstatus",
lateout("t0") scause,
lateout("t1") sstatus,
options(nostack)
);
//log::info!("Exception handler incoming: sepc={:x} scause={:x} sstatus={:x}", regs.iret.sepc, scause, sstatus);
//log::info!("Exception handler incoming: sepc={:x} scause={:x} sstatus={:x}", regs.iret.sepc, scause, sstatus);
let user_mode = sstatus & (1 << 8) == 0;
let user_mode = sstatus & (1 << 8) == 0;
if (scause as isize) < 0 {
handle_interrupt(scause & 0xF);
} else if page_fault(scause, regs, user_mode) {
} else if user_mode {
handle_user_exception(scause, regs);
} else {
handle_system_exception(scause, regs);
if (scause as isize) < 0 {
handle_interrupt(scause & 0xF);
} else if page_fault(scause, regs, user_mode) {
} else if user_mode {
handle_user_exception(scause, regs);
} else {
handle_system_exception(scause, regs);
}
//log::info!("Exception handler outgoing");
}
//log::info!("Exception handler outgoing");
}
unsafe fn handle_system_exception(scause: usize, regs: &InterruptStack) {
let stval: usize;
let tp: usize;
core::arch::asm!(
"csrr t0, stval",
"mv t1, tp",
lateout("t0") stval,
lateout("t1") tp,
options(nostack)
);
unsafe {
let stval: usize;
let tp: usize;
core::arch::asm!(
"csrr t0, stval",
"mv t1, tp",
lateout("t0") stval,
lateout("t1") tp,
options(nostack)
);
error!(
"S-mode exception! scause={:#016x}, stval={:#016x}",
scause, stval
);
regs.dump();
error!(
"S-mode exception! scause={:#016x}, stval={:#016x}",
scause, stval
);
regs.dump();
if tp == 0 {
// Early failure - before misc::init and potentially before RMM init
// Do not attempt to trace stack because it would probably trap again
if tp == 0 {
// Early failure - before misc::init and potentially before RMM init
// Do not attempt to trace stack because it would probably trap again
loop {}
}
stack_trace();
loop {}
}
stack_trace();
loop {}
}
unsafe fn handle_interrupt(interrupt: usize) {
// FIXME retrieve from percpu area
// For now all the interrupts go to boot hart so this suffices...
let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed);
irqchip::hlic::interrupt(hart, interrupt);
unsafe {
// FIXME retrieve from percpu area
// For now all the interrupts go to boot hart so this suffices...
let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed);
irqchip::hlic::interrupt(hart, interrupt);
}
}
unsafe fn handle_user_exception(scause: usize, regs: &mut InterruptStack) {
if scause == USERMODE_ECALL {
let r = &mut regs.registers;
regs.iret.sepc += 4; // skip ecall
let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14);
r.x10 = ret;
return;
}
if scause == BREAKPOINT {
if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_some() {
unsafe {
if scause == USERMODE_ECALL {
let r = &mut regs.registers;
regs.iret.sepc += 4; // skip ecall
let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14);
r.x10 = ret;
return;
}
}
let stval: usize;
core::arch::asm!(
"csrr t0, stval",
lateout("t0") stval,
options(nostack)
);
if scause == BREAKPOINT {
if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None).is_some() {
return;
}
}
info!(
"U-mode exception! scause={:#016x}, stval={:#016x}",
scause, stval
);
regs.dump();
// TODO
/*
let signal = match scause {
0 | 4 | 6 | 18 | 19 => SIGBUS, // misaligned / machine check
2 | 8 | 9 => SIGILL, // Illegal instruction / breakpoint / ecall
BREAKPOINT => SIGTRAP,
_ => SIGSEGV,
};
*/
excp_handler(Exception { kind: scause });
}
unsafe fn page_fault(scause: usize, regs: &mut InterruptStack, user_mode: bool) -> bool {
if scause != INSTRUCTION_PAGE_FAULT && scause != LOAD_PAGE_FAULT && scause != STORE_PAGE_FAULT {
return false;
}
let stval: usize;
core::arch::asm!(
let stval: usize;
core::arch::asm!(
"csrr t0, stval",
lateout("t0") stval,
options(nostack)
);
);
let address = VirtualAddress::new(stval);
let mut generic_flags = GenericPfFlags::empty();
info!(
"U-mode exception! scause={:#016x}, stval={:#016x}",
scause, stval
);
regs.dump();
generic_flags.set(GenericPfFlags::INVOLVED_WRITE, scause == STORE_PAGE_FAULT);
generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, user_mode);
generic_flags.set(
GenericPfFlags::INSTR_NOT_DATA,
scause == INSTRUCTION_PAGE_FAULT,
);
// FIXME can these conditions be distinguished? Should they be?
generic_flags.set(GenericPfFlags::INVL, false);
generic_flags.set(GenericPfFlags::PRESENT, false);
crate::memory::page_fault_handler(regs, generic_flags, address).is_ok()
// TODO
/*
let signal = match scause {
0 | 4 | 6 | 18 | 19 => SIGBUS, // misaligned / machine check
2 | 8 | 9 => SIGILL, // Illegal instruction / breakpoint / ecall
BREAKPOINT => SIGTRAP,
_ => SIGSEGV,
};
*/
excp_handler(Exception { kind: scause });
}
}
unsafe fn page_fault(scause: usize, regs: &mut InterruptStack, user_mode: bool) -> bool {
unsafe {
if scause != INSTRUCTION_PAGE_FAULT
&& scause != LOAD_PAGE_FAULT
&& scause != STORE_PAGE_FAULT
{
return false;
}
let stval: usize;
core::arch::asm!(
"csrr t0, stval",
lateout("t0") stval,
options(nostack)
);
let address = VirtualAddress::new(stval);
let mut generic_flags = GenericPfFlags::empty();
generic_flags.set(GenericPfFlags::INVOLVED_WRITE, scause == STORE_PAGE_FAULT);
generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, user_mode);
generic_flags.set(
GenericPfFlags::INSTR_NOT_DATA,
scause == INSTRUCTION_PAGE_FAULT,
);
// FIXME can these conditions be distinguished? Should they be?
generic_flags.set(GenericPfFlags::INVL, false);
generic_flags.set(GenericPfFlags::PRESENT, false);
crate::memory::page_fault_handler(regs, generic_flags, address).is_ok()
}
}
src/arch/riscv64/interrupt/handler.rs
+13 -11
View File
@@ -302,15 +302,17 @@ macro_rules! pop_registers {
#[naked]
pub unsafe extern "C" fn enter_usermode() -> ! {
core::arch::naked_asm!(concat!(
"jalr s11\n",
"li t0, 1 << 8\n", // force U mode on sret
"csrc sstatus, t0\n",
"li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode)
"csrs sstatus, t0\n",
"addi t0, sp, 32 * 8\n", // save S mode stack to percpu
"sd t0, 8(tp)\n",
pop_registers!(),
"sret\n",
))
unsafe {
core::arch::naked_asm!(concat!(
"jalr s11\n",
"li t0, 1 << 8\n", // force U mode on sret
"csrc sstatus, t0\n",
"li t0, 0x6000\n", // set FS to dirty (enable FPU in U mode)
"csrs sstatus, t0\n",
"addi t0, sp, 32 * 8\n", // save S mode stack to percpu
"sd t0, 8(tp)\n",
pop_registers!(),
"sret\n",
))
}
}
src/arch/riscv64/interrupt/mod.rs
+12 -10
View File
@@ -12,13 +12,13 @@ pub use handler::InterruptStack;
/// Clear interrupts
#[inline(always)]
pub unsafe fn disable() {
asm!("csrci sstatus, 1 << 1")
unsafe { asm!("csrci sstatus, 1 << 1") }
}
/// Set interrupts
#[inline(always)]
pub unsafe fn enable() {
asm!("csrsi sstatus, 1 << 1")
unsafe { asm!("csrsi sstatus, 1 << 1") }
}
/// Set interrupts and halt
@@ -26,7 +26,7 @@ pub unsafe fn enable() {
/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
#[inline(always)]
pub unsafe fn enable_and_halt() {
asm!("csrsi sstatus, 1 << 1", "wfi")
unsafe { asm!("csrsi sstatus, 1 << 1", "wfi") }
}
/// Set interrupts and nop
@@ -34,13 +34,13 @@ pub unsafe fn enable_and_halt() {
/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
#[inline(always)]
pub unsafe fn enable_and_nop() {
asm!("csrsi sstatus, 1 << 1", "nop")
unsafe { asm!("csrsi sstatus, 1 << 1", "nop") }
}
/// Halt instruction
#[inline(always)]
pub unsafe fn halt() {
asm!("wfi", options(nomem, nostack))
unsafe { asm!("wfi", options(nomem, nostack)) }
}
/// Pause instruction
@@ -55,9 +55,11 @@ pub fn pause() {
#[inline(always)]
pub unsafe fn init() {
// Setup interrupt handlers
asm!(
"la t0, exception_handler", // WARL=0 - direct mode combined handler
"csrw stvec, t0"
);
unsafe {
// Setup interrupt handlers
asm!(
"la t0, exception_handler", // WARL=0 - direct mode combined handler
"csrw stvec, t0"
);
}
}
src/arch/riscv64/interrupt/trace.rs
+19 -15
View File
@@ -8,24 +8,28 @@ pub struct StackTrace {
impl StackTrace {
#[inline(always)]
pub unsafe fn start() -> Option<Self> {
let fp: usize;
asm!("mv {}, fp", out(reg) fp);
unsafe {
let fp: usize;
asm!("mv {}, fp", out(reg) fp);
let pc_ptr = fp.checked_sub(mem::size_of::<usize>())?;
let fp = pc_ptr.checked_sub(mem::size_of::<usize>())?;
Some(StackTrace {
fp,
pc_ptr: pc_ptr as *const usize,
})
let pc_ptr = fp.checked_sub(mem::size_of::<usize>())?;
let fp = pc_ptr.checked_sub(mem::size_of::<usize>())?;
Some(StackTrace {
fp,
pc_ptr: pc_ptr as *const usize,
})
}
}
pub unsafe fn next(self) -> Option<Self> {
let fp = *(self.fp as *const usize);
let pc_ptr = fp.checked_sub(mem::size_of::<usize>())?;
let fp = pc_ptr.checked_sub(mem::size_of::<usize>())?;
Some(StackTrace {
fp: fp,
pc_ptr: pc_ptr as *const usize,
})
unsafe {
let fp = *(self.fp as *const usize);
let pc_ptr = fp.checked_sub(mem::size_of::<usize>())?;
let fp = pc_ptr.checked_sub(mem::size_of::<usize>())?;
Some(StackTrace {
fp: fp,
pc_ptr: pc_ptr as *const usize,
})
}
}
}
src/arch/riscv64/macros.rs
+2 -2
View File
@@ -11,6 +11,6 @@ macro_rules! print {
#[macro_export]
macro_rules! println {
() => (print!("\n"));
($fmt:expr) => (print!(concat!($fmt, "\n")));
($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
($fmt:expr_2021) => (print!(concat!($fmt, "\n")));
($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
}
src/arch/riscv64/misc.rs
+14 -12
View File
@@ -28,18 +28,20 @@ impl PercpuBlock {
#[cold]
pub unsafe fn init(cpu_id: LogicalCpuId) {
let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
let virt = RmmA::phys_to_virt(frame.base()).data() as *mut ArchPercpu;
unsafe {
let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
let virt = RmmA::phys_to_virt(frame.base()).data() as *mut ArchPercpu;
virt.write(ArchPercpu {
tmp: 0,
s_sp: 0,
percpu: PercpuBlock::init(cpu_id),
});
virt.write(ArchPercpu {
tmp: 0,
s_sp: 0,
percpu: PercpuBlock::init(cpu_id),
});
asm!(
"mv tp, {}",
"csrw sscratch, tp",
in(reg) virt as usize
);
asm!(
"mv tp, {}",
"csrw sscratch, tp",
in(reg) virt as usize
);
}
}
src/arch/riscv64/mod.rs
+6 -4
View File
@@ -18,11 +18,12 @@ use core::arch::naked_asm;
pub use arch_copy_to_user as arch_copy_from_user;
#[link_section = ".usercopy-fns"]
#[unsafe(link_section = ".usercopy-fns")]
#[naked]
pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
naked_asm!(
"
unsafe {
naked_asm!(
"
addi sp, sp, -16
sd fp, 0(sp)
sd ra, 8(sp)
@@ -59,7 +60,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
5: mv a0, x0
ret
"
)
)
}
}
pub const KFX_SIZE: usize = 1024;
src/arch/riscv64/start.rs
+154 -151
View File
@@ -65,168 +65,171 @@ fn get_boot_hart_id(env: &[u8]) -> Option<usize> {
}
/// The entry to Rust, all things must be initialized
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! {
asm!(
"mv tp, x0", // reset percpu until it is initialized
"csrw sscratch, tp",
"sd x0, -16(fp)", // and stop frame walker here
"sd x0, -8(fp)",
);
let bootstrap = {
let args = &*args_ptr;
// BSS should already be zero
{
assert_eq!(BSS_TEST_ZERO, 0);
assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
}
KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst);
KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst);
let env = slice::from_raw_parts(
(crate::PHYS_OFFSET + args.env_base) as *const u8,
args.env_size,
unsafe {
asm!(
"mv tp, x0", // reset percpu until it is initialized
"csrw sscratch, tp",
"sd x0, -16(fp)", // and stop frame walker here
"sd x0, -8(fp)",
);
let dtb_data = if args.acpi_base != 0 {
Some((crate::PHYS_OFFSET + args.acpi_base, args.acpi_size))
} else {
None
};
let dtb = dtb_data
.map(|(base, size)| unsafe { slice::from_raw_parts(base as *const u8, size) })
.and_then(|data| Fdt::new(data).ok());
let bootstrap = {
let args = &*args_ptr;
#[cfg(feature = "graphical_debug")]
graphical_debug::init(env);
// BSS should already be zero
{
assert_eq!(BSS_TEST_ZERO, 0);
assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
}
#[cfg(feature = "serial_debug")]
if let Some(dtb) = &dtb {
init_early(dtb);
}
KERNEL_BASE.store(args.kernel_base, Ordering::SeqCst);
KERNEL_SIZE.store(args.kernel_size, Ordering::SeqCst);
// Initialize logger
crate::log::init_logger(|r| {
use core::fmt::Write;
let _ = write!(
crate::debug::Writer::new(),
"{}:{} -- {}\n",
r.target(),
r.level(),
r.args()
let env = slice::from_raw_parts(
(crate::PHYS_OFFSET + args.env_base) as *const u8,
args.env_size,
);
});
::log::set_max_level(::log::LevelFilter::Debug);
info!("Redox OS starting...");
info!(
"Kernel: {:X}:{:X}",
{ args.kernel_base },
args.kernel_base + args.kernel_size
);
info!(
"Stack: {:X}:{:X}",
{ args.stack_base },
args.stack_base + args.stack_size
);
info!(
"Env: {:X}:{:X}",
{ args.env_base },
args.env_base + args.env_size
);
info!(
"RSDPs: {:X}:{:X}",
{ args.acpi_size },
args.acpi_size + args.acpi_size
);
info!(
"Areas: {:X}:{:X}",
{ args.areas_base },
args.areas_base + args.areas_size
);
info!(
"Bootstrap: {:X}:{:X}",
{ args.bootstrap_base },
args.bootstrap_base + args.bootstrap_size
);
let dtb_data = if args.acpi_base != 0 {
Some((crate::PHYS_OFFSET + args.acpi_base, args.acpi_size))
} else {
None
};
let dtb = dtb_data
.map(|(base, size)| unsafe { slice::from_raw_parts(base as *const u8, size) })
.and_then(|data| Fdt::new(data).ok());
if let Some(dtb) = &dtb {
device::dump_fdt(&dtb);
}
#[cfg(feature = "graphical_debug")]
graphical_debug::init(env);
interrupt::init();
#[cfg(feature = "serial_debug")]
if let Some(dtb) = &dtb {
init_early(dtb);
}
let bootstrap = crate::Bootstrap {
base: Frame::containing(PhysicalAddress::new(args.bootstrap_base)),
page_count: args.bootstrap_size / PAGE_SIZE,
env,
// Initialize logger
crate::log::init_logger(|r| {
use core::fmt::Write;
let _ = write!(
crate::debug::Writer::new(),
"{}:{} -- {}\n",
r.target(),
r.level(),
r.args()
);
});
::log::set_max_level(::log::LevelFilter::Debug);
info!("Redox OS starting...");
info!(
"Kernel: {:X}:{:X}",
{ args.kernel_base },
args.kernel_base + args.kernel_size
);
info!(
"Stack: {:X}:{:X}",
{ args.stack_base },
args.stack_base + args.stack_size
);
info!(
"Env: {:X}:{:X}",
{ args.env_base },
args.env_base + args.env_size
);
info!(
"RSDPs: {:X}:{:X}",
{ args.acpi_size },
args.acpi_size + args.acpi_size
);
info!(
"Areas: {:X}:{:X}",
{ args.areas_base },
args.areas_base + args.areas_size
);
info!(
"Bootstrap: {:X}:{:X}",
{ args.bootstrap_base },
args.bootstrap_base + args.bootstrap_size
);
if let Some(dtb) = &dtb {
device::dump_fdt(&dtb);
}
interrupt::init();
let bootstrap = crate::Bootstrap {
base: Frame::containing(PhysicalAddress::new(args.bootstrap_base)),
page_count: args.bootstrap_size / PAGE_SIZE,
env,
};
// Initialize RMM
register_bootloader_areas(args.areas_base, args.areas_size);
if let Some(dt) = &dtb {
register_dev_memory_ranges(dt);
}
register_memory_region(
args.kernel_base,
args.kernel_size,
BootloaderMemoryKind::Kernel,
);
register_memory_region(
args.stack_base,
args.stack_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.env_base,
args.env_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.acpi_base,
args.acpi_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.bootstrap_base,
args.bootstrap_size,
BootloaderMemoryKind::IdentityMap,
);
crate::startup::memory::init(None, None);
let boot_hart_id =
get_boot_hart_id(env).expect("Didn't get boot HART id from bootloader");
info!("Booting on HART {}", boot_hart_id);
BOOT_HART_ID.store(boot_hart_id, Ordering::Relaxed);
paging::init();
crate::misc::init(crate::cpu_set::LogicalCpuId::new(0));
CPU_COUNT.store(1, Ordering::SeqCst);
// Setup kernel heap
allocator::init();
// Activate memory logging
crate::log::init();
crate::dtb::init(dtb_data);
// Initialize devices
device::init();
// Initialize all of the non-core devices not otherwise needed to complete initialization
device::init_noncore();
// FIXME bringup AP HARTs
bootstrap
};
// Initialize RMM
register_bootloader_areas(args.areas_base, args.areas_size);
if let Some(dt) = &dtb {
register_dev_memory_ranges(dt);
}
register_memory_region(
args.kernel_base,
args.kernel_size,
BootloaderMemoryKind::Kernel,
);
register_memory_region(
args.stack_base,
args.stack_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.env_base,
args.env_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.acpi_base,
args.acpi_size,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.bootstrap_base,
args.bootstrap_size,
BootloaderMemoryKind::IdentityMap,
);
crate::startup::memory::init(None, None);
let boot_hart_id = get_boot_hart_id(env).expect("Didn't get boot HART id from bootloader");
info!("Booting on HART {}", boot_hart_id);
BOOT_HART_ID.store(boot_hart_id, Ordering::Relaxed);
paging::init();
crate::misc::init(crate::cpu_set::LogicalCpuId::new(0));
CPU_COUNT.store(1, Ordering::SeqCst);
// Setup kernel heap
allocator::init();
// Activate memory logging
crate::log::init();
crate::dtb::init(dtb_data);
// Initialize devices
device::init();
// Initialize all of the non-core devices not otherwise needed to complete initialization
device::init_noncore();
// FIXME bringup AP HARTs
bootstrap
};
crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap);
crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap);
}
}
src/arch/x86/gdt.rs
+76 -65
View File
@@ -135,9 +135,11 @@ pub struct ProcessorControlRegion {
pub struct TssWrapper(pub TaskStateSegment);
pub unsafe fn pcr() -> *mut ProcessorControlRegion {
let mut ret: *mut ProcessorControlRegion;
core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref)));
ret
unsafe {
let mut ret: *mut ProcessorControlRegion;
core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref)));
ret
}
}
#[cfg(feature = "pti")]
@@ -151,84 +153,93 @@ pub unsafe fn set_tss_stack(stack: usize) {
#[cfg(not(feature = "pti"))]
pub unsafe fn set_tss_stack(stack: usize) {
addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16);
addr_of_mut!((*pcr()).tss.0.esp0).write(stack as u32);
unsafe {
addr_of_mut!((*pcr()).tss.0.ss0).write((GDT_KERNEL_DATA << 3) as u16);
addr_of_mut!((*pcr()).tss.0.esp0).write(stack as u32);
}
}
pub unsafe fn set_userspace_io_allowed(allowed: bool) {
addr_of_mut!((*pcr()).tss.0.iobp_offset).write(if allowed {
mem::size_of::<TaskStateSegment>() as u16
} else {
0xFFFF
});
unsafe {
addr_of_mut!((*pcr()).tss.0.iobp_offset).write(if allowed {
mem::size_of::<TaskStateSegment>() as u16
} else {
0xFFFF
});
}
}
/// Initialize a minimal GDT without configuring percpu.
pub unsafe fn init() {
// Load the initial GDT, before the kernel remaps itself.
dtables::lgdt(&DescriptorTablePointer {
limit: (INIT_GDT.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: INIT_GDT.as_ptr() as *const SegmentDescriptor,
});
unsafe {
// Load the initial GDT, before the kernel remaps itself.
dtables::lgdt(&DescriptorTablePointer {
limit: (INIT_GDT.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: INIT_GDT.as_ptr() as *const SegmentDescriptor,
});
// Load the segment descriptors
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
// Load the segment descriptors
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_es(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_fs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
}
}
/// Initialize GDT and configure percpu.
pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
let alloc_order = mem::size_of::<ProcessorControlRegion>()
.div_ceil(PAGE_SIZE)
.next_power_of_two()
.trailing_zeros();
let pcr_frame =
crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR frame");
let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion);
unsafe {
let alloc_order = mem::size_of::<ProcessorControlRegion>()
.div_ceil(PAGE_SIZE)
.next_power_of_two()
.trailing_zeros();
let pcr_frame =
crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR frame");
let pcr =
&mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion);
pcr.self_ref = pcr as *const _ as usize;
pcr.gdt = BASE_GDT;
pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32);
pcr.self_ref = pcr as *const _ as usize;
pcr.gdt = BASE_GDT;
pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32);
let gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
limit: (pcr.gdt.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: pcr.gdt.as_ptr() as *const SegmentDescriptor,
};
let gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
limit: (pcr.gdt.len() * mem::size_of::<GdtEntry>() - 1) as u16,
base: pcr.gdt.as_ptr() as *const SegmentDescriptor,
};
{
pcr._all_ones = 0xFF;
pcr.tss.0.iobp_offset = 0xFFFF;
let tss = &pcr.tss.0 as *const _ as usize as u32;
{
pcr._all_ones = 0xFF;
pcr.tss.0.iobp_offset = 0xFFFF;
let tss = &pcr.tss.0 as *const _ as usize as u32;
pcr.gdt[GDT_TSS].set_offset(tss);
pcr.gdt[GDT_TSS]
.set_limit(mem::size_of::<TaskStateSegment>() as u32 + IOBITMAP_SIZE as u32);
pcr.gdt[GDT_TSS].set_offset(tss);
pcr.gdt[GDT_TSS]
.set_limit(mem::size_of::<TaskStateSegment>() as u32 + IOBITMAP_SIZE as u32);
}
// Load the new GDT, which is correctly located in thread local storage.
dtables::lgdt(&gdtr);
// Reload the segment descriptors
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3));
segmentation::load_es(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
// TODO: Use FS for kernel percpu on i686?
segmentation::load_fs(SegmentSelector::new(GDT_USER_FS as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0));
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(stack_offset);
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
pcr.percpu = crate::percpu::PercpuBlock::init(cpu_id);
crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu);
}
// Load the new GDT, which is correctly located in thread local storage.
dtables::lgdt(&gdtr);
// Reload the segment descriptors
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3));
segmentation::load_es(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
// TODO: Use FS for kernel percpu on i686?
segmentation::load_fs(SegmentSelector::new(GDT_USER_FS as u16, Ring::Ring0));
segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0));
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(stack_offset);
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
pcr.percpu = crate::percpu::PercpuBlock::init(cpu_id);
crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu);
}
#[derive(Copy, Clone, Debug)]
src/arch/x86/interrupt/handler.rs
+25 -21
View File
@@ -267,7 +267,7 @@ macro_rules! interrupt_stack {
// use idents directly instead.
($name:ident, |$stack:ident| $code:block) => {
#[naked]
pub unsafe extern "C" fn $name() {
pub unsafe extern "C" fn $name() { unsafe {
unsafe extern "fastcall" fn inner($stack: &mut $crate::arch::x86::interrupt::InterruptStack) {
// TODO: Force the declarations to specify unsafe?
@@ -308,7 +308,7 @@ macro_rules! interrupt_stack {
),
inner = sym inner,
);
}
}}
};
($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, |$stack| $code); };
($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, |$stack| $code); }
@@ -318,10 +318,10 @@ macro_rules! interrupt_stack {
macro_rules! interrupt {
($name:ident, || $code:block) => {
#[naked]
pub unsafe extern "C" fn $name() {
unsafe extern "C" fn inner() {
pub unsafe extern "C" fn $name() { unsafe {
unsafe extern "C" fn inner() { unsafe {
$code
}
}}
core::arch::naked_asm!(concat!(
// Backup all userspace registers to stack
@@ -350,7 +350,7 @@ macro_rules! interrupt {
),
inner = sym inner,
);
}
}}
};
}
@@ -358,7 +358,7 @@ macro_rules! interrupt {
macro_rules! interrupt_error {
($name:ident, |$stack:ident| $code:block) => {
#[naked]
pub unsafe extern "C" fn $name() {
pub unsafe extern "C" fn $name() { unsafe {
unsafe extern "C" fn inner($stack: &mut $crate::arch::x86::interrupt::handler::InterruptErrorStack) {
#[allow(unused_unsafe)]
unsafe {
@@ -408,20 +408,22 @@ macro_rules! interrupt_error {
"iretd\n",
),
inner = sym inner);
}
}}
};
}
#[naked]
unsafe extern "C" fn usercopy_trampoline() {
core::arch::naked_asm!(
"
unsafe {
core::arch::naked_asm!(
"
pop esi
pop edi
mov eax, 1
ret
"
);
);
}
}
impl ArchIntCtx for InterruptStack {
@@ -443,15 +445,17 @@ impl ArchIntCtx for InterruptStack {
#[naked]
pub unsafe extern "C" fn enter_usermode() {
core::arch::naked_asm!(concat!(
// TODO: Unmap PTI
// $crate::arch::x86::pti::unmap();
unsafe {
core::arch::naked_asm!(concat!(
// TODO: Unmap PTI
// $crate::arch::x86::pti::unmap();
// Exit kernel TLS segment
exit_gs!(),
// Restore all userspace registers
pop_preserved!(),
pop_scratch!(),
"iretd\n",
))
// Exit kernel TLS segment
exit_gs!(),
// Restore all userspace registers
pop_preserved!(),
pop_scratch!(),
"iretd\n",
))
}
}
src/arch/x86/interrupt/irq.rs
+57 -39
View File
@@ -71,85 +71,101 @@ fn irq_method() -> IrqMethod {
/// Notify the IRQ scheme that an IRQ has been registered. This should mask the IRQ until the
/// scheme user unmasks it ("acknowledges" it).
unsafe fn trigger(irq: u8) {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_mask(irq)
unsafe {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_mask(irq)
}
}
IrqMethod::Apic => ioapic_mask(irq),
}
IrqMethod::Apic => ioapic_mask(irq),
irq_trigger(irq);
}
irq_trigger(irq);
}
/// Unmask the IRQ. This is called from the IRQ scheme, which does this when a user process has
/// processed the IRQ.
pub unsafe fn acknowledge(irq: usize) {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_unmask(irq)
unsafe {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_unmask(irq)
}
}
IrqMethod::Apic => ioapic_unmask(irq),
}
IrqMethod::Apic => ioapic_unmask(irq),
}
}
/// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one.
pub unsafe fn eoi(irq: u8) {
PercpuBlock::current().stats.add_irq(irq);
unsafe {
PercpuBlock::current().stats.add_irq(irq);
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_eoi(irq)
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_eoi(irq)
}
}
IrqMethod::Apic => lapic_eoi(),
}
IrqMethod::Apic => lapic_eoi(),
}
}
unsafe fn pic_mask(irq: u8) {
debug_assert!(irq < 16);
unsafe {
debug_assert!(irq < 16);
if irq >= 8 {
pic::slave().mask_set(irq - 8);
} else {
pic::master().mask_set(irq);
if irq >= 8 {
pic::slave().mask_set(irq - 8);
} else {
pic::master().mask_set(irq);
}
}
}
unsafe fn ioapic_mask(irq: u8) {
ioapic::mask(irq);
unsafe {
ioapic::mask(irq);
}
}
unsafe fn pic_eoi(irq: u8) {
debug_assert!(irq < 16);
unsafe {
debug_assert!(irq < 16);
if irq >= 8 {
pic::master().ack();
pic::slave().ack();
} else {
pic::master().ack();
if irq >= 8 {
pic::master().ack();
pic::slave().ack();
} else {
pic::master().ack();
}
}
}
unsafe fn lapic_eoi() {
local_apic::the_local_apic().eoi()
unsafe { local_apic::the_local_apic().eoi() }
}
unsafe fn pic_unmask(irq: usize) {
debug_assert!(irq < 16);
unsafe {
debug_assert!(irq < 16);
if irq >= 8 {
pic::slave().mask_clear(irq as u8 - 8);
} else {
pic::master().mask_clear(irq as u8);
if irq >= 8 {
pic::slave().mask_clear(irq as u8 - 8);
} else {
pic::master().mask_clear(irq as u8);
}
}
}
unsafe fn ioapic_unmask(irq: usize) {
ioapic::unmask(irq as u8);
unsafe {
ioapic::unmask(irq as u8);
}
}
interrupt_stack!(pit_stack, |_stack| {
@@ -286,7 +302,7 @@ interrupt!(lapic_error, || {
// XXX: This would look way prettier using const generics.
macro_rules! allocatable_irq(
( $idt:expr, $number:literal, $name:ident ) => {
( $idt:expr_2021, $number:literal, $name:ident ) => {
interrupt!($name, || {
allocatable_irq_generic($number);
});
@@ -294,8 +310,10 @@ macro_rules! allocatable_irq(
);
pub unsafe fn allocatable_irq_generic(number: u8) {
irq_trigger(number - 32);
lapic_eoi();
unsafe {
irq_trigger(number - 32);
lapic_eoi();
}
}
define_default_irqs!();
src/arch/x86/macros.rs
+4 -4
View File
@@ -11,13 +11,13 @@ macro_rules! print {
#[macro_export]
macro_rules! println {
() => (print!("\n"));
($fmt:expr) => (print!(concat!($fmt, "\n")));
($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
($fmt:expr_2021) => (print!(concat!($fmt, "\n")));
($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
}
#[macro_export]
macro_rules! irqs(
( [ $( ($idt:expr, $number:literal, $name:ident) ,)* ], $submac:ident ) => {
( [ $( ($idt:expr_2021, $number:literal, $name:ident) ,)* ], $submac:ident ) => {
$(
$submac!($idt, $number, $name);
)*
@@ -28,7 +28,7 @@ macro_rules! irqs(
// allocatable_irq_NUM.
#[macro_export]
macro_rules! default_irqs(
($idt:expr, $submac:ident) => {
($idt:expr_2021, $submac:ident) => {
irqs!([
// interrupt vectors below 32 are exceptions
// vectors 32..=47 are used for standard 8259 pic irqs.
src/arch/x86/mod.rs
+6 -4
View File
@@ -20,10 +20,11 @@ pub mod flags {
}
#[naked]
#[link_section = ".usercopy-fns"]
#[unsafe(link_section = ".usercopy-fns")]
pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
core::arch::naked_asm!(
"
unsafe {
core::arch::naked_asm!(
"
push edi
push esi
@@ -38,7 +39,8 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
xor eax, eax
ret
"
);
);
}
}
pub use arch_copy_to_user as arch_copy_from_user;
src/arch/x86_64/alternative.rs
+196 -192
View File
@@ -38,212 +38,216 @@ pub struct AltReloc {
#[cold]
pub unsafe fn early_init(bsp: bool) {
let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start();
let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset;
unsafe {
let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start();
let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset;
assert_eq!(relocs_size % size_of::<AltReloc>(), 0);
let relocs = core::slice::from_raw_parts(
relocs_offset as *const AltReloc,
relocs_size / size_of::<AltReloc>(),
);
let mut enable = KcpuFeatures::empty();
if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) {
// SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any
// userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This
// limits user-memory accesses to the UserSlice wrapper, so that no data outside of
// usercopy functions can be accidentally accessed by the kernel.
x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP);
// Clear CLAC in (the probably unlikely) case the bootloader set it earlier.
x86::bits64::rflags::clac();
enable |= KcpuFeatures::SMAP;
} else {
assert!(cfg!(not(cpu_feature_always = "smap")));
}
if cfg!(not(cpu_feature_never = "fsgsbase"))
&& let Some(f) = cpuid().get_extended_feature_info()
&& f.has_fsgsbase()
{
x86::controlregs::cr4_write(
x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE,
assert_eq!(relocs_size % size_of::<AltReloc>(), 0);
let relocs = core::slice::from_raw_parts(
relocs_offset as *const AltReloc,
relocs_size / size_of::<AltReloc>(),
);
enable |= KcpuFeatures::FSGSBASE;
} else {
assert!(cfg!(not(cpu_feature_always = "fsgsbase")));
let mut enable = KcpuFeatures::empty();
if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) {
// SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any
// userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This
// limits user-memory accesses to the UserSlice wrapper, so that no data outside of
// usercopy functions can be accidentally accessed by the kernel.
x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP);
// Clear CLAC in (the probably unlikely) case the bootloader set it earlier.
x86::bits64::rflags::clac();
enable |= KcpuFeatures::SMAP;
} else {
assert!(cfg!(not(cpu_feature_always = "smap")));
}
if cfg!(not(cpu_feature_never = "fsgsbase"))
&& let Some(f) = cpuid().get_extended_feature_info()
&& f.has_fsgsbase()
{
x86::controlregs::cr4_write(
x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE,
);
enable |= KcpuFeatures::FSGSBASE;
} else {
assert!(cfg!(not(cpu_feature_always = "fsgsbase")));
}
#[cfg(not(cpu_feature_never = "xsave"))]
if feature_info().has_xsave() {
use raw_cpuid::{ExtendedRegisterStateLocation, ExtendedRegisterType};
x86::controlregs::cr4_write(
x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE,
);
let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE;
x86::controlregs::xcr0_write(xcr0);
let ext_state_info = cpuid()
.get_extended_state_info()
.expect("must be present if XSAVE is supported");
enable |= KcpuFeatures::XSAVE;
enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt());
let info = xsave::XsaveInfo {
ymm_upper_offset: feature_info().has_avx().then(|| {
xcr0 |= Xcr0::XCR0_AVX_STATE;
x86::controlregs::xcr0_write(xcr0);
let state = ext_state_info
.iter()
.find(|state| {
state.register() == ExtendedRegisterType::Avx
&& state.location() == ExtendedRegisterStateLocation::Xcr0
})
.expect("CPUID said AVX was supported but there's no state info");
if state.size() as usize != 16 * core::mem::size_of::<u128>() {
log::warn!("Unusual AVX state size {}", state.size());
}
state.offset()
}),
xsave_size: ext_state_info.xsave_area_size_enabled_features(),
};
log::debug!("XSAVE: {:?}", info);
xsave::XSAVE_INFO.call_once(|| info);
} else {
assert!(cfg!(not(cpu_feature_always = "xsave")));
}
if !bsp {
return;
}
overwrite(&relocs, enable);
if cfg!(not(feature = "self_modifying")) {
assert!(
cfg!(not(cpu_feature_auto = "smap"))
&& cfg!(not(cpu_feature_auto = "fsgsbase"))
&& cfg!(not(cpu_feature_auto = "xsave"))
&& cfg!(not(cpu_feature_auto = "xsaveopt"))
);
}
FEATURES.call_once(|| enable);
}
#[cfg(not(cpu_feature_never = "xsave"))]
if feature_info().has_xsave() {
use raw_cpuid::{ExtendedRegisterStateLocation, ExtendedRegisterType};
x86::controlregs::cr4_write(
x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE,
);
let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE;
x86::controlregs::xcr0_write(xcr0);
let ext_state_info = cpuid()
.get_extended_state_info()
.expect("must be present if XSAVE is supported");
enable |= KcpuFeatures::XSAVE;
enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt());
let info = xsave::XsaveInfo {
ymm_upper_offset: feature_info().has_avx().then(|| {
xcr0 |= Xcr0::XCR0_AVX_STATE;
x86::controlregs::xcr0_write(xcr0);
let state = ext_state_info
.iter()
.find(|state| {
state.register() == ExtendedRegisterType::Avx
&& state.location() == ExtendedRegisterStateLocation::Xcr0
})
.expect("CPUID said AVX was supported but there's no state info");
if state.size() as usize != 16 * core::mem::size_of::<u128>() {
log::warn!("Unusual AVX state size {}", state.size());
}
state.offset()
}),
xsave_size: ext_state_info.xsave_area_size_enabled_features(),
};
log::debug!("XSAVE: {:?}", info);
xsave::XSAVE_INFO.call_once(|| info);
} else {
assert!(cfg!(not(cpu_feature_always = "xsave")));
}
if !bsp {
return;
}
overwrite(&relocs, enable);
if cfg!(not(feature = "self_modifying")) {
assert!(
cfg!(not(cpu_feature_auto = "smap"))
&& cfg!(not(cpu_feature_auto = "fsgsbase"))
&& cfg!(not(cpu_feature_auto = "xsave"))
&& cfg!(not(cpu_feature_auto = "xsaveopt"))
);
}
FEATURES.call_once(|| enable);
}
unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) {
if cfg!(not(feature = "self_modifying")) {
return;
}
log::info!("self-modifying features: {:?}", enable);
let mut mapper = KernelMapper::lock();
for reloc in relocs.iter().copied() {
let name = core::str::from_utf8(core::slice::from_raw_parts(
reloc.name_start,
reloc.name_len,
))
.expect("invalid feature name");
let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len);
let dst_pages = PageSpan::between(
Page::containing_address(VirtualAddress::new(reloc.code_start as usize)),
Page::containing_address(VirtualAddress::new(
(reloc.code_start as usize + reloc.padded_len).next_multiple_of(PAGE_SIZE),
)),
);
for page in dst_pages.pages() {
mapper
.get_mut()
.unwrap()
.remap(
page.start_address(),
PageFlags::new().write(true).execute(true).global(true),
)
.unwrap()
.flush();
unsafe {
if cfg!(not(feature = "self_modifying")) {
return;
}
let code = core::slice::from_raw_parts_mut(reloc.code_start, reloc.padded_len);
log::info!("self-modifying features: {:?}", enable);
log::trace!(
"feature {} current {:x?} altcode {:x?}",
name,
code,
altcode
);
let mut mapper = KernelMapper::lock();
for reloc in relocs.iter().copied() {
let name = core::str::from_utf8(core::slice::from_raw_parts(
reloc.name_start,
reloc.name_len,
))
.expect("invalid feature name");
let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len);
let feature_is_enabled = match name {
"smap" => enable.contains(KcpuFeatures::SMAP),
"fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE),
"xsave" => enable.contains(KcpuFeatures::XSAVE),
"xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT),
//_ => panic!("unknown altcode relocation: {}", name),
_ => true,
};
// XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X`
// only outputs the (slower) single-byte 0x90 NOP.
// This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November
// 2020).
const NOPS_TABLE: [&[u8]; 11] = [
&[0x90],
&[0x66, 0x90],
&[0x0f, 0x1f, 0x00],
&[0x0f, 0x1f, 0x40, 0x00],
&[0x0f, 0x1f, 0x44, 0x00, 0x00],
&[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00],
&[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00],
&[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
&[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
&[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
&[
0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
],
];
if feature_is_enabled {
log::trace!("feature {} origcode {:x?}", name, code);
let (dst, dst_nops) = code.split_at_mut(altcode.len());
dst.copy_from_slice(altcode);
for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) {
chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]);
}
log::trace!("feature {} new {:x?} altcode {:x?}", name, code, altcode);
} else {
log::trace!("feature !{} origcode {:x?}", name, code);
let (_, padded) = code.split_at_mut(reloc.origcode_len);
// Not strictly necessary, but reduces the number of instructions using longer nop
// instructions.
for chunk in padded.chunks_mut(NOPS_TABLE.len()) {
chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]);
let dst_pages = PageSpan::between(
Page::containing_address(VirtualAddress::new(reloc.code_start as usize)),
Page::containing_address(VirtualAddress::new(
(reloc.code_start as usize + reloc.padded_len).next_multiple_of(PAGE_SIZE),
)),
);
for page in dst_pages.pages() {
mapper
.get_mut()
.unwrap()
.remap(
page.start_address(),
PageFlags::new().write(true).execute(true).global(true),
)
.unwrap()
.flush();
}
log::trace!("feature !{} new {:x?}", name, code);
}
let code = core::slice::from_raw_parts_mut(reloc.code_start, reloc.padded_len);
for page in dst_pages.pages() {
mapper
.get_mut()
.unwrap()
.remap(
page.start_address(),
PageFlags::new().write(false).execute(true).global(true),
)
.unwrap()
.flush();
log::trace!(
"feature {} current {:x?} altcode {:x?}",
name,
code,
altcode
);
let feature_is_enabled = match name {
"smap" => enable.contains(KcpuFeatures::SMAP),
"fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE),
"xsave" => enable.contains(KcpuFeatures::XSAVE),
"xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT),
//_ => panic!("unknown altcode relocation: {}", name),
_ => true,
};
// XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X`
// only outputs the (slower) single-byte 0x90 NOP.
// This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November
// 2020).
const NOPS_TABLE: [&[u8]; 11] = [
&[0x90],
&[0x66, 0x90],
&[0x0f, 0x1f, 0x00],
&[0x0f, 0x1f, 0x40, 0x00],
&[0x0f, 0x1f, 0x44, 0x00, 0x00],
&[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00],
&[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00],
&[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
&[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
&[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
&[
0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
],
];
if feature_is_enabled {
log::trace!("feature {} origcode {:x?}", name, code);
let (dst, dst_nops) = code.split_at_mut(altcode.len());
dst.copy_from_slice(altcode);
for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) {
chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]);
}
log::trace!("feature {} new {:x?} altcode {:x?}", name, code, altcode);
} else {
log::trace!("feature !{} origcode {:x?}", name, code);
let (_, padded) = code.split_at_mut(reloc.origcode_len);
// Not strictly necessary, but reduces the number of instructions using longer nop
// instructions.
for chunk in padded.chunks_mut(NOPS_TABLE.len()) {
chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]);
}
log::trace!("feature !{} new {:x?}", name, code);
}
for page in dst_pages.pages() {
mapper
.get_mut()
.unwrap()
.remap(
page.start_address(),
PageFlags::new().write(false).execute(true).global(true),
)
.unwrap()
.flush();
}
}
}
}
src/arch/x86_64/gdt.rs
+96 -81
View File
@@ -142,12 +142,14 @@ const _: () = {
};
pub unsafe fn pcr() -> *mut ProcessorControlRegion {
// Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that
// obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably
// microcoded) instructions.
let mut ret: *mut ProcessorControlRegion;
core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref)));
ret
unsafe {
// Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that
// obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably
// microcoded) instructions.
let mut ret: *mut ProcessorControlRegion;
core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref)));
ret
}
}
#[cfg(feature = "pti")]
@@ -160,109 +162,122 @@ pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) {
#[cfg(not(feature = "pti"))]
pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) {
// TODO: If this increases performance, read gs:[offset] directly
core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64);
unsafe {
// TODO: If this increases performance, read gs:[offset] directly
core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64);
}
}
pub unsafe fn set_userspace_io_allowed(pcr: *mut ProcessorControlRegion, allowed: bool) {
let offset = if allowed {
u16::try_from(size_of::<TaskStateSegment>()).unwrap()
} else {
0xFFFF
};
core::ptr::addr_of_mut!((*pcr).tss.iomap_base).write(offset);
unsafe {
let offset = if allowed {
u16::try_from(size_of::<TaskStateSegment>()).unwrap()
} else {
0xFFFF
};
core::ptr::addr_of_mut!((*pcr).tss.iomap_base).write(offset);
}
}
// Initialize startup GDT
#[cold]
pub unsafe fn init() {
// Before the kernel can remap itself, it needs to switch to a GDT it controls. Start with a
// minimal kernel-only GDT.
dtables::lgdt(&DescriptorTablePointer {
limit: (INIT_GDT.len() * size_of::<GdtEntry>() - 1) as u16,
base: INIT_GDT.as_ptr() as *const SegmentDescriptor,
});
unsafe {
// Before the kernel can remap itself, it needs to switch to a GDT it controls. Start with a
// minimal kernel-only GDT.
dtables::lgdt(&DescriptorTablePointer {
limit: (INIT_GDT.len() * size_of::<GdtEntry>() - 1) as u16,
base: INIT_GDT.as_ptr() as *const SegmentDescriptor,
});
load_segments();
load_segments();
}
}
#[cold]
unsafe fn load_segments() {
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
unsafe {
segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
segmentation::load_ds(SegmentSelector::from_raw(0));
segmentation::load_es(SegmentSelector::from_raw(0));
segmentation::load_fs(SegmentSelector::from_raw(0));
segmentation::load_ds(SegmentSelector::from_raw(0));
segmentation::load_es(SegmentSelector::from_raw(0));
segmentation::load_fs(SegmentSelector::from_raw(0));
// What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However,
// GSBASE is set later, and percpu is not used until gdt::init_paging().
segmentation::load_gs(SegmentSelector::from_raw(0));
// What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However,
// GSBASE is set later, and percpu is not used until gdt::init_paging().
segmentation::load_gs(SegmentSelector::from_raw(0));
}
}
/// Initialize GDT and PCR.
#[cold]
pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
let alloc_order = size_of::<ProcessorControlRegion>()
.div_ceil(PAGE_SIZE)
.next_power_of_two()
.trailing_zeros();
let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR");
let pcr = &mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion);
unsafe {
let alloc_order = size_of::<ProcessorControlRegion>()
.div_ceil(PAGE_SIZE)
.next_power_of_two()
.trailing_zeros();
let pcr_frame =
crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR");
let pcr =
&mut *(RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion);
pcr.self_ref = pcr as *mut ProcessorControlRegion as usize;
pcr.self_ref = pcr as *mut ProcessorControlRegion as usize;
// Setup the GDT.
pcr.gdt = BASE_GDT;
// Setup the GDT.
pcr.gdt = BASE_GDT;
let limit = (pcr.gdt.len() * size_of::<GdtEntry>() - 1)
.try_into()
.expect("main GDT way too large");
let base = pcr.gdt.as_ptr() as *const SegmentDescriptor;
let limit = (pcr.gdt.len() * size_of::<GdtEntry>() - 1)
.try_into()
.expect("main GDT way too large");
let base = pcr.gdt.as_ptr() as *const SegmentDescriptor;
let gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer { limit, base };
let gdtr: DescriptorTablePointer<SegmentDescriptor> =
DescriptorTablePointer { limit, base };
{
pcr.tss.iomap_base = 0xFFFF;
pcr._all_ones = 0xFF;
{
pcr.tss.iomap_base = 0xFFFF;
pcr._all_ones = 0xFF;
let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64;
let tss_lo = (tss & 0xFFFF_FFFF) as u32;
let tss_hi = (tss >> 32) as u32;
let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64;
let tss_lo = (tss & 0xFFFF_FFFF) as u32;
let tss_hi = (tss >> 32) as u32;
pcr.gdt[GDT_TSS].set_offset(tss_lo);
pcr.gdt[GDT_TSS].set_limit(size_of::<TaskStateSegment>() as u32 + IOBITMAP_SIZE);
pcr.gdt[GDT_TSS].set_offset(tss_lo);
pcr.gdt[GDT_TSS].set_limit(size_of::<TaskStateSegment>() as u32 + IOBITMAP_SIZE);
(&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry)
.cast::<u32>()
.write(tss_hi);
(&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry)
.cast::<u32>()
.write(tss_hi);
}
// Load the new GDT, which is correctly located in thread local storage.
dtables::lgdt(&gdtr);
// Load segments again, possibly resetting FSBASE and GSBASE.
load_segments();
// Ensure that GSBASE always points to the PCR in kernel space.
x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64);
// While GSBASE points to the PCR in kernel space, userspace is free to set it to other values.
// Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is
// because entering usermode will entail executing the SWAPGS instruction.
x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0);
// Set the userspace FSBASE to zero.
x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0);
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(pcr, stack_offset);
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
pcr.percpu = PercpuBlock::init(cpu_id);
crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu);
}
// Load the new GDT, which is correctly located in thread local storage.
dtables::lgdt(&gdtr);
// Load segments again, possibly resetting FSBASE and GSBASE.
load_segments();
// Ensure that GSBASE always points to the PCR in kernel space.
x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64);
// While GSBASE points to the PCR in kernel space, userspace is free to set it to other values.
// Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is
// because entering usermode will entail executing the SWAPGS instruction.
x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0);
// Set the userspace FSBASE to zero.
x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0);
// Set the stack pointer to use when coming back from userspace.
set_tss_stack(pcr, stack_offset);
// Load the task register
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
pcr.percpu = PercpuBlock::init(cpu_id);
crate::percpu::init_tlb_shootdown(cpu_id, &mut pcr.percpu);
}
#[derive(Copy, Clone, Debug)]
#[repr(C, packed)]
src/arch/x86_64/interrupt/handler.rs
+13 -13
View File
@@ -360,12 +360,12 @@ macro_rules! nop {
macro_rules! interrupt_stack {
// XXX: Apparently we cannot use $expr and check for bool exhaustiveness, so we will have to
// use idents directly instead.
($name:ident, $save1:ident!, $save2:ident!, $rstor2:ident!, $rstor1:ident!, is_paranoid: $is_paranoid:expr, |$stack:ident| $code:block) => {
($name:ident, $save1:ident!, $save2:ident!, $rstor2:ident!, $rstor1:ident!, is_paranoid: $is_paranoid:expr_2021, |$stack:ident| $code:block) => {
#[naked]
pub unsafe extern "C" fn $name() {
unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) {
pub unsafe extern "C" fn $name() { unsafe {
unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) { unsafe {
$code
}
}}
core::arch::naked_asm!(concat!(
// Clear direction flag, required by ABI when running any Rust code in the kernel.
"cld;",
@@ -405,7 +405,7 @@ macro_rules! interrupt_stack {
PCR_GDT_OFFSET = const(core::mem::offset_of!(crate::gdt::ProcessorControlRegion, gdt)),
);
}
}}
};
($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, swapgs_iff_ring3_fast!, nop!, nop!, swapgs_iff_ring3_fast!, is_paranoid: false, |$stack| $code); };
($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, conditional_swapgs_paranoid!, conditional_swapgs_back_paranoid!, nop!, is_paranoid: true, |$stack| $code); }
@@ -415,10 +415,10 @@ macro_rules! interrupt_stack {
macro_rules! interrupt {
($name:ident, || $code:block) => {
#[naked]
pub unsafe extern "C" fn $name() {
unsafe extern "C" fn inner() {
pub unsafe extern "C" fn $name() { unsafe {
unsafe extern "C" fn inner() { unsafe {
$code
}
}}
core::arch::naked_asm!(concat!(
// Clear direction flag, required by ABI when running any Rust code in the kernel.
@@ -447,7 +447,7 @@ macro_rules! interrupt {
inner = sym inner,
);
}
}}
};
}
@@ -455,10 +455,10 @@ macro_rules! interrupt {
macro_rules! interrupt_error {
($name:ident, |$stack:ident, $error_code:ident| $code:block) => {
#[naked]
pub unsafe extern "C" fn $name() {
unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::handler::InterruptStack, $error_code: usize) {
pub unsafe extern "C" fn $name() { unsafe {
unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::handler::InterruptStack, $error_code: usize) { unsafe {
$code
}
}}
core::arch::naked_asm!(concat!(
// Clear direction flag, required by ABI when running any Rust code in the kernel.
@@ -500,7 +500,7 @@ macro_rules! interrupt_error {
inner = sym inner,
rax_offset = const(::core::mem::size_of::<$crate::interrupt::handler::PreservedRegisters>() + ::core::mem::size_of::<$crate::interrupt::handler::ScratchRegisters>() - 8),
);
}
}}
};
}
src/arch/x86_64/interrupt/irq.rs
+53 -37
View File
@@ -70,85 +70,101 @@ fn irq_method() -> IrqMethod {
/// Notify the IRQ scheme that an IRQ has been registered. This should mask the IRQ until the
/// scheme user unmasks it ("acknowledges" it).
unsafe fn trigger(irq: u8) {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_mask(irq)
unsafe {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_mask(irq)
}
}
IrqMethod::Apic => ioapic_mask(irq),
}
IrqMethod::Apic => ioapic_mask(irq),
irq_trigger(irq);
}
irq_trigger(irq);
}
/// Unmask the IRQ. This is called from the IRQ scheme, which does this when a user process has
/// processed the IRQ.
pub unsafe fn acknowledge(irq: usize) {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_unmask(irq)
unsafe {
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_unmask(irq)
}
}
IrqMethod::Apic => ioapic_unmask(irq),
}
IrqMethod::Apic => ioapic_unmask(irq),
}
}
/// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one.
pub unsafe fn eoi(irq: u8) {
PercpuBlock::current().stats.add_irq(irq);
unsafe {
PercpuBlock::current().stats.add_irq(irq);
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_eoi(irq)
match irq_method() {
IrqMethod::Pic => {
if irq < 16 {
pic_eoi(irq)
}
}
IrqMethod::Apic => lapic_eoi(),
}
IrqMethod::Apic => lapic_eoi(),
}
}
unsafe fn pic_mask(irq: u8) {
debug_assert!(irq < 16);
unsafe {
debug_assert!(irq < 16);
if irq >= 8 {
pic::slave().mask_set(irq - 8);
} else {
pic::master().mask_set(irq);
if irq >= 8 {
pic::slave().mask_set(irq - 8);
} else {
pic::master().mask_set(irq);
}
}
}
unsafe fn ioapic_mask(irq: u8) {
ioapic::mask(irq);
unsafe {
ioapic::mask(irq);
}
}
unsafe fn pic_eoi(irq: u8) {
debug_assert!(irq < 16);
unsafe {
debug_assert!(irq < 16);
if irq >= 8 {
pic::master().ack();
pic::slave().ack();
} else {
pic::master().ack();
if irq >= 8 {
pic::master().ack();
pic::slave().ack();
} else {
pic::master().ack();
}
}
}
unsafe fn lapic_eoi() {
local_apic::the_local_apic().eoi()
unsafe { local_apic::the_local_apic().eoi() }
}
unsafe fn pic_unmask(irq: usize) {
debug_assert!(irq < 16);
unsafe {
debug_assert!(irq < 16);
if irq >= 8 {
pic::slave().mask_clear(irq as u8 - 8);
} else {
pic::master().mask_clear(irq as u8);
if irq >= 8 {
pic::slave().mask_clear(irq as u8 - 8);
} else {
pic::master().mask_clear(irq as u8);
}
}
}
unsafe fn ioapic_unmask(irq: usize) {
ioapic::unmask(irq as u8);
unsafe {
ioapic::unmask(irq as u8);
}
}
interrupt_stack!(pit_stack, |_stack| {
@@ -311,7 +327,7 @@ __generic_interrupts_start:
__generic_interrupts_end:
", sym generic_irq);
extern "C" {
unsafe extern "C" {
pub fn __generic_interrupts_start();
pub fn __generic_interrupts_end();
}
src/arch/x86_64/interrupt/syscall.rs
+144 -137
View File
@@ -11,173 +11,179 @@ use x86::{
};
pub unsafe fn init() {
// IA32_STAR[31:0] are reserved.
unsafe {
// IA32_STAR[31:0] are reserved.
// The base selector of the two consecutive segments for kernel code and the immediately
// suceeding stack (data).
let syscall_cs_ss_base = (gdt::GDT_KERNEL_CODE as u16) << 3;
// The base selector of the three consecutive segments (of which two are used) for user code
// and user data. It points to a 32-bit code segment, which must be followed by a data segment
// (stack), and a 64-bit code segment.
let sysret_cs_ss_base = ((gdt::GDT_USER_CODE32_UNUSED as u16) << 3) | 3;
let star_high = u32::from(syscall_cs_ss_base) | (u32::from(sysret_cs_ss_base) << 16);
// The base selector of the two consecutive segments for kernel code and the immediately
// suceeding stack (data).
let syscall_cs_ss_base = (gdt::GDT_KERNEL_CODE as u16) << 3;
// The base selector of the three consecutive segments (of which two are used) for user code
// and user data. It points to a 32-bit code segment, which must be followed by a data segment
// (stack), and a 64-bit code segment.
let sysret_cs_ss_base = ((gdt::GDT_USER_CODE32_UNUSED as u16) << 3) | 3;
let star_high = u32::from(syscall_cs_ss_base) | (u32::from(sysret_cs_ss_base) << 16);
msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32);
msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64);
msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32);
msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64);
// DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK,
// userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and
// cause all sorts of memory corruption.
//
// IF needs to be cleared, as the kernel currently assumes interrupts are disabled except in
// usermode and in kmain.
//
// TF needs to be cleared, as enabling userspace-rflags-controlled singlestep in the kernel
// would be a bad idea.
//
// AC it should always be cleared when entering the kernel (and never be set except in usercopy
// functions), if for some reason AC was set before entering userspace (AC can only be modified
// by kernel code).
//
// The other flags could indeed be preserved and excluded from FMASK, but since they are not
// used to pass data to the kernel, they might as well be masked with *marginal* security
// benefits.
//
// Flags not included here are IOPL (not relevant to the kernel at all), "CPUID flag" (not used
// at all in 64-bit mode), RF (not used yet, but DR breakpoints would remain enabled both in
// user and kernel mode), VM8086 (not used at all), and VIF/VIP (system-level status flags?).
// DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK,
// userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and
// cause all sorts of memory corruption.
//
// IF needs to be cleared, as the kernel currently assumes interrupts are disabled except in
// usermode and in kmain.
//
// TF needs to be cleared, as enabling userspace-rflags-controlled singlestep in the kernel
// would be a bad idea.
//
// AC it should always be cleared when entering the kernel (and never be set except in usercopy
// functions), if for some reason AC was set before entering userspace (AC can only be modified
// by kernel code).
//
// The other flags could indeed be preserved and excluded from FMASK, but since they are not
// used to pass data to the kernel, they might as well be masked with *marginal* security
// benefits.
//
// Flags not included here are IOPL (not relevant to the kernel at all), "CPUID flag" (not used
// at all in 64-bit mode), RF (not used yet, but DR breakpoints would remain enabled both in
// user and kernel mode), VM8086 (not used at all), and VIF/VIP (system-level status flags?).
let mask_critical = RFlags::FLAGS_DF | RFlags::FLAGS_IF | RFlags::FLAGS_TF | RFlags::FLAGS_AC;
let mask_other = RFlags::FLAGS_CF
| RFlags::FLAGS_PF
| RFlags::FLAGS_AF
| RFlags::FLAGS_ZF
| RFlags::FLAGS_SF
| RFlags::FLAGS_OF;
msr::wrmsr(msr::IA32_FMASK, (mask_critical | mask_other).bits());
let mask_critical =
RFlags::FLAGS_DF | RFlags::FLAGS_IF | RFlags::FLAGS_TF | RFlags::FLAGS_AC;
let mask_other = RFlags::FLAGS_CF
| RFlags::FLAGS_PF
| RFlags::FLAGS_AF
| RFlags::FLAGS_ZF
| RFlags::FLAGS_SF
| RFlags::FLAGS_OF;
msr::wrmsr(msr::IA32_FMASK, (mask_critical | mask_other).bits());
let efer = msr::rdmsr(msr::IA32_EFER);
msr::wrmsr(msr::IA32_EFER, efer | 1);
let efer = msr::rdmsr(msr::IA32_EFER);
msr::wrmsr(msr::IA32_EFER, efer | 1);
}
}
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) {
let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None)
.and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE)));
unsafe {
let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None)
.and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE)));
if allowed.unwrap_or(true) {
let scratch = &(*stack).scratch;
if allowed.unwrap_or(true) {
let scratch = &(*stack).scratch;
let ret = syscall::syscall(
scratch.rax,
scratch.rdi,
scratch.rsi,
scratch.rdx,
scratch.r10,
scratch.r8,
);
(*stack).scratch.rax = ret;
let ret = syscall::syscall(
scratch.rax,
scratch.rdi,
scratch.rsi,
scratch.rdx,
scratch.r10,
scratch.r8,
);
(*stack).scratch.rax = ret;
}
ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None);
}
ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None);
}
#[naked]
#[allow(named_asm_labels)]
pub unsafe extern "C" fn syscall_instruction() {
core::arch::naked_asm!(concat!(
// Yes, this is magic. No, you don't need to understand
"swapgs;", // Swap KGSBASE with GSBASE, allowing fast TSS access.
"mov gs:[{sp}], rsp;", // Save userspace stack pointer
"mov rsp, gs:[{ksp}];", // Load kernel stack pointer
"push QWORD PTR {ss_sel};", // Push fake userspace SS (resembling iret frame)
"push QWORD PTR gs:[{sp}];", // Push userspace rsp
"push r11;", // Push rflags
"push QWORD PTR {cs_sel};", // Push fake CS (resembling iret stack frame)
"push rcx;", // Push userspace return pointer
unsafe {
core::arch::naked_asm!(concat!(
// Yes, this is magic. No, you don't need to understand
"swapgs;", // Swap KGSBASE with GSBASE, allowing fast TSS access.
"mov gs:[{sp}], rsp;", // Save userspace stack pointer
"mov rsp, gs:[{ksp}];", // Load kernel stack pointer
"push QWORD PTR {ss_sel};", // Push fake userspace SS (resembling iret frame)
"push QWORD PTR gs:[{sp}];", // Push userspace rsp
"push r11;", // Push rflags
"push QWORD PTR {cs_sel};", // Push fake CS (resembling iret stack frame)
"push rcx;", // Push userspace return pointer
// Push context registers
"push rax;",
push_scratch!(),
push_preserved!(),
// Push context registers
"push rax;",
push_scratch!(),
push_preserved!(),
// TODO: Map PTI
// $crate::arch::x86_64::pti::map();
// TODO: Map PTI
// $crate::arch::x86_64::pti::map();
// Call inner funtion
"mov rdi, rsp;",
"call __inner_syscall_instruction;",
// Call inner funtion
"mov rdi, rsp;",
"call __inner_syscall_instruction;",
// TODO: Unmap PTI
// $crate::arch::x86_64::pti::unmap();
// TODO: Unmap PTI
// $crate::arch::x86_64::pti::unmap();
"
"
.globl enter_usermode
enter_usermode:
",
// Pop context registers
pop_preserved!(),
pop_scratch!(),
// Pop context registers
pop_preserved!(),
pop_scratch!(),
// Restore user GSBASE by swapping GSBASE and KGSBASE.
"swapgs;",
// Restore user GSBASE by swapping GSBASE and KGSBASE.
"swapgs;",
// TODO: Should we unconditionally jump or avoid jumping, to hint to the branch predictor that
// singlestep is NOT set?
//
// It appears Intel CPUs assume (previously unknown) forward conditional branches to not be
// taken, and AMD appears to assume all previously unknown conditional branches will not be
// taken.
// TODO: Should we unconditionally jump or avoid jumping, to hint to the branch predictor that
// singlestep is NOT set?
//
// It appears Intel CPUs assume (previously unknown) forward conditional branches to not be
// taken, and AMD appears to assume all previously unknown conditional branches will not be
// taken.
// Check if the Trap Flag (singlestep flag) is set. If so, sysretq will return to before the
// instruction, whereas debuggers expect the iretq behavior of returning to after the
// instruction.
// Check if the Trap Flag (singlestep flag) is set. If so, sysretq will return to before the
// instruction, whereas debuggers expect the iretq behavior of returning to after the
// instruction.
// TODO: Which one is faster?
// bt DWORD PTR [rsp + 16], 8
// or,
// bt BYTE PTR [rsp + 17], 0
// or,
// test BYTE PTR [rsp + 17], 1
// or,
// test WORD PTR [rsp + 16], 0x100
// or,
// test DWORD PTR [rsp + 16], 0x100
// ?
// TODO: Which one is faster?
// bt DWORD PTR [rsp + 16], 8
// or,
// bt BYTE PTR [rsp + 17], 0
// or,
// test BYTE PTR [rsp + 17], 1
// or,
// test WORD PTR [rsp + 16], 0x100
// or,
// test DWORD PTR [rsp + 16], 0x100
// ?
"test BYTE PTR [rsp + 17], 1;",
// If set, return using IRETQ instead.
"jnz 2f;",
"test BYTE PTR [rsp + 17], 1;",
// If set, return using IRETQ instead.
"jnz 2f;",
// Otherwise, continue with the fast sysretq.
// Otherwise, continue with the fast sysretq.
// Pop userspace return pointer
"pop rcx;",
// Pop userspace return pointer
"pop rcx;",
// We must ensure RCX is canonical; if it is not when running sysretq, the consequences can be
// fatal from a security perspective.
//
// See https://xenproject.org/2012/06/13/the-intel-sysret-privilege-escalation/.
//
// This is not just theoretical; ptrace allows userspace to change RCX (via RIP) of target
// processes.
//
// While we could also conditionally IRETQ here, an easier method is to simply sign-extend RCX:
// We must ensure RCX is canonical; if it is not when running sysretq, the consequences can be
// fatal from a security perspective.
//
// See https://xenproject.org/2012/06/13/the-intel-sysret-privilege-escalation/.
//
// This is not just theoretical; ptrace allows userspace to change RCX (via RIP) of target
// processes.
//
// While we could also conditionally IRETQ here, an easier method is to simply sign-extend RCX:
// Shift away the upper 16 bits (0xBAAD_8000_DEAD_BEEF => 0x8000_DEAD_BEEF_XXXX).
"shl rcx, 16;",
// Shift arithmetically right by 16 bits, effectively extending the 47th sign bit to bits
// 63:48 (0x8000_DEAD_BEEF_XXXX => 0xFFFF_8000_DEAD_BEEF).
"sar rcx, 16;",
// Shift away the upper 16 bits (0xBAAD_8000_DEAD_BEEF => 0x8000_DEAD_BEEF_XXXX).
"shl rcx, 16;",
// Shift arithmetically right by 16 bits, effectively extending the 47th sign bit to bits
// 63:48 (0x8000_DEAD_BEEF_XXXX => 0xFFFF_8000_DEAD_BEEF).
"sar rcx, 16;",
"add rsp, 8;", // Pop fake userspace CS
"pop r11;", // Pop rflags
"pop rsp;", // Restore userspace stack pointer
"sysretq;", // Return into userspace; RCX=>RIP,R11=>RFLAGS
"add rsp, 8;", // Pop fake userspace CS
"pop r11;", // Pop rflags
"pop rsp;", // Restore userspace stack pointer
"sysretq;", // Return into userspace; RCX=>RIP,R11=>RFLAGS
// IRETQ fallback:
"
// IRETQ fallback:
"
.p2align 4
2:
xor rcx, rcx
@@ -185,13 +191,14 @@ pub unsafe extern "C" fn syscall_instruction() {
iretq
"),
sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)),
ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)),
ss_sel = const(SegmentSelector::new(gdt::GDT_USER_DATA as u16, x86::Ring::Ring3).bits()),
cs_sel = const(SegmentSelector::new(gdt::GDT_USER_CODE as u16, x86::Ring::Ring3).bits()),
);
sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)),
ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)),
ss_sel = const(SegmentSelector::new(gdt::GDT_USER_DATA as u16, x86::Ring::Ring3).bits()),
cs_sel = const(SegmentSelector::new(gdt::GDT_USER_CODE as u16, x86::Ring::Ring3).bits()),
);
}
}
extern "C" {
unsafe extern "C" {
// TODO: macro?
pub fn enter_usermode();
}
src/arch/x86_64/macros.rs
+6 -6
View File
@@ -11,18 +11,18 @@ macro_rules! print {
#[macro_export]
macro_rules! println {
() => (print!("\n"));
($fmt:expr) => (print!(concat!($fmt, "\n")));
($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
($fmt:expr_2021) => (print!(concat!($fmt, "\n")));
($fmt:expr_2021, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
}
macro_rules! expand_bool(
($value:expr) => {
($value:expr_2021) => {
concat!($value)
}
);
macro_rules! alternative(
(feature: $feature:literal, then: [$($then:expr),*], default: [$($default:expr),*]) => {
(feature: $feature:literal, then: [$($then:expr_2021),*], default: [$($default:expr_2021),*]) => {
alternative2!(feature1: $feature, then1: [$($then),*], feature2: "", then2: [""], default: [$($default),*])
}
);
@@ -39,7 +39,7 @@ macro_rules! saturating_sub(
//
// An empty string as feature is equivalent with "never".
macro_rules! alternative2(
(feature1: $feature1:literal, then1: [$($then1:expr),*], feature2: $feature2:literal, then2: [$($then2:expr),*], default: [$($default:expr),*]) => {
(feature1: $feature1:literal, then1: [$($then1:expr_2021),*], feature2: $feature2:literal, then2: [$($then2:expr_2021),*], default: [$($default:expr_2021),*]) => {
concat!("
.set true, 1
.set false, 0
@@ -70,7 +70,7 @@ macro_rules! alternative2(
};
);
macro_rules! alternative_auto(
($first_digit:literal, $feature:literal, [$($then:expr),*]) => { concat!(
($first_digit:literal, $feature:literal, [$($then:expr_2021),*]) => { concat!(
".if ", expand_bool!(cfg!(cpu_feature_auto = $feature)), "
.pushsection .altcode.", $feature, ",\"a\"
", $first_digit, "0:
src/arch/x86_64/misc.rs
+18 -16
View File
@@ -6,22 +6,24 @@ use crate::{
};
pub unsafe fn init(cpu_id: LogicalCpuId) {
if has_ext_feat(|feat| feat.has_umip()) {
// UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT,
// SMSW and STR. KASLR is currently not implemented, but this protects against leaking
// addresses.
x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_UMIP);
}
if has_ext_feat(|feat| feat.has_smep()) {
// SMEP (Supervisor-Mode Execution Prevention) forbids the kernel from executing
// instruction on any page marked "userspace-accessible". This improves security for
// obvious reasons.
x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP);
}
unsafe {
if has_ext_feat(|feat| feat.has_umip()) {
// UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT,
// SMSW and STR. KASLR is currently not implemented, but this protects against leaking
// addresses.
x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_UMIP);
}
if has_ext_feat(|feat| feat.has_smep()) {
// SMEP (Supervisor-Mode Execution Prevention) forbids the kernel from executing
// instruction on any page marked "userspace-accessible". This improves security for
// obvious reasons.
x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP);
}
if let Some(feats) = cpuid().get_extended_processor_and_feature_identifiers()
&& feats.has_rdtscp()
{
x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into());
if let Some(feats) = cpuid().get_extended_processor_and_feature_identifiers()
&& feats.has_rdtscp()
{
x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into());
}
}
}
src/arch/x86_64/mod.rs
+9 -7
View File
@@ -30,13 +30,14 @@ pub mod flags {
// TODO: Maybe support rewriting relocations (using LD's --emit-relocs) when working with entire
// functions?
#[naked]
#[link_section = ".usercopy-fns"]
#[unsafe(link_section = ".usercopy-fns")]
pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
// TODO: spectre_v1
unsafe {
// TODO: spectre_v1
core::arch::naked_asm!(alternative!(
feature: "smap",
then: ["
core::arch::naked_asm!(alternative!(
feature: "smap",
then: ["
xor eax, eax
mov rcx, rdx
stac
@@ -44,13 +45,14 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
clac
ret
"],
default: ["
default: ["
xor eax, eax
mov rcx, rdx
rep movsb
ret
"]
));
));
}
}
pub use arch_copy_to_user as arch_copy_from_user;
src/arch/x86_shared/device/hpet.rs
+85 -79
View File
@@ -23,97 +23,103 @@ pub(crate) const T0_COMPARATOR_OFFSET: usize = 0x108;
const PER_INT_CAP: u64 = 0x10;
pub unsafe fn init(hpet: &mut Hpet) -> bool {
println!("HPET @ {:#x}", { hpet.base_address.address });
debug_caps(hpet);
unsafe {
println!("HPET @ {:#x}", { hpet.base_address.address });
debug_caps(hpet);
println!("HPET Before Init");
debug_config(hpet);
println!("HPET Before Init");
debug_config(hpet);
// Disable HPET
{
let mut config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET);
config_word &= !(LEG_RT_CNF | ENABLE_CNF);
hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word);
// Disable HPET
{
let mut config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET);
config_word &= !(LEG_RT_CNF | ENABLE_CNF);
hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word);
}
let capability = hpet.read_u64(CAPABILITY_OFFSET);
if capability & LEG_RT_CAP == 0 {
log::warn!("HPET missing capability LEG_RT_CAP");
return false;
}
let period_fs = capability >> 32;
let divisor = (pit::RATE as u64 * 1_000_000) / period_fs;
let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
if t0_capabilities & PER_INT_CAP == 0 {
log::warn!("HPET T0 missing capability PER_INT_CAP");
return false;
}
let counter = hpet.read_u64(MAIN_COUNTER_OFFSET);
let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF;
hpet.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word);
// set accumulator value
hpet.write_u64(T0_COMPARATOR_OFFSET, counter + divisor);
// set interval
hpet.write_u64(T0_COMPARATOR_OFFSET, divisor);
// Enable interrupts from the HPET
{
let mut config_word: u64 = hpet.read_u64(GENERAL_CONFIG_OFFSET);
config_word |= LEG_RT_CNF | ENABLE_CNF;
hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word);
}
println!("HPET After Init");
debug_config(hpet);
true
}
let capability = hpet.read_u64(CAPABILITY_OFFSET);
if capability & LEG_RT_CAP == 0 {
log::warn!("HPET missing capability LEG_RT_CAP");
return false;
}
let period_fs = capability >> 32;
let divisor = (pit::RATE as u64 * 1_000_000) / period_fs;
let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
if t0_capabilities & PER_INT_CAP == 0 {
log::warn!("HPET T0 missing capability PER_INT_CAP");
return false;
}
let counter = hpet.read_u64(MAIN_COUNTER_OFFSET);
let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF;
hpet.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word);
// set accumulator value
hpet.write_u64(T0_COMPARATOR_OFFSET, counter + divisor);
// set interval
hpet.write_u64(T0_COMPARATOR_OFFSET, divisor);
// Enable interrupts from the HPET
{
let mut config_word: u64 = hpet.read_u64(GENERAL_CONFIG_OFFSET);
config_word |= LEG_RT_CNF | ENABLE_CNF;
hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word);
}
println!("HPET After Init");
debug_config(hpet);
true
}
unsafe fn debug_caps(hpet: &mut Hpet) {
let capability = hpet.read_u64(CAPABILITY_OFFSET);
println!(" caps: {:#x}", capability);
println!(
" clock period: {:?}",
Duration::from_nanos((capability >> 32) / 1_000_000)
);
println!(
" ID: {:#x} revision: {}",
(capability >> 16) as u16,
capability as u8
);
println!(
" LEG_RT_CAP: {} COUNT_SIZE_CAP: {}",
capability & (1 << 15) == (1 << 15),
capability & (1 << 13) == (1 << 13)
);
// The NUM_TIM_CAP field contains the index of the last timer.
// Add 1 to get the amount of timers.
println!(" timers: {}", (capability >> 8) as u8 & 0x1F + 1);
unsafe {
let capability = hpet.read_u64(CAPABILITY_OFFSET);
println!(" caps: {:#x}", capability);
println!(
" clock period: {:?}",
Duration::from_nanos((capability >> 32) / 1_000_000)
);
println!(
" ID: {:#x} revision: {}",
(capability >> 16) as u16,
capability as u8
);
println!(
" LEG_RT_CAP: {} COUNT_SIZE_CAP: {}",
capability & (1 << 15) == (1 << 15),
capability & (1 << 13) == (1 << 13)
);
// The NUM_TIM_CAP field contains the index of the last timer.
// Add 1 to get the amount of timers.
println!(" timers: {}", (capability >> 8) as u8 & 0x1F + 1);
let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
println!(
" T0 interrupt routing: {:#x}",
(t0_capabilities >> 32) as u32
);
let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
println!(
" T0 interrupt routing: {:#x}",
(t0_capabilities >> 32) as u32
);
}
}
unsafe fn debug_config(hpet: &mut Hpet) {
let config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET);
println!(" config: {:#x}", config_word);
unsafe {
let config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET);
println!(" config: {:#x}", config_word);
let interrupt_status = hpet.read_u64(GENERAL_INTERRUPT_OFFSET);
println!(" interrupt status: {:#x}", interrupt_status);
let interrupt_status = hpet.read_u64(GENERAL_INTERRUPT_OFFSET);
println!(" interrupt status: {:#x}", interrupt_status);
let counter = hpet.read_u64(MAIN_COUNTER_OFFSET);
println!(" counter: {:#x}", counter);
let counter = hpet.read_u64(MAIN_COUNTER_OFFSET);
println!(" counter: {:#x}", counter);
let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
println!(" T0 flags: {:#x}", t0_capabilities as u32);
let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
println!(" T0 flags: {:#x}", t0_capabilities as u32);
let t0_comparator = hpet.read_u64(T0_COMPARATOR_OFFSET);
println!(" T0 comparator: {:#x}", t0_comparator);
let t0_comparator = hpet.read_u64(T0_COMPARATOR_OFFSET);
println!(" T0 comparator: {:#x}", t0_comparator);
}
}
src/arch/x86_shared/device/ioapic.rs
+161 -153
View File
@@ -238,188 +238,196 @@ pub fn src_overrides() -> &'static [Override] {
#[cfg(feature = "acpi")]
pub unsafe fn handle_ioapic(mapper: &mut KernelMapper, madt_ioapic: &'static MadtIoApic) {
// map the I/O APIC registers
unsafe {
// map the I/O APIC registers
let frame = Frame::containing(PhysicalAddress::new(madt_ioapic.address as usize));
#[cfg(target_arch = "x86")]
let page = Page::containing_address(rmm::VirtualAddress::new(crate::IOAPIC_OFFSET));
#[cfg(target_arch = "x86_64")]
let page = Page::containing_address(RmmA::phys_to_virt(frame.base()));
let frame = Frame::containing(PhysicalAddress::new(madt_ioapic.address as usize));
#[cfg(target_arch = "x86")]
let page = Page::containing_address(rmm::VirtualAddress::new(crate::IOAPIC_OFFSET));
#[cfg(target_arch = "x86_64")]
let page = Page::containing_address(RmmA::phys_to_virt(frame.base()));
assert!(mapper.translate(page.start_address()).is_none());
assert!(mapper.translate(page.start_address()).is_none());
mapper
.get_mut()
.expect("expected KernelMapper not to be locked re-entrant while mapping I/O APIC memory")
.map_phys(
page.start_address(),
frame.base(),
PageFlags::new()
.write(true)
.custom_flag(EntryFlags::NO_CACHE.bits(), true),
)
.expect("failed to map I/O APIC")
.flush();
mapper
.get_mut()
.expect(
"expected KernelMapper not to be locked re-entrant while mapping I/O APIC memory",
)
.map_phys(
page.start_address(),
frame.base(),
PageFlags::new()
.write(true)
.custom_flag(EntryFlags::NO_CACHE.bits(), true),
)
.expect("failed to map I/O APIC")
.flush();
let ioapic_registers = page.start_address().data() as *const u32;
let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base);
let ioapic_registers = page.start_address().data() as *const u32;
let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base);
assert_eq!(
ioapic.regs.lock().id(),
madt_ioapic.id,
"mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC"
);
assert_eq!(
ioapic.regs.lock().id(),
madt_ioapic.id,
"mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC"
);
(*IOAPICS.get()).get_or_insert_with(Vec::new).push(ioapic);
(*IOAPICS.get()).get_or_insert_with(Vec::new).push(ioapic);
}
}
#[cfg(feature = "acpi")]
pub unsafe fn handle_src_override(src_override: &'static MadtIntSrcOverride) {
let flags = src_override.flags;
unsafe {
let flags = src_override.flags;
let polarity_raw = (flags & 0x0003) as u8;
let trigger_mode_raw = ((flags & 0x000C) >> 2) as u8;
let polarity_raw = (flags & 0x0003) as u8;
let trigger_mode_raw = ((flags & 0x000C) >> 2) as u8;
let polarity = match polarity_raw {
0b00 => Polarity::ConformsToSpecs,
0b01 => Polarity::ActiveHigh,
0b10 => return, // reserved
0b11 => Polarity::ActiveLow,
let polarity = match polarity_raw {
0b00 => Polarity::ConformsToSpecs,
0b01 => Polarity::ActiveHigh,
0b10 => return, // reserved
0b11 => Polarity::ActiveLow,
_ => unreachable!(),
};
_ => unreachable!(),
};
let trigger_mode = match trigger_mode_raw {
0b00 => TriggerMode::ConformsToSpecs,
0b01 => TriggerMode::Edge,
0b10 => return, // reserved
0b11 => TriggerMode::Level,
_ => unreachable!(),
};
let trigger_mode = match trigger_mode_raw {
0b00 => TriggerMode::ConformsToSpecs,
0b01 => TriggerMode::Edge,
0b10 => return, // reserved
0b11 => TriggerMode::Level,
_ => unreachable!(),
};
let over = Override {
bus_irq: src_override.irq_source,
gsi: src_override.gsi_base,
polarity,
trigger_mode,
};
(*SRC_OVERRIDES.get())
.get_or_insert_with(Vec::new)
.push(over);
let over = Override {
bus_irq: src_override.irq_source,
gsi: src_override.gsi_base,
polarity,
trigger_mode,
};
(*SRC_OVERRIDES.get())
.get_or_insert_with(Vec::new)
.push(over);
}
}
#[allow(dead_code)]
pub unsafe fn init(active_table: &mut KernelMapper) {
let bsp_apic_id = ApicId::new(u32::from(
cpuid().get_feature_info().unwrap().initial_local_apic_id(),
)); // TODO: remove unwraps
unsafe {
let bsp_apic_id = ApicId::new(u32::from(
cpuid().get_feature_info().unwrap().initial_local_apic_id(),
)); // TODO: remove unwraps
// search the madt for all IOAPICs.
#[cfg(feature = "acpi")]
{
let madt: &'static Madt = match madt::madt() {
Some(m) => m,
// TODO: Parse MP tables too.
None => return,
};
if madt.flags & madt::FLAG_PCAT != 0 {
pic::disable();
}
// search the madt for all IOAPICs.
#[cfg(feature = "acpi")]
{
let madt: &'static Madt = match madt::madt() {
Some(m) => m,
// TODO: Parse MP tables too.
None => return,
};
if madt.flags & madt::FLAG_PCAT != 0 {
pic::disable();
}
// find all I/O APICs (usually one).
// find all I/O APICs (usually one).
for entry in madt.iter() {
match entry {
MadtEntry::IoApic(ioapic) => handle_ioapic(active_table, ioapic),
MadtEntry::IntSrcOverride(src_override) => handle_src_override(src_override),
_ => (),
for entry in madt.iter() {
match entry {
MadtEntry::IoApic(ioapic) => handle_ioapic(active_table, ioapic),
MadtEntry::IntSrcOverride(src_override) => handle_src_override(src_override),
_ => (),
}
}
}
}
println!(
"I/O APICs: {:?}, overrides: {:?}",
ioapics(),
src_overrides()
);
println!(
"I/O APICs: {:?}, overrides: {:?}",
ioapics(),
src_overrides()
);
// map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it
// wouldn't have been disabled due to this I/O APIC)
for legacy_irq in 0..=15 {
let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) {
Some(over) => (over.gsi, over.trigger_mode, over.polarity),
None => {
if src_overrides()
.iter()
.any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq)
&& !src_overrides()
// map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it
// wouldn't have been disabled due to this I/O APIC)
for legacy_irq in 0..=15 {
let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) {
Some(over) => (over.gsi, over.trigger_mode, over.polarity),
None => {
if src_overrides()
.iter()
.any(|over| over.bus_irq == legacy_irq)
{
// there's an IRQ conflict, making this legacy IRQ inaccessible.
.any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq)
&& !src_overrides()
.iter()
.any(|over| over.bus_irq == legacy_irq)
{
// there's an IRQ conflict, making this legacy IRQ inaccessible.
continue;
}
(
legacy_irq.into(),
TriggerMode::ConformsToSpecs,
Polarity::ConformsToSpecs,
)
}
};
let apic = match find_ioapic(gsi) {
Some(ioapic) => ioapic,
None => {
println!("Unable to find a suitable APIC for legacy IRQ {} (GSI {}). It will not be mapped.", legacy_irq, gsi);
continue;
}
(
legacy_irq.into(),
TriggerMode::ConformsToSpecs,
Polarity::ConformsToSpecs,
)
}
};
let apic = match find_ioapic(gsi) {
Some(ioapic) => ioapic,
None => {
println!("Unable to find a suitable APIC for legacy IRQ {} (GSI {}). It will not be mapped.", legacy_irq, gsi);
continue;
}
};
let redir_tbl_index = (gsi - apic.gsi_start) as u8;
};
let redir_tbl_index = (gsi - apic.gsi_start) as u8;
let map_info = MapInfo {
// only send to the BSP
dest: bsp_apic_id,
dest_mode: DestinationMode::Physical,
delivery_mode: DeliveryMode::Fixed,
mask: false,
polarity: match polarity {
Polarity::ActiveHigh => ApicPolarity::ActiveHigh,
Polarity::ActiveLow => ApicPolarity::ActiveLow,
Polarity::ConformsToSpecs => ApicPolarity::ActiveHigh,
},
trigger_mode: match trigger_mode {
TriggerMode::Edge => ApicTriggerMode::Edge,
TriggerMode::Level => ApicTriggerMode::Level,
TriggerMode::ConformsToSpecs => ApicTriggerMode::Edge,
},
vector: 32 + legacy_irq,
};
apic.map(redir_tbl_index, map_info);
}
println!(
"I/O APICs: {:?}, overrides: {:?}",
ioapics(),
src_overrides()
);
irq::set_irq_method(irq::IrqMethod::Apic);
// tell the firmware that we're using APIC rather than the default 8259 PIC.
// FIXME: With ACPI moved to userspace, we should instead allow userspace to check whether the
// IOAPIC has been initialized, and then subsequently let some ACPI driver call the AML from
// userspace.
/*#[cfg(feature = "acpi")]
{
let method = {
let namespace_guard = crate::acpi::ACPI_TABLE.namespace.read();
if let Some(value) = namespace_guard.as_ref().unwrap().get("\\_PIC") {
value.get_as_method().ok()
} else {
None
}
};
if let Some(m) = method {
m.execute("\\_PIC".into(), vec!(crate::acpi::aml::AmlValue::Integer(1)));
let map_info = MapInfo {
// only send to the BSP
dest: bsp_apic_id,
dest_mode: DestinationMode::Physical,
delivery_mode: DeliveryMode::Fixed,
mask: false,
polarity: match polarity {
Polarity::ActiveHigh => ApicPolarity::ActiveHigh,
Polarity::ActiveLow => ApicPolarity::ActiveLow,
Polarity::ConformsToSpecs => ApicPolarity::ActiveHigh,
},
trigger_mode: match trigger_mode {
TriggerMode::Edge => ApicTriggerMode::Edge,
TriggerMode::Level => ApicTriggerMode::Level,
TriggerMode::ConformsToSpecs => ApicTriggerMode::Edge,
},
vector: 32 + legacy_irq,
};
apic.map(redir_tbl_index, map_info);
}
}*/
println!(
"I/O APICs: {:?}, overrides: {:?}",
ioapics(),
src_overrides()
);
irq::set_irq_method(irq::IrqMethod::Apic);
// tell the firmware that we're using APIC rather than the default 8259 PIC.
// FIXME: With ACPI moved to userspace, we should instead allow userspace to check whether the
// IOAPIC has been initialized, and then subsequently let some ACPI driver call the AML from
// userspace.
/*#[cfg(feature = "acpi")]
{
let method = {
let namespace_guard = crate::acpi::ACPI_TABLE.namespace.read();
if let Some(value) = namespace_guard.as_ref().unwrap().get("\\_PIC") {
value.get_as_method().ok()
} else {
None
}
};
if let Some(m) = method {
m.execute("\\_PIC".into(), vec!(crate::acpi::aml::AmlValue::Integer(1)));
}
}*/
}
}
fn get_override(irq: u8) -> Option<&'static Override> {
src_overrides().iter().find(|over| over.bus_irq == irq)
src/arch/x86_shared/device/local_apic.rs
+129 -95
View File
@@ -30,15 +30,19 @@ static LOCAL_APIC: SyncUnsafeCell<LocalApic> = SyncUnsafeCell::new(LocalApic {
x2: false,
});
pub unsafe fn the_local_apic() -> &'static mut LocalApic {
&mut *LOCAL_APIC.get()
unsafe { &mut *LOCAL_APIC.get() }
}
pub unsafe fn init(active_table: &mut KernelMapper) {
the_local_apic().init(active_table);
unsafe {
the_local_apic().init(active_table);
}
}
pub unsafe fn init_ap() {
the_local_apic().init_ap();
unsafe {
the_local_apic().init_ap();
}
}
/// Local APIC
@@ -49,63 +53,69 @@ pub struct LocalApic {
impl LocalApic {
unsafe fn init(&mut self, mapper: &mut KernelMapper) {
let mapper = mapper
.get_mut()
.expect("expected KernelMapper not to be locked re-entrant while initializing LAPIC");
unsafe {
let mapper = mapper.get_mut().expect(
"expected KernelMapper not to be locked re-entrant while initializing LAPIC",
);
let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000);
#[cfg(target_arch = "x86")]
let virtaddr = rmm::VirtualAddress::new(crate::LAPIC_OFFSET);
#[cfg(target_arch = "x86_64")]
let virtaddr = {
use rmm::Arch;
crate::memory::RmmA::phys_to_virt(physaddr)
};
let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000);
#[cfg(target_arch = "x86")]
let virtaddr = rmm::VirtualAddress::new(crate::LAPIC_OFFSET);
#[cfg(target_arch = "x86_64")]
let virtaddr = {
use rmm::Arch;
crate::memory::RmmA::phys_to_virt(physaddr)
};
self.address = virtaddr.data();
self.x2 = cpuid()
.get_feature_info()
.map_or(false, |feature_info| feature_info.has_x2apic());
self.address = virtaddr.data();
self.x2 = cpuid()
.get_feature_info()
.map_or(false, |feature_info| feature_info.has_x2apic());
if !self.x2 {
log::info!("Detected xAPIC at {:#x}", physaddr.data());
if let Some((_entry, _, flush)) = mapper.unmap_phys(virtaddr, true) {
// Unmap xAPIC page if already mapped
flush.flush();
if !self.x2 {
log::info!("Detected xAPIC at {:#x}", physaddr.data());
if let Some((_entry, _, flush)) = mapper.unmap_phys(virtaddr, true) {
// Unmap xAPIC page if already mapped
flush.flush();
}
mapper
.map_phys(virtaddr, physaddr, PageFlags::new().write(true))
.expect("failed to map local APIC memory")
.flush();
} else {
log::info!("Detected x2APIC");
}
mapper
.map_phys(virtaddr, physaddr, PageFlags::new().write(true))
.expect("failed to map local APIC memory")
.flush();
} else {
log::info!("Detected x2APIC");
}
self.init_ap();
self.init_ap();
}
}
unsafe fn init_ap(&mut self) {
if self.x2 {
wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | 1 << 10);
wrmsr(IA32_X2APIC_SIVR, 0x100);
} else {
self.write(0xF0, 0x100);
}
self.setup_error_int();
//self.setup_timer();
unsafe {
if self.x2 {
wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | 1 << 10);
wrmsr(IA32_X2APIC_SIVR, 0x100);
} else {
self.write(0xF0, 0x100);
}
self.setup_error_int();
//self.setup_timer();
PercpuBlock::current()
.misc_arch_info
.apic_id_opt
.set(Some(self.id()));
PercpuBlock::current()
.misc_arch_info
.apic_id_opt
.set(Some(self.id()));
}
}
unsafe fn read(&self, reg: u32) -> u32 {
read_volatile((self.address + reg as usize) as *const u32)
unsafe { read_volatile((self.address + reg as usize) as *const u32) }
}
unsafe fn write(&mut self, reg: u32, value: u32) {
write_volatile((self.address + reg as usize) as *mut u32, value);
unsafe {
write_volatile((self.address + reg as usize) as *mut u32, value);
}
}
pub fn id(&self) -> ApicId {
@@ -162,90 +172,114 @@ impl LocalApic {
}
pub unsafe fn eoi(&mut self) {
if self.x2 {
wrmsr(IA32_X2APIC_EOI, 0);
} else {
self.write(0xB0, 0);
unsafe {
if self.x2 {
wrmsr(IA32_X2APIC_EOI, 0);
} else {
self.write(0xB0, 0);
}
}
}
/// Reads the Error Status Register.
pub unsafe fn esr(&mut self) -> u32 {
if self.x2 {
// update the ESR to the current state of the local apic.
wrmsr(IA32_X2APIC_ESR, 0);
// read the updated value
rdmsr(IA32_X2APIC_ESR) as u32
} else {
self.write(0x280, 0);
self.read(0x280)
unsafe {
if self.x2 {
// update the ESR to the current state of the local apic.
wrmsr(IA32_X2APIC_ESR, 0);
// read the updated value
rdmsr(IA32_X2APIC_ESR) as u32
} else {
self.write(0x280, 0);
self.read(0x280)
}
}
}
pub unsafe fn lvt_timer(&mut self) -> u32 {
if self.x2 {
rdmsr(IA32_X2APIC_LVT_TIMER) as u32
} else {
self.read(0x320)
unsafe {
if self.x2 {
rdmsr(IA32_X2APIC_LVT_TIMER) as u32
} else {
self.read(0x320)
}
}
}
pub unsafe fn set_lvt_timer(&mut self, value: u32) {
if self.x2 {
wrmsr(IA32_X2APIC_LVT_TIMER, u64::from(value));
} else {
self.write(0x320, value);
unsafe {
if self.x2 {
wrmsr(IA32_X2APIC_LVT_TIMER, u64::from(value));
} else {
self.write(0x320, value);
}
}
}
pub unsafe fn init_count(&mut self) -> u32 {
if self.x2 {
rdmsr(IA32_X2APIC_INIT_COUNT) as u32
} else {
self.read(0x380)
unsafe {
if self.x2 {
rdmsr(IA32_X2APIC_INIT_COUNT) as u32
} else {
self.read(0x380)
}
}
}
pub unsafe fn set_init_count(&mut self, initial_count: u32) {
if self.x2 {
wrmsr(IA32_X2APIC_INIT_COUNT, u64::from(initial_count));
} else {
self.write(0x380, initial_count);
unsafe {
if self.x2 {
wrmsr(IA32_X2APIC_INIT_COUNT, u64::from(initial_count));
} else {
self.write(0x380, initial_count);
}
}
}
pub unsafe fn cur_count(&mut self) -> u32 {
if self.x2 {
rdmsr(IA32_X2APIC_CUR_COUNT) as u32
} else {
self.read(0x390)
unsafe {
if self.x2 {
rdmsr(IA32_X2APIC_CUR_COUNT) as u32
} else {
self.read(0x390)
}
}
}
pub unsafe fn div_conf(&mut self) -> u32 {
if self.x2 {
rdmsr(IA32_X2APIC_DIV_CONF) as u32
} else {
self.read(0x3E0)
unsafe {
if self.x2 {
rdmsr(IA32_X2APIC_DIV_CONF) as u32
} else {
self.read(0x3E0)
}
}
}
pub unsafe fn set_div_conf(&mut self, div_conf: u32) {
if self.x2 {
wrmsr(IA32_X2APIC_DIV_CONF, u64::from(div_conf));
} else {
self.write(0x3E0, div_conf);
unsafe {
if self.x2 {
wrmsr(IA32_X2APIC_DIV_CONF, u64::from(div_conf));
} else {
self.write(0x3E0, div_conf);
}
}
}
pub unsafe fn lvt_error(&mut self) -> u32 {
if self.x2 {
rdmsr(IA32_X2APIC_LVT_ERROR) as u32
} else {
self.read(0x370)
unsafe {
if self.x2 {
rdmsr(IA32_X2APIC_LVT_ERROR) as u32
} else {
self.read(0x370)
}
}
}
pub unsafe fn set_lvt_error(&mut self, lvt_error: u32) {
if self.x2 {
wrmsr(IA32_X2APIC_LVT_ERROR, u64::from(lvt_error));
} else {
self.write(0x370, lvt_error);
unsafe {
if self.x2 {
wrmsr(IA32_X2APIC_LVT_ERROR, u64::from(lvt_error));
} else {
self.write(0x370, lvt_error);
}
}
}
unsafe fn setup_error_int(&mut self) {
let vector = 49u32;
self.set_lvt_error(vector);
unsafe {
let vector = 49u32;
self.set_lvt_error(vector);
}
}
}
src/arch/x86_shared/device/mod.rs
+40 -31
View File
@@ -16,11 +16,13 @@ pub mod system76_ec;
pub mod tsc;
pub unsafe fn init() {
pic::init();
local_apic::init(&mut KernelMapper::lock());
unsafe {
pic::init();
local_apic::init(&mut KernelMapper::lock());
// Run here for the side-effect of printing if KVM was used to avoid interleaved logs.
tsc::get_kvm_support();
// Run here for the side-effect of printing if KVM was used to avoid interleaved logs.
tsc::get_kvm_support();
}
}
pub unsafe fn init_after_acpi() {
// this will disable the IOAPIC if needed.
@@ -29,16 +31,19 @@ pub unsafe fn init_after_acpi() {
#[cfg(feature = "acpi")]
unsafe fn init_hpet() -> bool {
use crate::acpi::ACPI_TABLE;
if let Some(ref mut hpet) = *ACPI_TABLE.hpet.write() {
if cfg!(target_arch = "x86") {
//TODO: fix HPET on i686
log::warn!("HPET found but implemented on i686");
return false;
unsafe {
use crate::acpi::ACPI_TABLE;
match *ACPI_TABLE.hpet.write() {
Some(ref mut hpet) => {
if cfg!(target_arch = "x86") {
//TODO: fix HPET on i686
log::warn!("HPET found but implemented on i686");
return false;
}
hpet::init(hpet)
}
_ => false,
}
hpet::init(hpet)
} else {
false
}
}
@@ -48,30 +53,34 @@ unsafe fn init_hpet() -> bool {
}
pub unsafe fn init_noncore() {
log::info!("Initializing system timer");
unsafe {
log::info!("Initializing system timer");
#[cfg(feature = "x86_kvm_pv")]
if tsc::init() {
log::info!("TSC used as system clock source");
#[cfg(feature = "x86_kvm_pv")]
if tsc::init() {
log::info!("TSC used as system clock source");
}
if init_hpet() {
log::info!("HPET used as system timer");
} else {
pit::init();
log::info!("PIT used as system timer");
}
log::info!("Initializing serial");
serial::init();
log::info!("Finished initializing devices");
}
if init_hpet() {
log::info!("HPET used as system timer");
} else {
pit::init();
log::info!("PIT used as system timer");
}
log::info!("Initializing serial");
serial::init();
log::info!("Finished initializing devices");
}
pub unsafe fn init_ap() {
local_apic::init_ap();
unsafe {
local_apic::init_ap();
#[cfg(feature = "x86_kvm_pv")]
tsc::init();
#[cfg(feature = "x86_kvm_pv")]
tsc::init();
}
}
#[derive(Default)]
src/arch/x86_shared/device/pic.rs
+30 -26
View File
@@ -10,48 +10,52 @@ static SLAVE: SyncUnsafeCell<Pic> = SyncUnsafeCell::new(Pic::new(0xA0));
// SAFETY: must be main thread
pub unsafe fn master<'a>() -> &'a mut Pic {
&mut *MASTER.get()
unsafe { &mut *MASTER.get() }
}
// SAFETY: must be main thread
pub unsafe fn slave<'a>() -> &'a mut Pic {
&mut *SLAVE.get()
unsafe { &mut *SLAVE.get() }
}
pub unsafe fn init() {
let master = master();
let slave = slave();
unsafe {
let master = master();
let slave = slave();
// Start initialization
master.cmd.write(0x11);
slave.cmd.write(0x11);
// Start initialization
master.cmd.write(0x11);
slave.cmd.write(0x11);
// Set offsets
master.data.write(0x20);
slave.data.write(0x28);
// Set offsets
master.data.write(0x20);
slave.data.write(0x28);
// Set up cascade
master.data.write(4);
slave.data.write(2);
// Set up cascade
master.data.write(4);
slave.data.write(2);
// Set up interrupt mode (1 is 8086/88 mode, 2 is auto EOI)
master.data.write(1);
slave.data.write(1);
// Set up interrupt mode (1 is 8086/88 mode, 2 is auto EOI)
master.data.write(1);
slave.data.write(1);
// Unmask interrupts
master.data.write(0);
slave.data.write(0);
// Unmask interrupts
master.data.write(0);
slave.data.write(0);
// Ack remaining interrupts
master.ack();
slave.ack();
// Ack remaining interrupts
master.ack();
slave.ack();
// probably already set to PIC, but double-check
irq::set_irq_method(irq::IrqMethod::Pic);
// probably already set to PIC, but double-check
irq::set_irq_method(irq::IrqMethod::Pic);
}
}
pub unsafe fn disable() {
master().data.write(0xFF);
slave().data.write(0xFF);
unsafe {
master().data.write(0xFF);
slave().data.write(0xFF);
}
}
pub struct Pic {
src/arch/x86_shared/device/pit.rs
+15 -11
View File
@@ -9,11 +9,11 @@ static COMMAND: SyncUnsafeCell<Pio<u8>> = SyncUnsafeCell::new(Pio::new(0x43));
// SAFETY: must be externally syncd
pub unsafe fn chan0<'a>() -> &'a mut Pio<u8> {
&mut *CHAN0.get()
unsafe { &mut *CHAN0.get() }
}
// SAFETY: must be externally syncd
pub unsafe fn command<'a>() -> &'a mut Pio<u8> {
&mut *COMMAND.get()
unsafe { &mut *COMMAND.get() }
}
const SELECT_CHAN0: u8 = 0b00 << 6;
@@ -31,16 +31,20 @@ pub const CHAN0_DIVISOR: u16 = 4847;
pub const RATE: u128 = (CHAN0_DIVISOR as u128 * PERIOD_FS) / 1_000_000;
pub unsafe fn init() {
command().write(SELECT_CHAN0 | ACCESS_LOHI | MODE_2);
chan0().write(CHAN0_DIVISOR as u8);
chan0().write((CHAN0_DIVISOR >> 8) as u8);
unsafe {
command().write(SELECT_CHAN0 | ACCESS_LOHI | MODE_2);
chan0().write(CHAN0_DIVISOR as u8);
chan0().write((CHAN0_DIVISOR >> 8) as u8);
}
}
pub unsafe fn read() -> u16 {
command().write(SELECT_CHAN0 | ACCESS_LATCH);
let low = chan0().read();
let high = chan0().read();
let counter = ((high as u16) << 8) | (low as u16);
// Counter is inverted, subtract from CHAN0_DIVISOR
CHAN0_DIVISOR.saturating_sub(counter)
unsafe {
command().write(SELECT_CHAN0 | ACCESS_LATCH);
let low = chan0().read();
let high = chan0().read();
let counter = ((high as u16) << 8) | (low as u16);
// Counter is inverted, subtract from CHAN0_DIVISOR
CHAN0_DIVISOR.saturating_sub(counter)
}
}
src/arch/x86_shared/device/tsc.rs
+33 -31
View File
@@ -121,39 +121,41 @@ pub fn get_kvm_support() -> &'static Option<KvmSupport> {
}
pub unsafe fn init() -> bool {
let cpuid = crate::cpuid::cpuid();
if !cpuid.get_feature_info().map_or(false, |f| f.has_tsc()) {
return false;
}
unsafe {
let cpuid = crate::cpuid::cpuid();
if !cpuid.get_feature_info().map_or(false, |f| f.has_tsc()) {
return false;
}
let kvm_support = get_kvm_support();
let kvm_support = get_kvm_support();
if let Some(kvm_support) = kvm_support
&& kvm_support
.supp_feats
.contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE)
{
let frame = allocate_frame().expect("failed to allocate timer page");
x86::msr::wrmsr(MSR_KVM_SYSTEM_TIME_NEW, (frame.base().data() as u64) | 1);
let ptr =
crate::paging::RmmA::phys_to_virt(frame.base()).data() as *const PvclockVcpuTimeInfo;
PercpuBlock::current()
.misc_arch_info
.tsc_info
.vcpu_page
.set(ptr);
if let Some(kvm_support) = kvm_support
&& kvm_support
.supp_feats
.contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE)
{
let frame = allocate_frame().expect("failed to allocate timer page");
x86::msr::wrmsr(MSR_KVM_SYSTEM_TIME_NEW, (frame.base().data() as u64) | 1);
let ptr = crate::paging::RmmA::phys_to_virt(frame.base()).data()
as *const PvclockVcpuTimeInfo;
PercpuBlock::current()
.misc_arch_info
.tsc_info
.vcpu_page
.set(ptr);
/*let tsc_ghz = loop {
let val1 = ptr.read_volatile();
let val2 = ptr.read_volatile();
if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version {
continue;
}
let val1
break tsc_hz / 1_000_000_000;
};*/
true
} else {
false
/*let tsc_ghz = loop {
let val1 = ptr.read_volatile();
let val2 = ptr.read_volatile();
if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version {
continue;
}
let val1
break tsc_hz / 1_000_000_000;
};*/
true
} else {
false
}
}
}
src/arch/x86_shared/idt.rs
+127 -118
View File
@@ -92,23 +92,25 @@ pub fn available_irqs_iter(cpu_id: LogicalCpuId) -> impl Iterator<Item = u8> + '
#[cfg(target_arch = "x86")]
macro_rules! use_irq(
( $idt: expr, $number:literal, $func:ident ) => {{
( $idt: expr_2021, $number:literal, $func:ident ) => {{
$idt[$number].set_func($func);
}}
);
#[cfg(target_arch = "x86")]
macro_rules! use_default_irqs(
($idt:expr) => {{
($idt:expr_2021) => {{
use crate::interrupt::irq::*;
default_irqs!($idt, use_irq);
}}
);
pub unsafe fn init() {
let idt = &mut *INIT_IDT.get();
set_exceptions(idt);
dtables::lidt(&DescriptorTablePointer::new(&idt));
unsafe {
let idt = &mut *INIT_IDT.get();
set_exceptions(idt);
dtables::lidt(&DescriptorTablePointer::new(&idt));
}
}
fn set_exceptions(idt: &mut [IdtEntry]) {
@@ -155,143 +157,150 @@ const fn new_idt_reservations() -> [AtomicU32; 8] {
/// Initialize the IDT for a processor
pub unsafe fn init_paging_post_heap(cpu_id: LogicalCpuId) {
let mut idts_guard = IDTS.write();
let idts_btree = idts_guard.get_or_insert_with(HashMap::new);
unsafe {
let mut idts_guard = IDTS.write();
let idts_btree = idts_guard.get_or_insert_with(HashMap::new);
if cpu_id == LogicalCpuId::BSP {
idts_btree.insert(cpu_id, &mut *INIT_BSP_IDT.get());
} else {
let idt = idts_btree
.entry(cpu_id)
.or_insert_with(|| Box::leak(Box::new(Idt::new())));
init_generic(cpu_id, idt);
if cpu_id == LogicalCpuId::BSP {
idts_btree.insert(cpu_id, &mut *INIT_BSP_IDT.get());
} else {
let idt = idts_btree
.entry(cpu_id)
.or_insert_with(|| Box::leak(Box::new(Idt::new())));
init_generic(cpu_id, idt);
}
}
}
/// Initializes a fully functional IDT for use before it be moved into the map. This is ONLY called
/// on the BSP, since the kernel heap is ready for the APs.
pub unsafe fn init_paging_bsp() {
init_generic(LogicalCpuId::BSP, &mut *INIT_BSP_IDT.get());
unsafe {
init_generic(LogicalCpuId::BSP, &mut *INIT_BSP_IDT.get());
}
}
/// Initializes an IDT for any type of processor.
pub unsafe fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt) {
let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations);
unsafe {
let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations);
let idtr: DescriptorTablePointer<X86IdtEntry> = DescriptorTablePointer {
limit: (current_idt.len() * mem::size_of::<IdtEntry>() - 1) as u16,
base: current_idt.as_ptr() as *const X86IdtEntry,
};
let idtr: DescriptorTablePointer<X86IdtEntry> = DescriptorTablePointer {
limit: (current_idt.len() * mem::size_of::<IdtEntry>() - 1) as u16,
base: current_idt.as_ptr() as *const X86IdtEntry,
};
let backup_ist = {
// We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate
// stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are
// considered the most fatal, especially Double Faults which are caused by errors __when
// accessing the system IDT__. If that goes wrong, then kernel memory may be partially
// corrupt, and we want a separate stack.
//
// Note that each CPU has its own "backup interrupt stack".
let index = 1_u8;
let backup_ist = {
// We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate
// stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are
// considered the most fatal, especially Double Faults which are caused by errors __when
// accessing the system IDT__. If that goes wrong, then kernel memory may be partially
// corrupt, and we want a separate stack.
//
// Note that each CPU has its own "backup interrupt stack".
let index = 1_u8;
// Put them in the 1st entry of the IST.
#[cfg(target_arch = "x86_64")] // TODO: x86
{
use crate::paging::PAGE_SIZE;
// Allocate 64 KiB of stack space for the backup stack.
const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4;
let frames = crate::memory::allocate_p2frame(4)
.expect("failed to allocate pages for backup interrupt stack");
// Put them in the 1st entry of the IST.
#[cfg(target_arch = "x86_64")] // TODO: x86
{
use crate::paging::PAGE_SIZE;
// Allocate 64 KiB of stack space for the backup stack.
const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4;
let frames = crate::memory::allocate_p2frame(4)
.expect("failed to allocate pages for backup interrupt stack");
use crate::paging::{RmmA, RmmArch};
use crate::paging::{RmmA, RmmArch};
// Physical pages are mapped linearly. So is the linearly mapped virtual memory.
let base_address = RmmA::phys_to_virt(frames.base());
// Physical pages are mapped linearly. So is the linearly mapped virtual memory.
let base_address = RmmA::phys_to_virt(frames.base());
// Stack always grows downwards.
let address = base_address.data() + BACKUP_STACK_SIZE;
// Stack always grows downwards.
let address = base_address.data() + BACKUP_STACK_SIZE;
(*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64;
(*crate::gdt::pcr()).tss.ist[usize::from(index - 1)] = address as u64;
}
index
};
set_exceptions(current_idt);
current_idt[2].set_ist(backup_ist);
current_idt[8].set_ist(backup_ist);
current_idt[18].set_ist(backup_ist);
#[cfg(target_arch = "x86_64")]
assert_eq!(
__generic_interrupts_end as usize - __generic_interrupts_start as usize,
224 * 8
);
#[cfg(target_arch = "x86_64")]
for i in 0..224 {
current_idt[i + 32]
.set_func(mem::transmute(__generic_interrupts_start as usize + i * 8));
}
index
};
// reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions
*current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF;
set_exceptions(current_idt);
current_idt[2].set_ist(backup_ist);
current_idt[8].set_ist(backup_ist);
current_idt[18].set_ist(backup_ist);
if cpu_id == LogicalCpuId::BSP {
// Set up IRQs
current_idt[32].set_func(irq::pit_stack);
current_idt[33].set_func(irq::keyboard);
current_idt[34].set_func(irq::cascade);
current_idt[35].set_func(irq::com2);
current_idt[36].set_func(irq::com1);
current_idt[37].set_func(irq::lpt2);
current_idt[38].set_func(irq::floppy);
current_idt[39].set_func(irq::lpt1);
current_idt[40].set_func(irq::rtc);
current_idt[41].set_func(irq::pci1);
current_idt[42].set_func(irq::pci2);
current_idt[43].set_func(irq::pci3);
current_idt[44].set_func(irq::mouse);
current_idt[45].set_func(irq::fpu);
current_idt[46].set_func(irq::ata1);
current_idt[47].set_func(irq::ata2);
current_idt[48].set_func(irq::lapic_timer);
current_idt[49].set_func(irq::lapic_error);
#[cfg(target_arch = "x86_64")]
assert_eq!(
__generic_interrupts_end as usize - __generic_interrupts_start as usize,
224 * 8
);
// reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error.
*current_reservations[1].get_mut() |= 0x0003_FFFF;
} else {
// TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU
current_idt[49].set_func(irq::lapic_error);
#[cfg(target_arch = "x86_64")]
for i in 0..224 {
current_idt[i + 32].set_func(mem::transmute(__generic_interrupts_start as usize + i * 8));
// reserve bit 49
*current_reservations[1].get_mut() |= 1 << 17;
}
#[cfg(target_arch = "x86")]
use_default_irqs!(current_idt);
// Set IPI handlers
current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup);
current_idt[IpiKind::Switch as usize].set_func(ipi::switch);
current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb);
current_idt[IpiKind::Pit as usize].set_func(ipi::pit);
idt.set_reserved_mut(IpiKind::Wakeup as u8, true);
idt.set_reserved_mut(IpiKind::Switch as u8, true);
idt.set_reserved_mut(IpiKind::Tlb as u8, true);
idt.set_reserved_mut(IpiKind::Pit as u8, true);
#[cfg(target_arch = "x86")]
{
let current_idt = &mut idt.entries;
// Set syscall function
current_idt[0x80].set_func(syscall::syscall);
current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT);
idt.set_reserved_mut(0x80, true);
}
#[cfg(feature = "profiling")]
crate::profiling::maybe_setup_timer(idt, cpu_id);
dtables::lidt(&idtr);
}
// reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions
*current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF;
if cpu_id == LogicalCpuId::BSP {
// Set up IRQs
current_idt[32].set_func(irq::pit_stack);
current_idt[33].set_func(irq::keyboard);
current_idt[34].set_func(irq::cascade);
current_idt[35].set_func(irq::com2);
current_idt[36].set_func(irq::com1);
current_idt[37].set_func(irq::lpt2);
current_idt[38].set_func(irq::floppy);
current_idt[39].set_func(irq::lpt1);
current_idt[40].set_func(irq::rtc);
current_idt[41].set_func(irq::pci1);
current_idt[42].set_func(irq::pci2);
current_idt[43].set_func(irq::pci3);
current_idt[44].set_func(irq::mouse);
current_idt[45].set_func(irq::fpu);
current_idt[46].set_func(irq::ata1);
current_idt[47].set_func(irq::ata2);
current_idt[48].set_func(irq::lapic_timer);
current_idt[49].set_func(irq::lapic_error);
// reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error.
*current_reservations[1].get_mut() |= 0x0003_FFFF;
} else {
// TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU
current_idt[49].set_func(irq::lapic_error);
// reserve bit 49
*current_reservations[1].get_mut() |= 1 << 17;
}
#[cfg(target_arch = "x86")]
use_default_irqs!(current_idt);
// Set IPI handlers
current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup);
current_idt[IpiKind::Switch as usize].set_func(ipi::switch);
current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb);
current_idt[IpiKind::Pit as usize].set_func(ipi::pit);
idt.set_reserved_mut(IpiKind::Wakeup as u8, true);
idt.set_reserved_mut(IpiKind::Switch as u8, true);
idt.set_reserved_mut(IpiKind::Tlb as u8, true);
idt.set_reserved_mut(IpiKind::Pit as u8, true);
#[cfg(target_arch = "x86")]
{
let current_idt = &mut idt.entries;
// Set syscall function
current_idt[0x80].set_func(syscall::syscall);
current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT);
idt.set_reserved_mut(0x80, true);
}
#[cfg(feature = "profiling")]
crate::profiling::maybe_setup_timer(idt, cpu_id);
dtables::lidt(&idtr);
}
bitflags! {
src/arch/x86_shared/interrupt/mod.rs
+12 -4
View File
@@ -8,7 +8,9 @@ pub use super::idt::{available_irqs_iter, is_reserved, set_reserved};
/// Clear interrupts
#[inline(always)]
pub unsafe fn disable() {
core::arch::asm!("cli", options(nomem, nostack));
unsafe {
core::arch::asm!("cli", options(nomem, nostack));
}
}
/// Set interrupts and halt
@@ -16,7 +18,9 @@ pub unsafe fn disable() {
/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
#[inline(always)]
pub unsafe fn enable_and_halt() {
core::arch::asm!("sti; hlt", options(nomem, nostack));
unsafe {
core::arch::asm!("sti; hlt", options(nomem, nostack));
}
}
/// Set interrupts and nop
@@ -24,13 +28,17 @@ pub unsafe fn enable_and_halt() {
/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
#[inline(always)]
pub unsafe fn enable_and_nop() {
core::arch::asm!("sti; nop", options(nomem, nostack));
unsafe {
core::arch::asm!("sti; nop", options(nomem, nostack));
}
}
/// Halt instruction
#[inline(always)]
pub unsafe fn halt() {
core::arch::asm!("hlt", options(nomem, nostack));
unsafe {
core::arch::asm!("hlt", options(nomem, nostack));
}
}
/// Pause instruction
src/arch/x86_shared/interrupt/trace.rs
+20 -16
View File
@@ -8,24 +8,28 @@ pub struct StackTrace {
impl StackTrace {
#[inline(always)]
pub unsafe fn start() -> Option<Self> {
let mut fp: usize;
#[cfg(target_arch = "x86")]
core::arch::asm!("mov {}, ebp", out(reg) fp);
#[cfg(target_arch = "x86_64")]
core::arch::asm!("mov {}, rbp", out(reg) fp);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(Self {
fp,
pc_ptr: pc_ptr as *const usize,
})
unsafe {
let mut fp: usize;
#[cfg(target_arch = "x86")]
core::arch::asm!("mov {}, ebp", out(reg) fp);
#[cfg(target_arch = "x86_64")]
core::arch::asm!("mov {}, rbp", out(reg) fp);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(Self {
fp,
pc_ptr: pc_ptr as *const usize,
})
}
}
pub unsafe fn next(self) -> Option<Self> {
let fp = *(self.fp as *const usize);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(Self {
fp: fp,
pc_ptr: pc_ptr as *const usize,
})
unsafe {
let fp = *(self.fp as *const usize);
let pc_ptr = fp.checked_add(mem::size_of::<usize>())?;
Some(Self {
fp: fp,
pc_ptr: pc_ptr as *const usize,
})
}
}
}
src/arch/x86_shared/paging/mod.rs
+30 -26
View File
@@ -32,40 +32,44 @@ pub const PAGE_MASK: usize = RmmA::PAGE_OFFSET_MASK;
/// Setup page attribute table
#[cold]
unsafe fn init_pat() {
let uncacheable = 0;
let write_combining = 1;
let write_through = 4;
//let write_protected = 5;
let write_back = 6;
let uncached = 7;
unsafe {
let uncacheable = 0;
let write_combining = 1;
let write_through = 4;
//let write_protected = 5;
let write_back = 6;
let uncached = 7;
let pat0 = write_back;
let pat1 = write_through;
let pat2 = uncached;
let pat3 = uncacheable;
let pat0 = write_back;
let pat1 = write_through;
let pat2 = uncached;
let pat3 = uncacheable;
let pat4 = write_combining;
let pat5 = pat1;
let pat6 = pat2;
let pat7 = pat3;
let pat4 = write_combining;
let pat5 = pat1;
let pat6 = pat2;
let pat7 = pat3;
msr::wrmsr(
msr::IA32_PAT,
pat7 << 56
| pat6 << 48
| pat5 << 40
| pat4 << 32
| pat3 << 24
| pat2 << 16
| pat1 << 8
| pat0,
);
msr::wrmsr(
msr::IA32_PAT,
pat7 << 56
| pat6 << 48
| pat5 << 40
| pat4 << 32
| pat3 << 24
| pat2 << 16
| pat1 << 8
| pat0,
);
}
}
/// Initialize PAT
#[cold]
pub unsafe fn init() {
init_pat();
unsafe {
init_pat();
}
}
/// Page
src/arch/x86_shared/start.rs
+210 -206
View File
@@ -65,190 +65,192 @@ pub struct KernelArgs {
}
/// The entry to Rust, all things must be initialized
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! {
let bootstrap = {
let args = args_ptr.read();
unsafe {
let bootstrap = {
let args = args_ptr.read();
// BSS should already be zero
{
assert_eq!(BSS_TEST_ZERO.get().read(), 0);
assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value());
}
// BSS should already be zero
{
assert_eq!(BSS_TEST_ZERO.get().read(), 0);
assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value());
}
KERNEL_BASE.store(args.kernel_base as usize, Ordering::SeqCst);
KERNEL_SIZE.store(args.kernel_size as usize, Ordering::SeqCst);
KERNEL_BASE.store(args.kernel_base as usize, Ordering::SeqCst);
KERNEL_SIZE.store(args.kernel_size as usize, Ordering::SeqCst);
// Convert env to slice
let env = slice::from_raw_parts(
(args.env_base as usize + crate::PHYS_OFFSET) as *const u8,
args.env_size as usize,
);
// Set up serial debug
#[cfg(feature = "serial_debug")]
device::serial::init();
// Set up graphical debug
#[cfg(feature = "graphical_debug")]
graphical_debug::init(env);
#[cfg(feature = "system76_ec_debug")]
device::system76_ec::init();
// Initialize logger
crate::log::init_logger(|r| {
use core::fmt::Write;
let _ = writeln!(
super::debug::Writer::new(),
"{}:{} -- {}",
r.target(),
r.level(),
r.args()
// Convert env to slice
let env = slice::from_raw_parts(
(args.env_base as usize + crate::PHYS_OFFSET) as *const u8,
args.env_size as usize,
);
});
info!("Redox OS starting...");
info!(
"Kernel: {:X}:{:X}",
{ args.kernel_base },
{ args.kernel_base } + { args.kernel_size }
);
info!(
"Stack: {:X}:{:X}",
{ args.stack_base },
{ args.stack_base } + { args.stack_size }
);
info!(
"Env: {:X}:{:X}",
{ args.env_base },
{ args.env_base } + { args.env_size }
);
info!(
"RSDP: {:X}:{:X}",
{ args.acpi_rsdp_base },
{ args.acpi_rsdp_base } + { args.acpi_rsdp_size }
);
info!(
"Areas: {:X}:{:X}",
{ args.areas_base },
{ args.areas_base } + { args.areas_size }
);
info!(
"Bootstrap: {:X}:{:X}",
{ args.bootstrap_base },
{ args.bootstrap_base } + { args.bootstrap_size }
);
// Set up serial debug
#[cfg(feature = "serial_debug")]
device::serial::init();
// Set up GDT before paging
gdt::init();
// Set up graphical debug
#[cfg(feature = "graphical_debug")]
graphical_debug::init(env);
// Set up IDT before paging
idt::init();
#[cfg(feature = "system76_ec_debug")]
device::system76_ec::init();
// Initialize RMM
register_bootloader_areas(args.areas_base as usize, args.areas_size as usize);
register_memory_region(
args.kernel_base as usize,
args.kernel_size as usize,
BootloaderMemoryKind::Kernel,
);
register_memory_region(
args.stack_base as usize,
args.stack_size as usize,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.env_base as usize,
args.env_size as usize,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.acpi_rsdp_base as usize,
args.acpi_rsdp_size as usize,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.bootstrap_base as usize,
args.bootstrap_size as usize,
BootloaderMemoryKind::IdentityMap,
);
#[cfg(target_arch = "x86")]
crate::startup::memory::init(Some(0x100000), Some(0x40000000));
#[cfg(target_arch = "x86_64")]
crate::startup::memory::init(Some(0x100000), None);
// Initialize paging
paging::init();
// Set up GDT after paging with TLS
gdt::init_paging(
args.stack_base as usize + args.stack_size as usize,
LogicalCpuId::BSP,
);
// Set up IDT
idt::init_paging_bsp();
#[cfg(target_arch = "x86_64")]
crate::alternative::early_init(true);
// Set up syscall instruction
interrupt::syscall::init();
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
AP_READY.store(false, Ordering::SeqCst);
BSP_READY.store(false, Ordering::SeqCst);
// Setup kernel heap
allocator::init();
#[cfg(all(target_arch = "x86_64", feature = "profiling"))]
crate::profiling::init();
// Set up double buffer for graphical debug now that heap is available
#[cfg(feature = "graphical_debug")]
graphical_debug::init_heap();
idt::init_paging_post_heap(LogicalCpuId::BSP);
// Activate memory logging
crate::log::init();
// Initialize miscellaneous processor features
#[cfg(target_arch = "x86_64")]
crate::misc::init(LogicalCpuId::BSP);
// Initialize devices
device::init();
// Read ACPI tables, starts APs
#[cfg(feature = "acpi")]
{
acpi::init(if args.acpi_rsdp_base != 0 {
Some((args.acpi_rsdp_base as usize + crate::PHYS_OFFSET) as *const u8)
} else {
None
// Initialize logger
crate::log::init_logger(|r| {
use core::fmt::Write;
let _ = writeln!(
super::debug::Writer::new(),
"{}:{} -- {}",
r.target(),
r.level(),
r.args()
);
});
device::init_after_acpi();
}
// Initialize all of the non-core devices not otherwise needed to complete initialization
device::init_noncore();
info!("Redox OS starting...");
info!(
"Kernel: {:X}:{:X}",
{ args.kernel_base },
{ args.kernel_base } + { args.kernel_size }
);
info!(
"Stack: {:X}:{:X}",
{ args.stack_base },
{ args.stack_base } + { args.stack_size }
);
info!(
"Env: {:X}:{:X}",
{ args.env_base },
{ args.env_base } + { args.env_size }
);
info!(
"RSDP: {:X}:{:X}",
{ args.acpi_rsdp_base },
{ args.acpi_rsdp_base } + { args.acpi_rsdp_size }
);
info!(
"Areas: {:X}:{:X}",
{ args.areas_base },
{ args.areas_base } + { args.areas_size }
);
info!(
"Bootstrap: {:X}:{:X}",
{ args.bootstrap_base },
{ args.bootstrap_base } + { args.bootstrap_size }
);
BSP_READY.store(true, Ordering::SeqCst);
// Set up GDT before paging
gdt::init();
crate::Bootstrap {
base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new(
// Set up IDT before paging
idt::init();
// Initialize RMM
register_bootloader_areas(args.areas_base as usize, args.areas_size as usize);
register_memory_region(
args.kernel_base as usize,
args.kernel_size as usize,
BootloaderMemoryKind::Kernel,
);
register_memory_region(
args.stack_base as usize,
args.stack_size as usize,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.env_base as usize,
args.env_size as usize,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.acpi_rsdp_base as usize,
args.acpi_rsdp_size as usize,
BootloaderMemoryKind::IdentityMap,
);
register_memory_region(
args.bootstrap_base as usize,
)),
page_count: (args.bootstrap_size as usize) / crate::memory::PAGE_SIZE,
env,
}
};
args.bootstrap_size as usize,
BootloaderMemoryKind::IdentityMap,
);
#[cfg(target_arch = "x86")]
crate::startup::memory::init(Some(0x100000), Some(0x40000000));
#[cfg(target_arch = "x86_64")]
crate::startup::memory::init(Some(0x100000), None);
crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap);
// Initialize paging
paging::init();
// Set up GDT after paging with TLS
gdt::init_paging(
args.stack_base as usize + args.stack_size as usize,
LogicalCpuId::BSP,
);
// Set up IDT
idt::init_paging_bsp();
#[cfg(target_arch = "x86_64")]
crate::alternative::early_init(true);
// Set up syscall instruction
interrupt::syscall::init();
// Reset AP variables
CPU_COUNT.store(1, Ordering::SeqCst);
AP_READY.store(false, Ordering::SeqCst);
BSP_READY.store(false, Ordering::SeqCst);
// Setup kernel heap
allocator::init();
#[cfg(all(target_arch = "x86_64", feature = "profiling"))]
crate::profiling::init();
// Set up double buffer for graphical debug now that heap is available
#[cfg(feature = "graphical_debug")]
graphical_debug::init_heap();
idt::init_paging_post_heap(LogicalCpuId::BSP);
// Activate memory logging
crate::log::init();
// Initialize miscellaneous processor features
#[cfg(target_arch = "x86_64")]
crate::misc::init(LogicalCpuId::BSP);
// Initialize devices
device::init();
// Read ACPI tables, starts APs
#[cfg(feature = "acpi")]
{
acpi::init(if args.acpi_rsdp_base != 0 {
Some((args.acpi_rsdp_base as usize + crate::PHYS_OFFSET) as *const u8)
} else {
None
});
device::init_after_acpi();
}
// Initialize all of the non-core devices not otherwise needed to complete initialization
device::init_noncore();
BSP_READY.store(true, Ordering::SeqCst);
crate::Bootstrap {
base: crate::memory::Frame::containing(crate::paging::PhysicalAddress::new(
args.bootstrap_base as usize,
)),
page_count: (args.bootstrap_size as usize) / crate::memory::PAGE_SIZE,
env,
}
};
crate::kmain(CPU_COUNT.load(Ordering::SeqCst), bootstrap);
}
}
#[repr(C, packed)]
@@ -263,56 +265,58 @@ pub struct KernelArgsAp {
/// Entry to rust for an AP
pub unsafe extern "C" fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
let cpu_id = {
let args = &*args_ptr;
let cpu_id = LogicalCpuId::new(args.cpu_id as u32);
let bsp_table = args.page_table as usize;
let _stack_start = args.stack_start as usize;
let stack_end = args.stack_end as usize;
unsafe {
let cpu_id = {
let args = &*args_ptr;
let cpu_id = LogicalCpuId::new(args.cpu_id as u32);
let bsp_table = args.page_table as usize;
let _stack_start = args.stack_start as usize;
let stack_end = args.stack_end as usize;
assert_eq!(BSS_TEST_ZERO.get().read(), 0);
assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value());
assert_eq!(BSS_TEST_ZERO.get().read(), 0);
assert_eq!(DATA_TEST_NONZERO.get().read(), usize::max_value());
// Set up GDT before paging
gdt::init();
// Set up GDT before paging
gdt::init();
// Set up IDT before paging
idt::init();
// Set up IDT before paging
idt::init();
// Initialize paging
RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table));
paging::init();
// Initialize paging
RmmA::set_table(TableKind::Kernel, PhysicalAddress::new(bsp_table));
paging::init();
// Set up GDT with TLS
gdt::init_paging(stack_end, cpu_id);
// Set up GDT with TLS
gdt::init_paging(stack_end, cpu_id);
#[cfg(all(target_arch = "x86_64", feature = "profiling"))]
crate::profiling::init();
#[cfg(all(target_arch = "x86_64", feature = "profiling"))]
crate::profiling::init();
// Set up IDT for AP
idt::init_paging_post_heap(cpu_id);
// Set up IDT for AP
idt::init_paging_post_heap(cpu_id);
#[cfg(target_arch = "x86_64")]
crate::alternative::early_init(false);
#[cfg(target_arch = "x86_64")]
crate::alternative::early_init(false);
// Set up syscall instruction
interrupt::syscall::init();
// Set up syscall instruction
interrupt::syscall::init();
// Initialize miscellaneous processor features
#[cfg(target_arch = "x86_64")]
crate::misc::init(cpu_id);
// Initialize miscellaneous processor features
#[cfg(target_arch = "x86_64")]
crate::misc::init(cpu_id);
// Initialize devices (for AP)
device::init_ap();
// Initialize devices (for AP)
device::init_ap();
AP_READY.store(true, Ordering::SeqCst);
AP_READY.store(true, Ordering::SeqCst);
cpu_id
};
cpu_id
};
while !BSP_READY.load(Ordering::SeqCst) {
interrupt::pause();
while !BSP_READY.load(Ordering::SeqCst) {
interrupt::pause();
}
crate::kmain_ap(cpu_id);
}
crate::kmain_ap(cpu_id);
}
src/arch/x86_shared/stop.rs
+47 -39
View File
@@ -4,24 +4,27 @@ use crate::{context, scheme::acpi, time};
use crate::syscall::io::{Io, Pio};
pub unsafe fn kreset() -> ! {
log::info!("kreset");
unsafe {
log::info!("kreset");
// 8042 reset
{
println!("Reset with 8042");
let mut port = Pio::<u8>::new(0x64);
while port.readf(2) {}
port.write(0xFE);
// 8042 reset
{
println!("Reset with 8042");
let mut port = Pio::<u8>::new(0x64);
while port.readf(2) {}
port.write(0xFE);
}
emergency_reset();
}
emergency_reset();
}
#[cfg(target_arch = "x86")]
pub unsafe fn emergency_reset() -> ! {
// Use triple fault to guarantee reset
core::arch::asm!(
"
unsafe {
// Use triple fault to guarantee reset
core::arch::asm!(
"
cli
sidt [esp+16]
// set IDT limit to zero
@@ -29,15 +32,17 @@ pub unsafe fn emergency_reset() -> ! {
lidt [esp+16]
int $3
",
options(noreturn)
);
options(noreturn)
);
}
}
#[cfg(target_arch = "x86_64")]
pub unsafe fn emergency_reset() -> ! {
// Use triple fault to guarantee reset
core::arch::asm!(
"
unsafe {
// Use triple fault to guarantee reset
core::arch::asm!(
"
cli
sidt [rsp+16]
// set IDT limit to zero
@@ -45,8 +50,9 @@ pub unsafe fn emergency_reset() -> ! {
lidt [rsp+16]
int $3
",
options(noreturn)
);
options(noreturn)
);
}
}
#[cfg(feature = "acpi")]
@@ -81,29 +87,31 @@ fn userspace_acpi_shutdown() {
}
pub unsafe fn kstop() -> ! {
log::info!("Running kstop()");
unsafe {
log::info!("Running kstop()");
#[cfg(feature = "acpi")]
userspace_acpi_shutdown();
#[cfg(feature = "acpi")]
userspace_acpi_shutdown();
// Magic shutdown code for bochs and qemu (older versions).
for c in "Shutdown".bytes() {
let port = 0x8900;
println!("Shutdown with outb(0x{:X}, '{}')", port, c as char);
Pio::<u8>::new(port).write(c);
}
// Magic shutdown code for bochs and qemu (older versions).
for c in "Shutdown".bytes() {
let port = 0x8900;
println!("Shutdown with outb(0x{:X}, '{}')", port, c as char);
Pio::<u8>::new(port).write(c);
}
// Magic shutdown using qemu default ACPI method
{
let port = 0x604;
let data = 0x2000;
println!("Shutdown with outb(0x{:X}, 0x{:X})", port, data);
Pio::<u16>::new(port).write(data);
}
// Magic shutdown using qemu default ACPI method
{
let port = 0x604;
let data = 0x2000;
println!("Shutdown with outb(0x{:X}, 0x{:X})", port, data);
Pio::<u16>::new(port).write(data);
}
// Magic code for VMWare. Also a hard lock.
println!("Shutdown with cli hlt");
loop {
core::arch::asm!("cli; hlt");
// Magic code for VMWare. Also a hard lock.
println!("Shutdown with cli hlt");
loop {
core::arch::asm!("cli; hlt");
}
}
}
src/common/mod.rs
+3 -3
View File
@@ -8,7 +8,7 @@ macro_rules! dbg {
() => {
$crate::println!("[{}:{}]", file!(), line!());
};
($val:expr) => {
($val:expr_2021) => {
// Use of `match` here is intentional because it affects the lifetimes
// of temporaries - https://stackoverflow.com/a/48732525/1063961
match $val {
@@ -20,8 +20,8 @@ macro_rules! dbg {
}
};
// Trailing comma with single argument is ignored
($val:expr,) => { $crate::dbg!($val) };
($($val:expr),+ $(,)?) => {
($val:expr_2021,) => { $crate::dbg!($val) };
($($val:expr_2021),+ $(,)?) => {
($($crate::dbg!($val)),+,)
};
}
src/context/arch/aarch64.rs
+99 -89
View File
@@ -200,92 +200,101 @@ pub static EMPTY_CR3: Once<rmm::PhysicalAddress> = Once::new();
// SAFETY: EMPTY_CR3 must be initialized.
pub unsafe fn empty_cr3() -> rmm::PhysicalAddress {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
unsafe {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
}
}
#[target_feature(enable = "neon")]
unsafe extern "C" fn fp_save(float_regs: &mut FloatRegisters) {
core::arch::asm!(
"stp q0, q1, [{3}, {0} + 16 * 0]",
"stp q2, q3, [{3}, {0} + 16 * 2]",
"stp q4, q5, [{3}, {0} + 16 * 4]",
"stp q6, q7, [{3}, {0} + 16 * 6]",
"stp q8, q9, [{3}, {0} + 16 * 8]",
"stp q10, q11, [{3}, {0} + 16 * 10]",
"stp q12, q13, [{3}, {0} + 16 * 12]",
"stp q14, q15, [{3}, {0} + 16 * 14]",
"stp q16, q17, [{3}, {0} + 16 * 16]",
"stp q18, q19, [{3}, {0} + 16 * 18]",
"stp q20, q21, [{3}, {0} + 16 * 20]",
"stp q22, q23, [{3}, {0} + 16 * 22]",
"stp q24, q25, [{3}, {0} + 16 * 24]",
"stp q26, q27, [{3}, {0} + 16 * 26]",
"stp q28, q29, [{3}, {0} + 16 * 28]",
"stp q30, q31, [{3}, {0} + 16 * 30]",
"mrs x9, fpcr",
"add {3}, {3}, {1}",
"str x9, [{3}]",
"mrs x9, fpsr",
"str x9, [{3}, {2} - {1}]",
const mem::offset_of!(FloatRegisters, fp_simd_regs),
const mem::offset_of!(FloatRegisters, fpcr),
const mem::offset_of!(FloatRegisters, fpsr),
inout(reg) float_regs => _,
);
unsafe {
core::arch::asm!(
"stp q0, q1, [{3}, {0} + 16 * 0]",
"stp q2, q3, [{3}, {0} + 16 * 2]",
"stp q4, q5, [{3}, {0} + 16 * 4]",
"stp q6, q7, [{3}, {0} + 16 * 6]",
"stp q8, q9, [{3}, {0} + 16 * 8]",
"stp q10, q11, [{3}, {0} + 16 * 10]",
"stp q12, q13, [{3}, {0} + 16 * 12]",
"stp q14, q15, [{3}, {0} + 16 * 14]",
"stp q16, q17, [{3}, {0} + 16 * 16]",
"stp q18, q19, [{3}, {0} + 16 * 18]",
"stp q20, q21, [{3}, {0} + 16 * 20]",
"stp q22, q23, [{3}, {0} + 16 * 22]",
"stp q24, q25, [{3}, {0} + 16 * 24]",
"stp q26, q27, [{3}, {0} + 16 * 26]",
"stp q28, q29, [{3}, {0} + 16 * 28]",
"stp q30, q31, [{3}, {0} + 16 * 30]",
"mrs x9, fpcr",
"add {3}, {3}, {1}",
"str x9, [{3}]",
"mrs x9, fpsr",
"str x9, [{3}, {2} - {1}]",
const mem::offset_of!(FloatRegisters, fp_simd_regs),
const mem::offset_of!(FloatRegisters, fpcr),
const mem::offset_of!(FloatRegisters, fpsr),
inout(reg) float_regs => _,
);
}
}
#[target_feature(enable = "neon")]
unsafe extern "C" fn fp_load(float_regs: &mut FloatRegisters) {
core::arch::asm!(
"ldp q0, q1, [{3}, {0} + 16 * 0]",
"ldp q2, q3, [{3}, {0} + 16 * 2]",
"ldp q4, q5, [{3}, {0} + 16 * 4]",
"ldp q6, q7, [{3}, {0} + 16 * 6]",
"ldp q8, q9, [{3}, {0} + 16 * 8]",
"ldp q10, q11, [{3}, {0} + 16 * 10]",
"ldp q12, q13, [{3}, {0} + 16 * 12]",
"ldp q14, q15, [{3}, {0} + 16 * 14]",
"ldp q16, q17, [{3}, {0} + 16 * 16]",
"ldp q18, q19, [{3}, {0} + 16 * 18]",
"ldp q20, q21, [{3}, {0} + 16 * 20]",
"ldp q22, q23, [{3}, {0} + 16 * 22]",
"ldp q24, q25, [{3}, {0} + 16 * 24]",
"ldp q26, q27, [{3}, {0} + 16 * 26]",
"ldp q28, q29, [{3}, {0} + 16 * 28]",
"ldp q30, q31, [{3}, {0} + 16 * 30]",
"add {3}, {3}, {1}",
"ldr x9, [{3}]",
"msr fpcr, x9",
"ldr x9, [{3}, {2} - {1}]",
"msr fpsr, x9",
const mem::offset_of!(FloatRegisters, fp_simd_regs),
const mem::offset_of!(FloatRegisters, fpcr),
const mem::offset_of!(FloatRegisters, fpsr),
inout(reg) float_regs => _,
);
unsafe {
core::arch::asm!(
"ldp q0, q1, [{3}, {0} + 16 * 0]",
"ldp q2, q3, [{3}, {0} + 16 * 2]",
"ldp q4, q5, [{3}, {0} + 16 * 4]",
"ldp q6, q7, [{3}, {0} + 16 * 6]",
"ldp q8, q9, [{3}, {0} + 16 * 8]",
"ldp q10, q11, [{3}, {0} + 16 * 10]",
"ldp q12, q13, [{3}, {0} + 16 * 12]",
"ldp q14, q15, [{3}, {0} + 16 * 14]",
"ldp q16, q17, [{3}, {0} + 16 * 16]",
"ldp q18, q19, [{3}, {0} + 16 * 18]",
"ldp q20, q21, [{3}, {0} + 16 * 20]",
"ldp q22, q23, [{3}, {0} + 16 * 22]",
"ldp q24, q25, [{3}, {0} + 16 * 24]",
"ldp q26, q27, [{3}, {0} + 16 * 26]",
"ldp q28, q29, [{3}, {0} + 16 * 28]",
"ldp q30, q31, [{3}, {0} + 16 * 30]",
"add {3}, {3}, {1}",
"ldr x9, [{3}]",
"msr fpcr, x9",
"ldr x9, [{3}, {2} - {1}]",
"msr fpsr, x9",
const mem::offset_of!(FloatRegisters, fp_simd_regs),
const mem::offset_of!(FloatRegisters, fpcr),
const mem::offset_of!(FloatRegisters, fpsr),
inout(reg) float_regs => _,
);
}
}
pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
fp_save(&mut *(prev.kfx.as_mut_ptr() as *mut FloatRegisters));
unsafe {
fp_save(&mut *(prev.kfx.as_mut_ptr() as *mut FloatRegisters));
prev.arch.fx_loadable = true;
prev.arch.fx_loadable = true;
if next.arch.fx_loadable {
fp_load(&mut *(next.kfx.as_mut_ptr() as *mut FloatRegisters));
if next.arch.fx_loadable {
fp_load(&mut *(next.kfx.as_mut_ptr() as *mut FloatRegisters));
}
PercpuBlock::current()
.new_addrsp_tmp
.set(next.addr_space.clone());
switch_to_inner(&mut prev.arch, &mut next.arch)
}
PercpuBlock::current()
.new_addrsp_tmp
.set(next.addr_space.clone());
switch_to_inner(&mut prev.arch, &mut next.arch)
}
#[naked]
unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) {
core::arch::naked_asm!(
"
unsafe {
core::arch::naked_asm!(
"
str x19, [x0, #{off_x19}]
ldr x19, [x1, #{off_x19}]
@@ -359,28 +368,29 @@ unsafe extern "C" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) {
b {switch_hook}
",
off_x19 = const(offset_of!(Context, x19)),
off_x20 = const(offset_of!(Context, x20)),
off_x21 = const(offset_of!(Context, x21)),
off_x22 = const(offset_of!(Context, x22)),
off_x23 = const(offset_of!(Context, x23)),
off_x24 = const(offset_of!(Context, x24)),
off_x25 = const(offset_of!(Context, x25)),
off_x26 = const(offset_of!(Context, x26)),
off_x27 = const(offset_of!(Context, x27)),
off_x28 = const(offset_of!(Context, x28)),
off_x29 = const(offset_of!(Context, fp)),
off_x30 = const(offset_of!(Context, lr)),
off_elr_el1 = const(offset_of!(Context, elr_el1)),
off_sp_el0 = const(offset_of!(Context, sp_el0)),
off_tpidr_el0 = const(offset_of!(Context, tpidr_el0)),
off_tpidrro_el0 = const(offset_of!(Context, tpidrro_el0)),
off_spsr_el1 = const(offset_of!(Context, spsr_el1)),
off_esr_el1 = const(offset_of!(Context, esr_el1)),
off_sp = const(offset_of!(Context, sp)),
off_x19 = const(offset_of!(Context, x19)),
off_x20 = const(offset_of!(Context, x20)),
off_x21 = const(offset_of!(Context, x21)),
off_x22 = const(offset_of!(Context, x22)),
off_x23 = const(offset_of!(Context, x23)),
off_x24 = const(offset_of!(Context, x24)),
off_x25 = const(offset_of!(Context, x25)),
off_x26 = const(offset_of!(Context, x26)),
off_x27 = const(offset_of!(Context, x27)),
off_x28 = const(offset_of!(Context, x28)),
off_x29 = const(offset_of!(Context, fp)),
off_x30 = const(offset_of!(Context, lr)),
off_elr_el1 = const(offset_of!(Context, elr_el1)),
off_sp_el0 = const(offset_of!(Context, sp_el0)),
off_tpidr_el0 = const(offset_of!(Context, tpidr_el0)),
off_tpidrro_el0 = const(offset_of!(Context, tpidrro_el0)),
off_spsr_el1 = const(offset_of!(Context, spsr_el1)),
off_esr_el1 = const(offset_of!(Context, esr_el1)),
off_sp = const(offset_of!(Context, sp)),
switch_hook = sym crate::context::switch_finish_hook,
);
switch_hook = sym crate::context::switch_finish_hook,
);
}
}
/// Allocates a new empty utable
src/context/arch/riscv64.rs
+31 -25
View File
@@ -134,23 +134,28 @@ pub static EMPTY_CR3: Once<rmm::PhysicalAddress> = Once::new();
// SAFETY: EMPTY_CR3 must be initialized.
pub unsafe fn empty_cr3() -> rmm::PhysicalAddress {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
unsafe {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
}
}
/// Switch to the next context by restoring its stack and registers
pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
// FIXME floating point
PercpuBlock::current()
.new_addrsp_tmp
.set(next.addr_space.clone());
unsafe {
// FIXME floating point
PercpuBlock::current()
.new_addrsp_tmp
.set(next.addr_space.clone());
switch_to_inner(&mut prev.arch, &mut next.arch);
switch_to_inner(&mut prev.arch, &mut next.arch);
}
}
#[naked]
unsafe extern "C" fn switch_to_inner(prev: &mut Context, next: &mut Context) {
core::arch::naked_asm!(r#"
unsafe {
core::arch::naked_asm!(r#"
sd s1, {off_s1}(a0)
ld s1, {off_s1}(a1)
@@ -203,24 +208,25 @@ unsafe extern "C" fn switch_to_inner(prev: &mut Context, next: &mut Context) {
j {switch_hook}
"#,
off_s1 = const(offset_of!(Context, s1)),
off_s2 = const(offset_of!(Context, s2)),
off_s3 = const(offset_of!(Context, s3)),
off_s4 = const(offset_of!(Context, s4)),
off_s5 = const(offset_of!(Context, s5)),
off_s6 = const(offset_of!(Context, s6)),
off_s7 = const(offset_of!(Context, s7)),
off_s8 = const(offset_of!(Context, s8)),
off_s9 = const(offset_of!(Context, s9)),
off_s10 = const(offset_of!(Context, s10)),
off_s11 = const(offset_of!(Context, s11)),
off_sp = const(offset_of!(Context, sp)),
off_ra = const(offset_of!(Context, ra)),
off_fp = const(offset_of!(Context, fp)),
off_sstatus = const(offset_of!(Context, sstatus)),
off_s1 = const(offset_of!(Context, s1)),
off_s2 = const(offset_of!(Context, s2)),
off_s3 = const(offset_of!(Context, s3)),
off_s4 = const(offset_of!(Context, s4)),
off_s5 = const(offset_of!(Context, s5)),
off_s6 = const(offset_of!(Context, s6)),
off_s7 = const(offset_of!(Context, s7)),
off_s8 = const(offset_of!(Context, s8)),
off_s9 = const(offset_of!(Context, s9)),
off_s10 = const(offset_of!(Context, s10)),
off_s11 = const(offset_of!(Context, s11)),
off_sp = const(offset_of!(Context, sp)),
off_ra = const(offset_of!(Context, ra)),
off_fp = const(offset_of!(Context, fp)),
off_sstatus = const(offset_of!(Context, sstatus)),
switch_hook = sym crate::context::switch_finish_hook,
);
switch_hook = sym crate::context::switch_finish_hook,
);
}
}
/// Allocates a new empty utable
src/context/arch/x86.rs
+48 -42
View File
@@ -209,57 +209,62 @@ pub static EMPTY_CR3: Once<rmm::PhysicalAddress> = Once::new();
// SAFETY: EMPTY_CR3 must be initialized.
pub unsafe fn empty_cr3() -> rmm::PhysicalAddress {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
unsafe {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
}
}
/// Switch to the next context by restoring its stack and registers
pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
if let Some(ref stack) = next.kstack {
crate::gdt::set_tss_stack(stack.initial_top() as usize);
}
crate::gdt::set_userspace_io_allowed(next.arch.userspace_io_allowed);
unsafe {
if let Some(ref stack) = next.kstack {
crate::gdt::set_tss_stack(stack.initial_top() as usize);
}
crate::gdt::set_userspace_io_allowed(next.arch.userspace_io_allowed);
core::arch::asm!("
core::arch::asm!("
fxsave [{prev_fx}]
fxrstor [{next_fx}]
", prev_fx = in(reg) prev.kfx.as_mut_ptr(),
next_fx = in(reg) next.kfx.as_ptr(),
);
next_fx = in(reg) next.kfx.as_ptr(),
);
{
let gdt = &mut (&mut *pcr()).gdt;
{
let gdt = &mut (&mut *pcr()).gdt;
prev.arch.fsbase = gdt[GDT_USER_FS].offset() as usize;
gdt[GDT_USER_FS].set_offset(next.arch.fsbase as u32);
prev.arch.gsbase = gdt[GDT_USER_GS].offset() as usize;
gdt[GDT_USER_GS].set_offset(next.arch.gsbase as u32);
prev.arch.fsbase = gdt[GDT_USER_FS].offset() as usize;
gdt[GDT_USER_FS].set_offset(next.arch.fsbase as u32);
prev.arch.gsbase = gdt[GDT_USER_GS].offset() as usize;
gdt[GDT_USER_GS].set_offset(next.arch.gsbase as u32);
}
PercpuBlock::current()
.new_addrsp_tmp
.set(next.addr_space.clone());
core::arch::asm!(
"call {inner}",
inner = sym switch_to_inner,
in("ecx") &mut prev.arch,
in("edx") &mut next.arch,
);
}
PercpuBlock::current()
.new_addrsp_tmp
.set(next.addr_space.clone());
core::arch::asm!(
"call {inner}",
inner = sym switch_to_inner,
in("ecx") &mut prev.arch,
in("edx") &mut next.arch,
);
}
// Check disassembly!
#[naked]
unsafe extern "cdecl" fn switch_to_inner() {
use Context as Cx;
unsafe {
use Context as Cx;
core::arch::naked_asm!(
// As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"):
//
// - the current parameters are passed in the registers `edi`, `esi`,
// - we can modify scratch registers, e.g. rax
// - we cannot change callee-preserved registers arbitrarily, e.g. ebx, which is why we
// store them here in the first place.
concat!("
core::arch::naked_asm!(
// As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"):
//
// - the current parameters are passed in the registers `edi`, `esi`,
// - we can modify scratch registers, e.g. rax
// - we cannot change callee-preserved registers arbitrarily, e.g. ebx, which is why we
// store them here in the first place.
concat!("
// ecx is prev, edx is next
// Save old registers, and load new ones
@@ -297,16 +302,17 @@ unsafe extern "cdecl" fn switch_to_inner() {
"),
off_eflags = const(offset_of!(Cx, eflags)),
off_eflags = const(offset_of!(Cx, eflags)),
off_ebx = const(offset_of!(Cx, ebx)),
off_edi = const(offset_of!(Cx, edi)),
off_esi = const(offset_of!(Cx, esi)),
off_ebp = const(offset_of!(Cx, ebp)),
off_esp = const(offset_of!(Cx, esp)),
off_ebx = const(offset_of!(Cx, ebx)),
off_edi = const(offset_of!(Cx, edi)),
off_esi = const(offset_of!(Cx, esi)),
off_ebp = const(offset_of!(Cx, ebp)),
off_esp = const(offset_of!(Cx, esp)),
switch_hook = sym crate::context::switch_finish_hook,
);
switch_hook = sym crate::context::switch_finish_hook,
);
}
}
/// Allocates a new identically mapped ktable and empty utable (same memory on x86)
src/context/arch/x86_64.rs
+71 -65
View File
@@ -227,56 +227,59 @@ pub static EMPTY_CR3: Once<rmm::PhysicalAddress> = Once::new();
// SAFETY: EMPTY_CR3 must be initialized.
pub unsafe fn empty_cr3() -> rmm::PhysicalAddress {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
unsafe {
debug_assert!(EMPTY_CR3.poll().is_some());
*EMPTY_CR3.get_unchecked()
}
}
/// Switch to the next context by restoring its stack and registers
pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
// Update contexts' timestamps
let switch_time = crate::time::monotonic();
prev.cpu_time += switch_time.saturating_sub(prev.switch_time);
next.switch_time = switch_time;
unsafe {
// Update contexts' timestamps
let switch_time = crate::time::monotonic();
prev.cpu_time += switch_time.saturating_sub(prev.switch_time);
next.switch_time = switch_time;
let pcr = crate::gdt::pcr();
let pcr = crate::gdt::pcr();
if let Some(ref stack) = next.kstack {
crate::gdt::set_tss_stack(pcr, stack.initial_top() as usize);
}
crate::gdt::set_userspace_io_allowed(pcr, next.arch.userspace_io_allowed);
if let Some(ref stack) = next.kstack {
crate::gdt::set_tss_stack(pcr, stack.initial_top() as usize);
}
crate::gdt::set_userspace_io_allowed(pcr, next.arch.userspace_io_allowed);
core::arch::asm!(
alternative2!(
feature1: "xsaveopt",
then1: ["
core::arch::asm!(
alternative2!(
feature1: "xsaveopt",
then1: ["
mov eax, 0xffffffff
mov edx, eax
xsaveopt64 [{prev_fx}]
xrstor64 [{next_fx}]
"],
feature2: "xsave",
then2: ["
feature2: "xsave",
then2: ["
mov eax, 0xffffffff
mov edx, eax
xsave64 [{prev_fx}]
xrstor64 [{next_fx}]
"],
default: ["
default: ["
fxsave64 [{prev_fx}]
fxrstor64 [{next_fx}]
"]
),
prev_fx = in(reg) prev.kfx.as_mut_ptr(),
next_fx = in(reg) next.kfx.as_ptr(),
out("eax") _,
out("edx") _,
);
),
prev_fx = in(reg) prev.kfx.as_mut_ptr(),
next_fx = in(reg) next.kfx.as_ptr(),
out("eax") _,
out("edx") _,
);
{
core::arch::asm!(
alternative!(
feature: "fsgsbase",
then: ["
{
core::arch::asm!(
alternative!(
feature: "fsgsbase",
then: ["
mov rax, [{next}+{fsbase_off}]
mov rcx, [{next}+{gsbase_off}]
@@ -290,10 +293,10 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
mov [{prev}+{fsbase_off}], rdx
mov [{prev}+{gsbase_off}], rax
"],
// TODO: Most applications will set FSBASE, but won't touch GSBASE. Maybe avoid
// wrmsr or even the swapgs+rdgsbase+wrgsbase+swapgs sequence if they are already
// equal?
default: ["
// TODO: Most applications will set FSBASE, but won't touch GSBASE. Maybe avoid
// wrmsr or even the swapgs+rdgsbase+wrgsbase+swapgs sequence if they are already
// equal?
default: ["
mov ecx, {MSR_FSBASE}
mov rdx, [{next}+{fsbase_off}]
mov eax, edx
@@ -308,35 +311,37 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
// {prev}
"]
),
out("rax") _,
out("rdx") _,
out("ecx") _, prev = in(reg) addr_of_mut!(prev.arch), next = in(reg) addr_of!(next.arch),
MSR_FSBASE = const msr::IA32_FS_BASE,
MSR_KERNEL_GSBASE = const msr::IA32_KERNEL_GSBASE,
gsbase_off = const offset_of!(Context, gsbase),
fsbase_off = const offset_of!(Context, fsbase),
);
),
out("rax") _,
out("rdx") _,
out("ecx") _, prev = in(reg) addr_of_mut!(prev.arch), next = in(reg) addr_of!(next.arch),
MSR_FSBASE = const msr::IA32_FS_BASE,
MSR_KERNEL_GSBASE = const msr::IA32_KERNEL_GSBASE,
gsbase_off = const offset_of!(Context, gsbase),
fsbase_off = const offset_of!(Context, fsbase),
);
}
(*pcr).percpu.new_addrsp_tmp.set(next.addr_space.clone());
switch_to_inner(&mut prev.arch, &mut next.arch)
}
(*pcr).percpu.new_addrsp_tmp.set(next.addr_space.clone());
switch_to_inner(&mut prev.arch, &mut next.arch)
}
// Check disassembly!
#[naked]
unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Context) {
use Context as Cx;
unsafe {
use Context as Cx;
core::arch::naked_asm!(
// As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"):
//
// - the current parameters are passed in the registers `rdi`, `rsi`,
// - we can modify scratch registers, e.g. rax
// - we cannot change callee-preserved registers arbitrarily, e.g. rbx, which is why we
// store them here in the first place.
concat!("
core::arch::naked_asm!(
// As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"):
//
// - the current parameters are passed in the registers `rdi`, `rsi`,
// - we can modify scratch registers, e.g. rax
// - we cannot change callee-preserved registers arbitrarily, e.g. rbx, which is why we
// store them here in the first place.
concat!("
// Save old registers, and load new ones
mov [rdi + {off_rbx}], rbx
mov rbx, [rsi + {off_rbx}]
@@ -378,18 +383,19 @@ unsafe extern "sysv64" fn switch_to_inner(_prev: &mut Context, _next: &mut Conte
"),
off_rflags = const(offset_of!(Cx, rflags)),
off_rflags = const(offset_of!(Cx, rflags)),
off_rbx = const(offset_of!(Cx, rbx)),
off_r12 = const(offset_of!(Cx, r12)),
off_r13 = const(offset_of!(Cx, r13)),
off_r14 = const(offset_of!(Cx, r14)),
off_r15 = const(offset_of!(Cx, r15)),
off_rbp = const(offset_of!(Cx, rbp)),
off_rsp = const(offset_of!(Cx, rsp)),
off_rbx = const(offset_of!(Cx, rbx)),
off_r12 = const(offset_of!(Cx, r12)),
off_r13 = const(offset_of!(Cx, r13)),
off_r14 = const(offset_of!(Cx, r14)),
off_r15 = const(offset_of!(Cx, r15)),
off_rbp = const(offset_of!(Cx, rbp)),
off_rsp = const(offset_of!(Cx, rsp)),
switch_hook = sym crate::context::switch_finish_hook,
);
switch_hook = sym crate::context::switch_finish_hook,
);
}
}
/// Allocates a new identically mapped ktable and empty utable (same memory on x86_64).
src/context/context.rs
+10 -9
View File
@@ -320,7 +320,7 @@ impl Context {
&mut self,
addr_space: Option<Arc<AddrSpaceWrapper>>,
) -> Option<Arc<AddrSpaceWrapper>> {
if let (Some(ref old), Some(ref new)) = (&self.addr_space, &addr_space)
if let (&Some(ref old), &Some(ref new)) = (&self.addr_space, &addr_space)
&& Arc::ptr_eq(old, new)
{
return addr_space;
@@ -346,17 +346,18 @@ impl Context {
addr_space.clone(),
);
if let Some(ref new) = addr_space {
let new_addrsp = new.acquire_read();
new_addrsp.used_by.atomic_set(this_percpu.cpu_id);
match addr_space {
Some(ref new) => {
let new_addrsp = new.acquire_read();
new_addrsp.used_by.atomic_set(this_percpu.cpu_id);
unsafe {
new_addrsp.table.utable.make_current();
unsafe {
new_addrsp.table.utable.make_current();
}
}
} else {
unsafe {
_ => unsafe {
crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3());
}
},
}
} else {
assert!(!self.running);
src/context/memory.rs
+86 -83
View File
@@ -2206,15 +2206,13 @@ impl GrantInfo {
flags
}
pub fn file_ref(&self) -> Option<&GrantFileRef> {
if let Provider::FmapBorrowed { ref file_ref, .. }
| Provider::Allocated {
cow_file_ref: Some(ref file_ref),
..
} = self.provider
{
Some(file_ref)
} else {
None
match self.provider {
Provider::FmapBorrowed { ref file_ref, .. }
| Provider::Allocated {
cow_file_ref: Some(ref file_ref),
..
} => Some(file_ref),
_ => None,
}
}
}
@@ -2513,94 +2511,99 @@ fn correct_inner<'l>(
let mut guard = foreign_address_space.acquire_upgradeable_read();
let src_page = src_base.next_by(pages_from_grant_start);
if let Some(_) = guard.grants.contains(src_page) {
let src_frame = if let Some((phys, _)) =
guard.table.utable.translate(src_page.start_address())
{
Frame::containing(phys)
} else {
// Grant was valid (TODO check), but we need to correct the underlying page.
// TODO: Access mode
match guard.grants.contains(src_page) {
Some(_) => {
let src_frame = match guard.table.utable.translate(src_page.start_address()) {
Some((phys, _)) => Frame::containing(phys),
_ => {
// Grant was valid (TODO check), but we need to correct the underlying page.
// TODO: Access mode
// TODO: Reasonable maximum?
let new_recursion_level = recursion_level
.checked_add(1)
.filter(|new_lvl| *new_lvl < 16)
.ok_or(PfError::RecursionLimitExceeded)?;
// TODO: Reasonable maximum?
let new_recursion_level = recursion_level
.checked_add(1)
.filter(|new_lvl| *new_lvl < 16)
.ok_or(PfError::RecursionLimitExceeded)?;
drop(guard);
drop(flusher);
drop(addr_space_guard);
drop(guard);
drop(flusher);
drop(addr_space_guard);
// FIXME: Can this result in invalid address space state?
let ext_addrspace = &foreign_address_space;
let (frame, _, _) = {
let g = ext_addrspace.acquire_write();
correct_inner(
ext_addrspace,
g,
src_page,
AccessMode::Read,
new_recursion_level,
)?
// FIXME: Can this result in invalid address space state?
let ext_addrspace = &foreign_address_space;
let (frame, _, _) = {
let g = ext_addrspace.acquire_write();
correct_inner(
ext_addrspace,
g,
src_page,
AccessMode::Read,
new_recursion_level,
)?
};
addr_space_guard = addr_space_lock.acquire_write();
addr_space = &mut *addr_space_guard;
flusher = Flusher::with_cpu_set(
&mut addr_space.used_by,
&addr_space_lock.tlb_ack,
);
guard = foreign_address_space.acquire_upgradeable_read();
frame
}
};
addr_space_guard = addr_space_lock.acquire_write();
addr_space = &mut *addr_space_guard;
flusher =
Flusher::with_cpu_set(&mut addr_space.used_by, &addr_space_lock.tlb_ack);
guard = foreign_address_space.acquire_upgradeable_read();
let info =
get_page_info(src_frame).expect("all allocated frames need a PageInfo");
frame
};
match info.add_ref(RefKind::Shared) {
Ok(()) => src_frame,
Err(AddRefError::CowToShared) => {
let CowResult {
new_frame,
old_frame,
} = cow(src_frame, info, RefKind::Shared)?;
let info = get_page_info(src_frame).expect("all allocated frames need a PageInfo");
if let Some(old_frame) = old_frame {
flusher.queue(old_frame, None, TlbShootdownActions::FREE);
flusher.flush();
}
match info.add_ref(RefKind::Shared) {
Ok(()) => src_frame,
Err(AddRefError::CowToShared) => {
let CowResult {
new_frame,
old_frame,
} = cow(src_frame, info, RefKind::Shared)?;
let mut guard = RwLockUpgradableGuard::upgrade(guard);
if let Some(old_frame) = old_frame {
flusher.queue(old_frame, None, TlbShootdownActions::FREE);
flusher.flush();
// TODO: flusher
unsafe {
guard
.table
.utable
.remap_with_full(src_page.start_address(), |_, f| {
(new_frame.base(), f)
});
}
new_frame
}
let mut guard = RwLockUpgradableGuard::upgrade(guard);
// TODO: flusher
unsafe {
guard
.table
.utable
.remap_with_full(src_page.start_address(), |_, f| {
(new_frame.base(), f)
});
}
new_frame
Err(AddRefError::SharedToCow) => unreachable!(),
Err(AddRefError::RcOverflow) => return Err(PfError::Oom),
}
Err(AddRefError::SharedToCow) => unreachable!(),
Err(AddRefError::RcOverflow) => return Err(PfError::Oom),
}
} else {
// Grant did not exist, but we did own a Provider::External mapping, and cannot
// simply let the current context fail. TODO: But all borrowed memory shouldn't
// really be lazy though? TODO: Should a grant be created?
_ => {
// Grant did not exist, but we did own a Provider::External mapping, and cannot
// simply let the current context fail. TODO: But all borrowed memory shouldn't
// really be lazy though? TODO: Should a grant be created?
let mut guard = RwLockUpgradableGuard::upgrade(guard);
let mut guard = RwLockUpgradableGuard::upgrade(guard);
// TODO: Should this be called?
log::warn!("Mapped zero page since grant didn't exist");
map_zeroed(
&mut guard.table.utable,
src_page,
grant_flags,
access == AccessMode::Write,
)?
// TODO: Should this be called?
log::warn!("Mapped zero page since grant didn't exist");
map_zeroed(
&mut guard.table.utable,
src_page,
grant_flags,
access == AccessMode::Write,
)?
}
}
}
// TODO: NonfatalInternalError if !MAP_LAZY and this page fault occurs.
src/context/switch.rs
+96 -85
View File
@@ -107,14 +107,19 @@ pub fn tick() {
/// # Safety
/// This function involves unsafe operations such as resetting state and releasing locks.
pub unsafe extern "C" fn switch_finish_hook() {
if let Some(switch_result) = PercpuBlock::current().switch_internals.switch_result.take() {
drop(switch_result);
} else {
// TODO: unreachable_unchecked()?
crate::arch::stop::emergency_reset();
unsafe {
match PercpuBlock::current().switch_internals.switch_result.take() {
Some(switch_result) => {
drop(switch_result);
}
_ => {
// TODO: unreachable_unchecked()?
crate::arch::stop::emergency_reset();
}
}
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
crate::percpu::switch_arch_hook();
}
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
crate::percpu::switch_arch_hook();
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -215,91 +220,97 @@ pub fn switch() -> SwitchResult {
};
// Switch process states, TSS stack pointer, and store new context ID
if let Some((mut prev_context_guard, mut next_context_guard)) = switch_context_opt {
// Update context states and prepare for the switch.
let prev_context = &mut *prev_context_guard;
let next_context = &mut *next_context_guard;
match switch_context_opt {
Some((mut prev_context_guard, mut next_context_guard)) => {
// Update context states and prepare for the switch.
let prev_context = &mut *prev_context_guard;
let next_context = &mut *next_context_guard;
// Set the previous context as "not running"
prev_context.running = false;
// Set the previous context as "not running"
prev_context.running = false;
// Set the next context as "running"
next_context.running = true;
// Set the CPU ID for the next context
next_context.cpu_id = Some(cpu_id);
// Set the next context as "running"
next_context.running = true;
// Set the CPU ID for the next context
next_context.cpu_id = Some(cpu_id);
let percpu = PercpuBlock::current();
unsafe {
percpu.switch_internals.set_current_context(Arc::clone(
ArcRwSpinlockWriteGuard::rwlock(&next_context_guard),
));
let percpu = PercpuBlock::current();
unsafe {
percpu.switch_internals.set_current_context(Arc::clone(
ArcRwSpinlockWriteGuard::rwlock(&next_context_guard),
));
}
// FIXME set the switch result in arch::switch_to instead
let prev_context = unsafe {
mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *prev_context_guard)
};
let next_context = unsafe {
mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *next_context_guard)
};
percpu
.switch_internals
.switch_result
.set(Some(SwitchResultInner {
_prev_guard: prev_context_guard,
_next_guard: next_context_guard,
}));
/*let (ptrace_session, ptrace_flags) = if let Some((session, bp)) = ptrace::sessions()
.get(&next_context.pid)
.map(|s| (Arc::downgrade(s), s.data.lock().breakpoint))
{
(Some(session), bp.map_or(PtraceFlags::empty(), |f| f.flags))
} else {
(None, PtraceFlags::empty())
};*/
let ptrace_flags = PtraceFlags::empty();
//*percpu.ptrace_session.borrow_mut() = ptrace_session;
percpu.ptrace_flags.set(ptrace_flags);
prev_context.inside_syscall =
percpu.inside_syscall.replace(next_context.inside_syscall);
#[cfg(feature = "syscall_debug")]
{
prev_context.syscall_debug_info = percpu
.syscall_debug_info
.replace(next_context.syscall_debug_info);
prev_context.syscall_debug_info.on_switch_from();
next_context.syscall_debug_info.on_switch_to();
}
percpu
.switch_internals
.being_sigkilled
.set(next_context.being_sigkilled);
unsafe {
arch::switch_to(prev_context, next_context);
}
// NOTE: After switch_to is called, the return address can even be different from the
// current return address, meaning that we cannot use local variables here, and that we
// need to use the `switch_finish_hook` to be able to release the locks. Newly created
// contexts will return directly to the function pointer passed to context::spawn, and not
// reach this code until the next context switch back.
if next_context.userspace {
percpu.stats.set_state(cpu_stats::CpuState::User);
} else {
percpu.stats.set_state(cpu_stats::CpuState::Kernel);
}
SwitchResult::Switched
}
_ => {
// No target was found, unset global lock and return
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
// FIXME set the switch result in arch::switch_to instead
let prev_context =
unsafe { mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *prev_context_guard) };
let next_context =
unsafe { mem::transmute::<&'_ mut Context, &'_ mut Context>(&mut *next_context_guard) };
percpu.stats.set_state(cpu_stats::CpuState::Idle);
percpu
.switch_internals
.switch_result
.set(Some(SwitchResultInner {
_prev_guard: prev_context_guard,
_next_guard: next_context_guard,
}));
/*let (ptrace_session, ptrace_flags) = if let Some((session, bp)) = ptrace::sessions()
.get(&next_context.pid)
.map(|s| (Arc::downgrade(s), s.data.lock().breakpoint))
{
(Some(session), bp.map_or(PtraceFlags::empty(), |f| f.flags))
} else {
(None, PtraceFlags::empty())
};*/
let ptrace_flags = PtraceFlags::empty();
//*percpu.ptrace_session.borrow_mut() = ptrace_session;
percpu.ptrace_flags.set(ptrace_flags);
prev_context.inside_syscall = percpu.inside_syscall.replace(next_context.inside_syscall);
#[cfg(feature = "syscall_debug")]
{
prev_context.syscall_debug_info = percpu
.syscall_debug_info
.replace(next_context.syscall_debug_info);
prev_context.syscall_debug_info.on_switch_from();
next_context.syscall_debug_info.on_switch_to();
SwitchResult::AllContextsIdle
}
percpu
.switch_internals
.being_sigkilled
.set(next_context.being_sigkilled);
unsafe {
arch::switch_to(prev_context, next_context);
}
// NOTE: After switch_to is called, the return address can even be different from the
// current return address, meaning that we cannot use local variables here, and that we
// need to use the `switch_finish_hook` to be able to release the locks. Newly created
// contexts will return directly to the function pointer passed to context::spawn, and not
// reach this code until the next context switch back.
if next_context.userspace {
percpu.stats.set_state(cpu_stats::CpuState::User);
} else {
percpu.stats.set_state(cpu_stats::CpuState::Kernel);
}
SwitchResult::Switched
} else {
// No target was found, unset global lock and return
arch::CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
percpu.stats.set_state(cpu_stats::CpuState::Idle);
SwitchResult::AllContextsIdle
}
}
src/devices/graphical_debug/display.rs
+31 -27
View File
@@ -48,42 +48,46 @@ impl Display {
/// Sync from offscreen to onscreen, unsafe because it trusts provided x, y, w, h
pub(super) unsafe fn sync(&mut self, x: usize, y: usize, w: usize, mut h: usize) {
if let Some(offscreen) = &self.offscreen {
let mut y = y;
while h > 0 {
let src_y = (self.offset_y + y) % self.height;
let src_offset = src_y * self.stride + x;
let dst_offset = y * self.stride + x;
unsafe {
if let Some(offscreen) = &self.offscreen {
let mut y = y;
while h > 0 {
let src_y = (self.offset_y + y) % self.height;
let src_offset = src_y * self.stride + x;
let dst_offset = y * self.stride + x;
ptr::copy(
offscreen.as_ptr().add(src_offset),
self.onscreen_ptr.add(dst_offset),
w,
);
ptr::copy(
offscreen.as_ptr().add(src_offset),
self.onscreen_ptr.add(dst_offset),
w,
);
y += 1;
h -= 1;
y += 1;
h -= 1;
}
}
}
}
// sync the whole screen (faster)
pub(super) unsafe fn sync_screen(&mut self) {
if let Some(offscreen) = &self.offscreen {
let stride_bytes = self.stride;
let first_part_len = (self.height - self.offset_y) * stride_bytes;
let second_part_len = self.offset_y * stride_bytes;
unsafe {
if let Some(offscreen) = &self.offscreen {
let stride_bytes = self.stride;
let first_part_len = (self.height - self.offset_y) * stride_bytes;
let second_part_len = self.offset_y * stride_bytes;
ptr::copy(
offscreen.as_ptr().add(self.offset_y * stride_bytes),
self.onscreen_ptr,
first_part_len,
);
ptr::copy(
offscreen.as_ptr(),
self.onscreen_ptr.add(first_part_len),
second_part_len,
);
ptr::copy(
offscreen.as_ptr().add(self.offset_y * stride_bytes),
self.onscreen_ptr,
first_part_len,
);
ptr::copy(
offscreen.as_ptr(),
self.onscreen_ptr.add(first_part_len),
second_part_len,
);
}
}
}
}
src/devices/uart_16550.rs
+2 -2
View File
@@ -67,14 +67,14 @@ impl SerialPort<Pio<u8>> {
impl SerialPort<Mmio<u32>> {
#[allow(dead_code)]
pub unsafe fn new(base: usize) -> &'static mut SerialPort<Mmio<u32>> {
&mut *(base as *mut Self)
unsafe { &mut *(base as *mut Self) }
}
}
impl SerialPort<Mmio<u8>> {
#[allow(dead_code)]
pub unsafe fn new(base: usize) -> &'static mut SerialPort<Mmio<u8>> {
&mut *(base as *mut Self)
unsafe { &mut *(base as *mut Self) }
}
}
src/dtb/irqchip.rs
+25 -15
View File
@@ -228,14 +228,17 @@ impl IrqChipList {
debug_assert!(queue[0..queue_idx].contains(&connection.parent));
if let Some(parent_interrupt) = connection.parent_interrupt {
let parent = &self.chips[connection.parent];
if let Ok(virq) = parent.ic.irq_xlate(parent_interrupt) {
// assert is unused
irq_desc[virq].basic.child_ic_idx = Some(cur_idx);
} else {
error!(
"Cannot connect irq chip {} to parent irq {} : {:?}",
cur_chip.phandle, parent.phandle, parent_interrupt
);
match parent.ic.irq_xlate(parent_interrupt) {
Ok(virq) => {
// assert is unused
irq_desc[virq].basic.child_ic_idx = Some(cur_idx);
}
_ => {
error!(
"Cannot connect irq chip {} to parent irq {} : {:?}",
cur_chip.phandle, parent.phandle, parent_interrupt
);
}
}
}
}
@@ -295,12 +298,17 @@ impl IrqChipCore {
pub fn trigger_virq(&mut self, virq: u32) {
if virq < 1024 {
let desc = &mut self.irq_desc[virq as usize];
if let Some(handler) = &mut desc.handler {
handler.irq_handler(virq);
} else if let Some(ic_idx) = desc.basic.child_ic_idx {
self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq);
} else {
irq_trigger(virq as u8);
match &mut desc.handler {
Some(handler) => {
handler.irq_handler(virq);
}
_ => {
if let Some(ic_idx) = desc.basic.child_ic_idx {
self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq);
} else {
irq_trigger(virq as u8);
}
}
}
}
}
@@ -336,7 +344,9 @@ impl IrqChipCore {
}
pub unsafe fn acknowledge(irq: usize) {
IRQ_CHIP.irq_eoi(irq as u32);
unsafe {
IRQ_CHIP.irq_eoi(irq as u32);
}
}
const INIT_HANDLER: Option<Box<dyn InterruptHandler>> = None;
src/externs.rs
+109 -101
View File
@@ -8,39 +8,41 @@ const WORD_SIZE: usize = mem::size_of::<usize>();
///
/// This faster implementation works by copying bytes not one-by-one, but in
/// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures).
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, len: usize) -> *mut u8 {
// TODO: Alignment? Some sources claim that even on relatively modern µ-arches, unaligned
// accesses spanning two pages, can take dozens of cycles. That means chunk-based memcpy can
// even be slower for small lengths if alignment is not taken into account.
//
// TODO: Optimize out smaller loops by first checking if len < WORD_SIZE, and possibly if
// dest + WORD_SIZE spans two pages, then doing one unaligned copy, then aligning up, and then
// doing one last unaligned copy?
//
// TODO: While we use the -fno-builtin equivalent, can we guarantee LLVM won't insert memcpy
// call inside here? Maybe write it in assembly?
unsafe {
// TODO: Alignment? Some sources claim that even on relatively modern µ-arches, unaligned
// accesses spanning two pages, can take dozens of cycles. That means chunk-based memcpy can
// even be slower for small lengths if alignment is not taken into account.
//
// TODO: Optimize out smaller loops by first checking if len < WORD_SIZE, and possibly if
// dest + WORD_SIZE spans two pages, then doing one unaligned copy, then aligning up, and then
// doing one last unaligned copy?
//
// TODO: While we use the -fno-builtin equivalent, can we guarantee LLVM won't insert memcpy
// call inside here? Maybe write it in assembly?
let mut i = 0_usize;
let mut i = 0_usize;
// First we copy len / WORD_SIZE chunks...
// First we copy len / WORD_SIZE chunks...
let chunks = len / WORD_SIZE;
let chunks = len / WORD_SIZE;
while i < chunks * WORD_SIZE {
dest.add(i)
.cast::<usize>()
.write_unaligned(src.add(i).cast::<usize>().read_unaligned());
i += WORD_SIZE;
while i < chunks * WORD_SIZE {
dest.add(i)
.cast::<usize>()
.write_unaligned(src.add(i).cast::<usize>().read_unaligned());
i += WORD_SIZE;
}
// .. then we copy len % WORD_SIZE bytes
while i < len {
dest.add(i).write(src.add(i).read());
i += 1;
}
dest
}
// .. then we copy len % WORD_SIZE bytes
while i < len {
dest.add(i).write(src.add(i).read());
i += 1;
}
dest
}
/// Memmove
@@ -49,48 +51,50 @@ pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, len: usize) -> *m
///
/// This faster implementation works by copying bytes not one-by-one, but in
/// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures).
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, len: usize) -> *mut u8 {
let chunks = len / WORD_SIZE;
unsafe {
let chunks = len / WORD_SIZE;
// TODO: also require dest - src < len before choosing to copy backwards?
if src < dest as *const u8 {
// We have to copy backwards if copying upwards.
// TODO: also require dest - src < len before choosing to copy backwards?
if src < dest as *const u8 {
// We have to copy backwards if copying upwards.
let mut i = len;
let mut i = len;
while i != chunks * WORD_SIZE {
i -= 1;
dest.add(i).write(src.add(i).read());
while i != chunks * WORD_SIZE {
i -= 1;
dest.add(i).write(src.add(i).read());
}
while i > 0 {
i -= WORD_SIZE;
dest.add(i)
.cast::<usize>()
.write_unaligned(src.add(i).cast::<usize>().read_unaligned());
}
} else {
// We have to copy forward if copying downwards.
let mut i = 0_usize;
while i < chunks * WORD_SIZE {
dest.add(i)
.cast::<usize>()
.write_unaligned(src.add(i).cast::<usize>().read_unaligned());
i += WORD_SIZE;
}
while i < len {
dest.add(i).write(src.add(i).read());
i += 1;
}
}
while i > 0 {
i -= WORD_SIZE;
dest.add(i)
.cast::<usize>()
.write_unaligned(src.add(i).cast::<usize>().read_unaligned());
}
} else {
// We have to copy forward if copying downwards.
let mut i = 0_usize;
while i < chunks * WORD_SIZE {
dest.add(i)
.cast::<usize>()
.write_unaligned(src.add(i).cast::<usize>().read_unaligned());
i += WORD_SIZE;
}
while i < len {
dest.add(i).write(src.add(i).read());
i += 1;
}
dest
}
dest
}
/// Memset
@@ -99,26 +103,28 @@ pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, len: usize) -> *
///
/// This faster implementation works by setting bytes not one-by-one, but in
/// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures).
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn memset(dest: *mut u8, byte: i32, len: usize) -> *mut u8 {
let byte = byte as u8;
unsafe {
let byte = byte as u8;
let mut i = 0;
let mut i = 0;
let broadcasted = usize::from_ne_bytes([byte; WORD_SIZE]);
let chunks = len / WORD_SIZE;
let broadcasted = usize::from_ne_bytes([byte; WORD_SIZE]);
let chunks = len / WORD_SIZE;
while i < chunks * WORD_SIZE {
dest.add(i).cast::<usize>().write_unaligned(broadcasted);
i += WORD_SIZE;
while i < chunks * WORD_SIZE {
dest.add(i).cast::<usize>().write_unaligned(broadcasted);
i += WORD_SIZE;
}
while i < len {
dest.add(i).write(byte);
i += 1;
}
dest
}
while i < len {
dest.add(i).write(byte);
i += 1;
}
dest
}
/// Memcmp
@@ -127,38 +133,40 @@ pub unsafe extern "C" fn memset(dest: *mut u8, byte: i32, len: usize) -> *mut u8
///
/// This faster implementation works by comparing bytes not one-by-one, but in
/// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures).
#[no_mangle]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, len: usize) -> i32 {
let mut i = 0_usize;
unsafe {
let mut i = 0_usize;
// First compare WORD_SIZE chunks...
let chunks = len / WORD_SIZE;
// First compare WORD_SIZE chunks...
let chunks = len / WORD_SIZE;
while i < chunks * WORD_SIZE {
let a = s1.add(i).cast::<usize>().read_unaligned();
let b = s2.add(i).cast::<usize>().read_unaligned();
while i < chunks * WORD_SIZE {
let a = s1.add(i).cast::<usize>().read_unaligned();
let b = s2.add(i).cast::<usize>().read_unaligned();
if a != b {
// x86 has had bswap since the 80486, and the compiler will likely use the faster
// movbe. AArch64 has the REV instruction, which I think is universally available.
let diff = usize::from_be(a).wrapping_sub(usize::from_be(b)) as isize;
if a != b {
// x86 has had bswap since the 80486, and the compiler will likely use the faster
// movbe. AArch64 has the REV instruction, which I think is universally available.
let diff = usize::from_be(a).wrapping_sub(usize::from_be(b)) as isize;
// TODO: If chunk size == 32 bits, diff can be returned directly.
return diff.signum() as i32;
// TODO: If chunk size == 32 bits, diff can be returned directly.
return diff.signum() as i32;
}
i += WORD_SIZE;
}
i += WORD_SIZE;
}
// ... and then compare bytes.
while i < len {
let a = s1.add(i).read();
let b = s2.add(i).read();
// ... and then compare bytes.
while i < len {
let a = s1.add(i).read();
let b = s2.add(i).read();
if a != b {
return i32::from(a) - i32::from(b);
if a != b {
return i32::from(a) - i32::from(b);
}
i += 1;
}
i += 1;
}
0
0
}
}
src/main.rs
+1 -1
View File
@@ -274,7 +274,7 @@ macro_rules! linker_offsets(
$(
#[inline]
pub fn $name() -> usize {
extern "C" {
unsafe extern "C" {
// TODO: UnsafeCell?
static $name: u8;
}
src/memory/mod.rs
+24 -20
View File
@@ -230,29 +230,31 @@ pub unsafe fn deallocate_p2frame(orig_frame: Frame, order: u32) {
}
pub unsafe fn deallocate_frame(frame: Frame) {
deallocate_p2frame(frame, 0)
unsafe { deallocate_p2frame(frame, 0) }
}
// Helper function for quickly mapping device memory
pub unsafe fn map_device_memory(addr: PhysicalAddress, len: usize) -> VirtualAddress {
let mut mapper_lock = KernelMapper::lock();
let mapper = mapper_lock
.get_mut()
.expect("KernelMapper mapper locked re-entrant in map_device_memory");
let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data()));
let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data()));
for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE {
let (_, flush) = mapper
.map_linearly(
base.add(page_idx * crate::memory::PAGE_SIZE),
PageFlags::new()
.write(true)
.custom_flag(EntryFlags::NO_CACHE.bits(), true),
)
.expect("failed to linearly map SDT");
flush.flush();
unsafe {
let mut mapper_lock = KernelMapper::lock();
let mapper = mapper_lock
.get_mut()
.expect("KernelMapper mapper locked re-entrant in map_device_memory");
let base = PhysicalAddress::new(crate::paging::round_down_pages(addr.data()));
let aligned_len = crate::paging::round_up_pages(len + (addr.data() - base.data()));
for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE {
let (_, flush) = mapper
.map_linearly(
base.add(page_idx * crate::memory::PAGE_SIZE),
PageFlags::new()
.write(true)
.custom_flag(EntryFlags::NO_CACHE.bits(), true),
)
.expect("failed to linearly map SDT");
flush.flush();
}
RmmA::phys_to_virt(addr)
}
RmmA::phys_to_virt(addr)
}
const ORDER_COUNT: u32 = 11;
@@ -1063,8 +1065,10 @@ impl FrameAllocator for TheFrameAllocator {
allocate_p2frame(order).map(|f| f.base())
}
unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) {
let order = count.data().next_power_of_two().trailing_zeros();
deallocate_p2frame(Frame::containing(address), order)
unsafe {
let order = count.data().next_power_of_two().trailing_zeros();
deallocate_p2frame(Frame::containing(address), order)
}
}
unsafe fn usage(&self) -> FrameUsage {
FrameUsage::new(
src/panic.rs
+61 -57
View File
@@ -59,37 +59,39 @@ fn rust_begin_unwind(info: &PanicInfo) -> ! {
/// Get a stack trace
#[inline(never)]
pub unsafe fn stack_trace() {
let mapper = KernelMapper::lock();
unsafe {
let mapper = KernelMapper::lock();
let mut frame = StackTrace::start();
let mut frame = StackTrace::start();
//Maximum 64 frames
for _ in 0..64 {
if let Some(frame_) = frame {
let fp_virt = VirtualAddress::new(frame_.fp);
let pc_virt = VirtualAddress::new(frame_.pc_ptr as usize);
if fp_virt.data() >= USER_END_OFFSET
&& pc_virt.data() >= USER_END_OFFSET
&& (fp_virt.data() as *const usize).is_aligned()
&& (pc_virt.data() as *const usize).is_aligned()
&& mapper.translate(fp_virt).is_some()
&& mapper.translate(pc_virt).is_some()
{
let pc = *frame_.pc_ptr;
if pc == 0 {
println!(" {:>016x}: EMPTY RETURN", frame_.fp);
break;
//Maximum 64 frames
for _ in 0..64 {
if let Some(frame_) = frame {
let fp_virt = VirtualAddress::new(frame_.fp);
let pc_virt = VirtualAddress::new(frame_.pc_ptr as usize);
if fp_virt.data() >= USER_END_OFFSET
&& pc_virt.data() >= USER_END_OFFSET
&& (fp_virt.data() as *const usize).is_aligned()
&& (pc_virt.data() as *const usize).is_aligned()
&& mapper.translate(fp_virt).is_some()
&& mapper.translate(pc_virt).is_some()
{
let pc = *frame_.pc_ptr;
if pc == 0 {
println!(" {:>016x}: EMPTY RETURN", frame_.fp);
break;
} else {
println!(" FP {:>016x}: PC {:>016x}", frame_.fp, pc);
symbol_trace(pc);
frame = frame_.next();
}
} else {
println!(" FP {:>016x}: PC {:>016x}", frame_.fp, pc);
symbol_trace(pc);
frame = frame_.next();
println!(" {:>016x}: GUARD PAGE", frame_.fp);
break;
}
} else {
println!(" {:>016x}: GUARD PAGE", frame_.fp);
break;
}
} else {
break;
}
}
}
@@ -98,45 +100,47 @@ pub unsafe fn stack_trace() {
//TODO: Do not create Elf object for every symbol lookup
#[inline(never)]
pub unsafe fn symbol_trace(addr: usize) {
let kernel_ptr = crate::KERNEL_OFFSET as *const u8;
let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst));
unsafe {
let kernel_ptr = crate::KERNEL_OFFSET as *const u8;
let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst));
if let Ok(elf) = Elf::from(kernel_slice) {
let mut strtab_opt = None;
for section in elf.sections() {
if section.sh_type == ::goblin::elf::section_header::SHT_STRTAB {
strtab_opt = Some(section);
break;
if let Ok(elf) = Elf::from(kernel_slice) {
let mut strtab_opt = None;
for section in elf.sections() {
if section.sh_type == ::goblin::elf::section_header::SHT_STRTAB {
strtab_opt = Some(section);
break;
}
}
}
if let Some(symbols) = elf.symbols() {
for sym in symbols {
if sym::st_type(sym.st_info) == sym::STT_FUNC
&& addr >= sym.st_value as usize
&& addr < (sym.st_value + sym.st_size) as usize
{
println!(
" {:>016X}+{:>04X}",
sym.st_value,
addr - sym.st_value as usize
);
if let Some(symbols) = elf.symbols() {
for sym in symbols {
if sym::st_type(sym.st_info) == sym::STT_FUNC
&& addr >= sym.st_value as usize
&& addr < (sym.st_value + sym.st_size) as usize
{
println!(
" {:>016X}+{:>04X}",
sym.st_value,
addr - sym.st_value as usize
);
if let Some(strtab) = strtab_opt {
let start = strtab.sh_offset as usize + sym.st_name as usize;
let mut end = start;
while end < elf.data.len() {
let b = elf.data[end];
end += 1;
if b == 0 {
break;
if let Some(strtab) = strtab_opt {
let start = strtab.sh_offset as usize + sym.st_name as usize;
let mut end = start;
while end < elf.data.len() {
let b = elf.data[end];
end += 1;
if b == 0 {
break;
}
}
}
if end > start {
let sym_slice = &elf.data[start..end - 1];
if let Ok(sym_name) = str::from_utf8(sym_slice) {
println!(" {:#}", demangle(sym_name));
if end > start {
let sym_slice = &elf.data[start..end - 1];
if let Ok(sym_name) = str::from_utf8(sym_slice) {
println!(" {:#}", demangle(sym_name));
}
}
}
}
src/percpu.rs
+35 -29
View File
@@ -11,7 +11,8 @@ use crate::{
context::{empty_cr3, memory::AddrSpaceWrapper, switch::ContextSwitchPercpu},
cpu_set::{LogicalCpuId, MAX_CPU_COUNT},
cpu_stats::CpuStats,
ptrace::Session,syscall::debug::SyscallDebugInfo
ptrace::Session,
syscall::debug::SyscallDebugInfo,
};
#[cfg(feature = "sys_stat")]
@@ -119,45 +120,50 @@ impl PercpuBlock {
crate::paging::RmmA::invalidate_all();
}
if let Some(ref addrsp) = &*self.current_addrsp.borrow() {
if let &Some(ref addrsp) = &*self.current_addrsp.borrow() {
addrsp.tlb_ack.fetch_add(1, Ordering::Release);
}
}
}
pub unsafe fn switch_arch_hook() {
let percpu = PercpuBlock::current();
unsafe {
let percpu = PercpuBlock::current();
let cur_addrsp = percpu.current_addrsp.borrow();
let next_addrsp = percpu.new_addrsp_tmp.take();
let cur_addrsp = percpu.current_addrsp.borrow();
let next_addrsp = percpu.new_addrsp_tmp.take();
let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) {
(Some(ref p), Some(ref n)) => Arc::ptr_eq(p, n),
(Some(_), None) | (None, Some(_)) => false,
(None, None) => true,
};
if retain_pgtbl {
// If we are not switching to a different address space, we can simply return early.
}
if let Some(ref prev_addrsp) = &*cur_addrsp {
prev_addrsp
.acquire_read()
.used_by
.atomic_clear(percpu.cpu_id);
}
let retain_pgtbl = match (&*cur_addrsp, &next_addrsp) {
(&Some(ref p), &Some(ref n)) => Arc::ptr_eq(p, n),
(Some(_), None) | (None, Some(_)) => false,
(None, None) => true,
};
if retain_pgtbl {
// If we are not switching to a different address space, we can simply return early.
}
if let &Some(ref prev_addrsp) = &*cur_addrsp {
prev_addrsp
.acquire_read()
.used_by
.atomic_clear(percpu.cpu_id);
}
drop(cur_addrsp);
drop(cur_addrsp);
// Tell future TLB shootdown handlers that old_addrsp_tmp is no longer the current address
// space.
*percpu.current_addrsp.borrow_mut() = next_addrsp;
// Tell future TLB shootdown handlers that old_addrsp_tmp is no longer the current address
// space.
*percpu.current_addrsp.borrow_mut() = next_addrsp;
if let Some(next_addrsp) = &*percpu.current_addrsp.borrow() {
let next = next_addrsp.acquire_read();
match &*percpu.current_addrsp.borrow() {
Some(next_addrsp) => {
let next = next_addrsp.acquire_read();
next.used_by.atomic_set(percpu.cpu_id);
next.table.utable.make_current();
} else {
crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3());
next.used_by.atomic_set(percpu.cpu_id);
next.table.utable.make_current();
}
_ => {
crate::paging::RmmA::set_table(rmm::TableKind::User, empty_cr3());
}
}
}
}
impl PercpuBlock {
src/scheme/irq.rs
+30 -28
View File
@@ -164,35 +164,37 @@ impl IrqScheme {
phandle: usize,
path_str: &str,
) -> Result<(Handle, InternalFlags)> {
let addr: Vec<u32> = path_str
.split(',')
.map(|x| u32::from_str(x).or(Err(Error::new(ENOENT))))
.try_collect()?;
let ic_idx = IRQ_CHIP
.phandle_to_ic_idx(phandle as u32)
.ok_or(Error::new(ENOENT))?;
Ok({
if flags & O_CREAT == 0 && flags & O_STAT == 0 {
return Err(Error::new(EINVAL));
}
let irq_number = IRQ_CHIP
.irq_xlate(ic_idx, addr.as_slice())
.or(Err(Error::new(ENOENT)))?;
log::debug!("open_phandle_irq virq={}", irq_number);
if flags & O_STAT == 0 {
if is_reserved(LogicalCpuId::new(0), irq_number as u8) {
return Err(Error::new(EEXIST));
unsafe {
let addr: Vec<u32> = path_str
.split(',')
.map(|x| u32::from_str(x).or(Err(Error::new(ENOENT))))
.try_collect()?;
let ic_idx = IRQ_CHIP
.phandle_to_ic_idx(phandle as u32)
.ok_or(Error::new(ENOENT))?;
Ok({
if flags & O_CREAT == 0 && flags & O_STAT == 0 {
return Err(Error::new(EINVAL));
}
set_reserved(LogicalCpuId::new(0), irq_number as u8, true);
}
(
Handle::Irq {
ack: AtomicUsize::new(0),
irq: irq_number as u8,
},
InternalFlags::empty(),
)
})
let irq_number = IRQ_CHIP
.irq_xlate(ic_idx, addr.as_slice())
.or(Err(Error::new(ENOENT)))?;
log::debug!("open_phandle_irq virq={}", irq_number);
if flags & O_STAT == 0 {
if is_reserved(LogicalCpuId::new(0), irq_number as u8) {
return Err(Error::new(EEXIST));
}
set_reserved(LogicalCpuId::new(0), irq_number as u8, true);
}
(
Handle::Irq {
ack: AtomicUsize::new(0),
irq: irq_number as u8,
},
InternalFlags::empty(),
)
})
}
}
}
src/scheme/mod.rs
+21 -15
View File
@@ -247,15 +247,18 @@ impl SchemeList {
return Err(Error::new(ENODEV));
};
if let Some(ref mut names) = self.names.get_mut(&to) {
if names
.insert(name.to_string().into_boxed_str(), id)
.is_some()
{
return Err(Error::new(EEXIST));
match self.names.get_mut(&to) {
Some(ref mut names) => {
if names
.insert(name.to_string().into_boxed_str(), id)
.is_some()
{
return Err(Error::new(EEXIST));
}
}
_ => {
panic!("scheme namespace not found");
}
} else {
panic!("scheme namespace not found");
}
}
@@ -344,13 +347,16 @@ impl SchemeList {
let (new_scheme, t) = scheme_fn(id);
assert!(self.map.insert(id, new_scheme).is_none());
if let Some(ref mut names) = self.names.get_mut(&ns) {
assert!(names
.insert(name.to_string().into_boxed_str(), id)
.is_none());
} else {
// Nonexistent namespace, posssibly null namespace
return Err(Error::new(ENODEV));
match self.names.get_mut(&ns) {
Some(ref mut names) => {
assert!(names
.insert(name.to_string().into_boxed_str(), id)
.is_none());
}
_ => {
// Nonexistent namespace, posssibly null namespace
return Err(Error::new(ENODEV));
}
}
Ok((id, t))
}
src/scheme/proc.rs
+8 -9
View File
@@ -259,12 +259,11 @@ impl ProcScheme {
let operation_name = operation_str.ok_or(Error::new(EINVAL))?;
let (mut handle, positioned) = match ty {
OpenTy::Ctxt(context) => {
if let Some((kind, positioned)) =
self.openat_context(operation_name, Arc::clone(&context))?
{
(Handle { context, kind }, positioned)
} else {
return Err(Error::new(EINVAL));
match self.openat_context(operation_name, Arc::clone(&context))? {
Some((kind, positioned)) => (Handle { context, kind }, positioned),
_ => {
return Err(Error::new(EINVAL));
}
}
}
OpenTy::Auth => {
@@ -1006,7 +1005,7 @@ impl ContextHandle {
verify_scheme(hopefully_this_scheme)?;
let mut handles = HANDLES.write();
let Handle {
let &Handle {
kind: ContextHandle::AddrSpace { ref addrspace },
..
} = handles.get(&number).ok_or(Error::new(EBADF))?
@@ -1201,7 +1200,7 @@ impl ContextHandle {
buf.copy_common_bytes_from_slice(src_buf)
}
ContextHandle::AddrSpace { ref addrspace } => {
&ContextHandle::AddrSpace { ref addrspace } => {
let Ok(offset) = usize::try_from(offset) else {
return Ok(0);
};
@@ -1239,7 +1238,7 @@ impl ContextHandle {
}
ContextHandle::Filetable { data, .. } => read_from(buf, &data, offset),
ContextHandle::MmapMinAddr(ref addrspace) => {
&ContextHandle::MmapMinAddr(ref addrspace) => {
buf.write_usize(addrspace.acquire_read().mmap_min)?;
Ok(mem::size_of::<usize>())
}
src/scheme/sys/context.rs
+15 -11
View File
@@ -21,14 +21,15 @@ pub fn resource() -> Result<Vec<u8>> {
let mut stat_string = String::new();
// TODO: All user programs must have some grant in order for executable memory to even
// exist, but is this a good indicator of whether it is user or kernel?
stat_string.push(if let Ok(addr_space) = context.addr_space() {
if addr_space.acquire_read().grants.is_empty() {
'K'
} else {
'U'
stat_string.push(match context.addr_space() {
Ok(addr_space) => {
if addr_space.acquire_read().grants.is_empty() {
'K'
} else {
'U'
}
}
} else {
'R'
_ => 'R',
});
match context.status {
context::Status::Runnable => {
@@ -49,10 +50,13 @@ pub fn resource() -> Result<Vec<u8>> {
stat_string.push('+');
}
let cpu_string = if let Some(cpu_id) = context.cpu_id {
format!("{}", cpu_id)
} else {
format!("?")
let cpu_string = match context.cpu_id {
Some(cpu_id) => {
format!("{}", cpu_id)
}
_ => {
format!("?")
}
};
let affinity = context.sched_affinity.to_string();
src/scheme/sys/mod.rs
+1 -1
View File
@@ -190,7 +190,7 @@ impl KernelScheme for SysScheme {
Handle::TopLevel | Handle::Resource { data: None, .. } => {
return Err(Error::new(EISDIR))
}
Handle::Resource {
&Handle::Resource {
data: Some(ref data),
..
} => {
src/scheme/user.rs
+11 -8
View File
@@ -955,7 +955,7 @@ impl UserInner {
.get_mut(tag as usize)
.ok_or(Error::new(EINVAL))?
{
State::Waiting { ref mut fds, .. } => {
&mut State::Waiting { ref mut fds, .. } => {
fds.take().ok_or(Error::new(ENOENT))?.remove(0)
}
_ => return Err(Error::new(ENOENT)),
@@ -1105,11 +1105,14 @@ impl UserInner {
.map(RwLock::into_inner)
.collect();
if let Some(context) = context.upgrade() {
context.write().unblock();
*o = State::Responded(response);
} else {
states.remove(tag as usize);
match context.upgrade() {
Some(context) => {
context.write().unblock();
*o = State::Responded(response);
}
_ => {
states.remove(tag as usize);
}
}
let unpin = true;
@@ -1333,7 +1336,7 @@ impl UserInner {
.get_mut(request_id)
.ok_or(Error::new(EINVAL))?
{
State::Waiting { ref mut fds, .. } => *fds = Some(descs),
&mut State::Waiting { ref mut fds, .. } => *fds = Some(descs),
_ => return Err(Error::new(ENOENT)),
};
@@ -1389,7 +1392,7 @@ impl UserInner {
.get_mut(request_id)
.ok_or(Error::new(EINVAL))?
{
State::Waiting { ref mut fds, .. } => fds.take().ok_or(Error::new(ENOENT))?,
&mut State::Waiting { ref mut fds, .. } => fds.take().ok_or(Error::new(ENOENT))?,
_ => return Err(Error::new(ENOENT)),
};
src/startup/memory.rs
+244 -238
View File
@@ -168,275 +168,281 @@ pub fn register_bootloader_areas(areas_base: usize, areas_size: usize) {
}
unsafe fn add_memory(areas: &mut [MemoryArea], area_i: &mut usize, mut area: MemoryEntry) {
for reservation in (*MEMORY_MAP.get()).non_free() {
if area.end > reservation.start && area.end <= reservation.end {
log::info!(
"Memory {:X}:{:X} overlaps with reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
area.end = reservation.start;
}
if area.start >= area.end {
return;
}
unsafe {
for reservation in (*MEMORY_MAP.get()).non_free() {
if area.end > reservation.start && area.end <= reservation.end {
log::info!(
"Memory {:X}:{:X} overlaps with reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
area.end = reservation.start;
}
if area.start >= area.end {
return;
}
if area.start >= reservation.start && area.start < reservation.end {
log::info!(
"Memory {:X}:{:X} overlaps with reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
area.start = reservation.end;
}
if area.start >= area.end {
return;
}
if area.start >= reservation.start && area.start < reservation.end {
log::info!(
"Memory {:X}:{:X} overlaps with reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
area.start = reservation.end;
}
if area.start >= area.end {
return;
}
if area.start <= reservation.start && area.end > reservation.start {
log::info!(
"Memory {:X}:{:X} contains reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
debug_assert!(area.start < reservation.start && reservation.end < area.end,
if area.start <= reservation.start && area.end > reservation.start {
log::info!(
"Memory {:X}:{:X} contains reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
debug_assert!(area.start < reservation.start && reservation.end < area.end,
"Should've contained reservation entirely: memory block {:X}:{:X} reservation {:X}:{:X}",
area.start, area.end,
reservation.start, reservation.end
);
// recurse on first part of split memory block
// recurse on first part of split memory block
add_memory(
areas,
area_i,
MemoryEntry {
end: reservation.start,
..area
},
);
add_memory(
areas,
area_i,
MemoryEntry {
end: reservation.start,
..area
},
);
// and continue with the second part
area.start = reservation.end;
}
debug_assert!(
area.intersect(reservation).is_none(),
"Intersects with reservation! memory block {:X}:{:X} reservation {:X}:{:X}",
area.start,
area.end,
reservation.start,
reservation.end
);
debug_assert!(
area.start < area.end,
"Empty memory block {:X}:{:X}",
area.start,
area.end
);
}
// Combine overlapping memory areas
let mut other_i = 0;
while other_i < *area_i {
let other = &areas[other_i];
let other = MemoryEntry {
start: other.base.data(),
end: other.base.data().saturating_add(other.size),
kind: BootloaderMemoryKind::Free,
};
if let Some(union) = area.combine(&other) {
log::debug!(
"{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}",
// and continue with the second part
area.start = reservation.end;
}
debug_assert!(
area.intersect(reservation).is_none(),
"Intersects with reservation! memory block {:X}:{:X} reservation {:X}:{:X}",
area.start,
area.end,
other.start,
other.end,
union.start,
union.end
reservation.start,
reservation.end
);
debug_assert!(
area.start < area.end,
"Empty memory block {:X}:{:X}",
area.start,
area.end
);
area = union;
*area_i -= 1; // delete the original memory chunk
areas[other_i] = areas[*area_i];
} else {
other_i += 1;
}
}
areas[*area_i].base = PhysicalAddress::new(area.start);
areas[*area_i].size = area.end - area.start;
*area_i += 1;
// Combine overlapping memory areas
let mut other_i = 0;
while other_i < *area_i {
let other = &areas[other_i];
let other = MemoryEntry {
start: other.base.data(),
end: other.base.data().saturating_add(other.size),
kind: BootloaderMemoryKind::Free,
};
if let Some(union) = area.combine(&other) {
log::debug!(
"{:X}:{:X} overlaps with area {:X}:{:X}, combining into {:X}:{:X}",
area.start,
area.end,
other.start,
other.end,
union.start,
union.end
);
area = union;
*area_i -= 1; // delete the original memory chunk
areas[other_i] = areas[*area_i];
} else {
other_i += 1;
}
}
areas[*area_i].base = PhysicalAddress::new(area.start);
areas[*area_i].size = area.end - area.start;
*area_i += 1;
}
}
unsafe fn map_memory<A: Arch>(areas: &[MemoryArea], mut bump_allocator: &mut BumpAllocator<A>) {
let mut mapper = PageMapper::<A, _>::create(TableKind::Kernel, &mut bump_allocator)
.expect("failed to create Mapper");
unsafe {
let mut mapper = PageMapper::<A, _>::create(TableKind::Kernel, &mut bump_allocator)
.expect("failed to create Mapper");
if cfg!(target_arch = "x86") {
// Pre-allocate all kernel PD entries so that when the page table is copied,
// these entries are synced between processes
for i in 512..1024 {
use rmm::{FrameAllocator, PageEntry};
if cfg!(target_arch = "x86") {
// Pre-allocate all kernel PD entries so that when the page table is copied,
// these entries are synced between processes
for i in 512..1024 {
use rmm::{FrameAllocator, PageEntry};
let phys = mapper
.allocator_mut()
.allocate_one()
.expect("failed to map page table");
let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE;
mapper
.table()
.set_entry(i, PageEntry::new(phys.data(), flags));
}
}
// Map all physical areas at PHYS_OFFSET
for area in areas.iter() {
for i in 0..area.size / PAGE_SIZE {
let phys = area.base.add(i * PAGE_SIZE);
let virt = A::phys_to_virt(phys);
let flags = page_flags::<A>(virt);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
let kernel_area = (*MEMORY_MAP.get()).kernel().unwrap();
let kernel_base = kernel_area.start;
let kernel_size = kernel_area.end - kernel_area.start;
// Map kernel at KERNEL_OFFSET and identity map too
for i in 0..kernel_size / A::PAGE_SIZE {
let phys = PhysicalAddress::new(kernel_base + i * PAGE_SIZE);
let virt = VirtualAddress::new(KERNEL_OFFSET + i * PAGE_SIZE);
let flags = page_flags::<A>(virt);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
let virt = A::phys_to_virt(phys);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
for area in (*MEMORY_MAP.get()).identity_mapped() {
let base = area.start;
let size = area.end - area.start;
for i in 0..size / PAGE_SIZE {
let phys = PhysicalAddress::new(base + i * PAGE_SIZE);
let virt = A::phys_to_virt(phys);
let flags = page_flags::<A>(virt);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
//map dev mem
for area in (*MEMORY_MAP.get()).devices() {
let base = area.start;
let size = area.end - area.start;
for i in 0..size / PAGE_SIZE {
let phys = PhysicalAddress::new(base + i * PAGE_SIZE);
let virt = A::phys_to_virt(phys);
// use the same mair_el1 value with bootloader,
// mair_el1 == 0x00000000000044FF
// set mem_attr == device memory
let flags = page_flags::<A>(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
// Ensure graphical debug region remains paged
#[cfg(feature = "graphical_debug")]
{
use crate::devices::graphical_debug::FRAMEBUFFER;
let (phys, virt, size) = *FRAMEBUFFER.lock();
let pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
for i in 0..pages {
let phys = PhysicalAddress::new(phys + i * PAGE_SIZE);
let virt = VirtualAddress::new(virt + i * PAGE_SIZE);
let flags = PageFlags::new().write(true).write_combining(true);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
log::debug!("Table: {:X}", mapper.table().phys().data());
for i in 0..A::PAGE_ENTRIES {
if let Some(entry) = mapper.table().entry(i) {
if entry.present() {
log::debug!("{}: {:X}", i, entry.data());
let phys = mapper
.allocator_mut()
.allocate_one()
.expect("failed to map page table");
let flags = A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE;
mapper
.table()
.set_entry(i, PageEntry::new(phys.data(), flags));
}
}
}
// Use the new table
mapper.make_current();
// Map all physical areas at PHYS_OFFSET
for area in areas.iter() {
for i in 0..area.size / PAGE_SIZE {
let phys = area.base.add(i * PAGE_SIZE);
let virt = A::phys_to_virt(phys);
let flags = page_flags::<A>(virt);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
let kernel_area = (*MEMORY_MAP.get()).kernel().unwrap();
let kernel_base = kernel_area.start;
let kernel_size = kernel_area.end - kernel_area.start;
// Map kernel at KERNEL_OFFSET and identity map too
for i in 0..kernel_size / A::PAGE_SIZE {
let phys = PhysicalAddress::new(kernel_base + i * PAGE_SIZE);
let virt = VirtualAddress::new(KERNEL_OFFSET + i * PAGE_SIZE);
let flags = page_flags::<A>(virt);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
let virt = A::phys_to_virt(phys);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
for area in (*MEMORY_MAP.get()).identity_mapped() {
let base = area.start;
let size = area.end - area.start;
for i in 0..size / PAGE_SIZE {
let phys = PhysicalAddress::new(base + i * PAGE_SIZE);
let virt = A::phys_to_virt(phys);
let flags = page_flags::<A>(virt);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
//map dev mem
for area in (*MEMORY_MAP.get()).devices() {
let base = area.start;
let size = area.end - area.start;
for i in 0..size / PAGE_SIZE {
let phys = PhysicalAddress::new(base + i * PAGE_SIZE);
let virt = A::phys_to_virt(phys);
// use the same mair_el1 value with bootloader,
// mair_el1 == 0x00000000000044FF
// set mem_attr == device memory
let flags = page_flags::<A>(virt).custom_flag(EntryFlags::DEV_MEM.bits(), true);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
// Ensure graphical debug region remains paged
#[cfg(feature = "graphical_debug")]
{
use crate::devices::graphical_debug::FRAMEBUFFER;
let (phys, virt, size) = *FRAMEBUFFER.lock();
let pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
for i in 0..pages {
let phys = PhysicalAddress::new(phys + i * PAGE_SIZE);
let virt = VirtualAddress::new(virt + i * PAGE_SIZE);
let flags = PageFlags::new().write(true).write_combining(true);
let flush = mapper
.map_phys(virt, phys, flags)
.expect("failed to map frame");
flush.ignore(); // Not the active table
}
}
log::debug!("Table: {:X}", mapper.table().phys().data());
for i in 0..A::PAGE_ENTRIES {
if let Some(entry) = mapper.table().entry(i) {
if entry.present() {
log::debug!("{}: {:X}", i, entry.data());
}
}
}
// Use the new table
mapper.make_current();
}
}
pub unsafe fn init(low_limit: Option<usize>, high_limit: Option<usize>) {
let physmem_limit = MemoryEntry {
start: align_up(low_limit.unwrap_or(0)),
end: align_down(high_limit.unwrap_or(usize::MAX)),
kind: BootloaderMemoryKind::Free,
};
unsafe {
let physmem_limit = MemoryEntry {
start: align_up(low_limit.unwrap_or(0)),
end: align_down(high_limit.unwrap_or(usize::MAX)),
kind: BootloaderMemoryKind::Free,
};
let areas = &mut *crate::memory::AREAS.get();
let mut area_i = 0;
let areas = &mut *crate::memory::AREAS.get();
let mut area_i = 0;
// Copy initial memory map, and page align it
for area in (*MEMORY_MAP.get()).free() {
log::debug!("{:X}:{:X}", area.start, area.end);
// Copy initial memory map, and page align it
for area in (*MEMORY_MAP.get()).free() {
log::debug!("{:X}:{:X}", area.start, area.end);
if let Some(area) = area.intersect(&physmem_limit) {
add_memory(areas, &mut area_i, area);
if let Some(area) = area.intersect(&physmem_limit) {
add_memory(areas, &mut area_i, area);
}
}
}
areas[..area_i].sort_unstable_by_key(|area| area.base);
crate::memory::AREA_COUNT.get().write(area_i as u16);
areas[..area_i].sort_unstable_by_key(|area| area.base);
crate::memory::AREA_COUNT.get().write(area_i as u16);
// free memory map in now ready
let areas = crate::memory::areas();
// free memory map in now ready
let areas = crate::memory::areas();
// First, calculate how much memory we have
let mut size = 0;
for area in areas.iter() {
if area.size > 0 {
log::debug!("{:X?}", area);
size += area.size;
// First, calculate how much memory we have
let mut size = 0;
for area in areas.iter() {
if area.size > 0 {
log::debug!("{:X?}", area);
size += area.size;
}
}
log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE);
// Create a basic allocator for the first pages
let mut bump_allocator = BumpAllocator::<CurrentRmmArch>::new(areas, 0);
map_memory(areas, &mut bump_allocator);
// Create the physical memory map
let offset = bump_allocator.offset();
log::info!(
"Permanently used: {} KB",
(offset + (KILOBYTE - 1)) / KILOBYTE
);
crate::memory::init_mm(bump_allocator);
}
log::info!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE);
// Create a basic allocator for the first pages
let mut bump_allocator = BumpAllocator::<CurrentRmmArch>::new(areas, 0);
map_memory(areas, &mut bump_allocator);
// Create the physical memory map
let offset = bump_allocator.offset();
log::info!(
"Permanently used: {} KB",
(offset + (KILOBYTE - 1)) / KILOBYTE
);
crate::memory::init_mm(bump_allocator);
}
src/sync/wait_queue.rs
+13 -8
View File
@@ -31,15 +31,20 @@ impl<T> WaitQueue<T> {
loop {
let mut inner = self.inner.lock();
if let Some(t) = inner.pop_front() {
return Ok(t);
} else if block {
if !self.condition.wait(inner, reason) {
return Err(Error::new(EINTR));
match inner.pop_front() {
Some(t) => {
return Ok(t);
}
_ => {
if block {
if !self.condition.wait(inner, reason) {
return Err(Error::new(EINTR));
}
continue;
} else {
return Err(Error::new(EAGAIN));
}
}
continue;
} else {
return Err(Error::new(EAGAIN));
}
}
}
src/syscall/debug.rs
+1 -1
View File
@@ -38,7 +38,7 @@ fn debug_buf(ptr: usize, len: usize) -> Result<Vec<u8>> {
})
}
unsafe fn read_struct<T>(ptr: usize) -> Result<T> {
UserSlice::ro(ptr, mem::size_of::<T>()).and_then(|slice| slice.read_exact::<T>())
unsafe { UserSlice::ro(ptr, mem::size_of::<T>()).and_then(|slice| slice.read_exact::<T>()) }
}
//TODO: calling format_call with arguments from another process space will not work
src/syscall/process.rs
+6 -4
View File
@@ -138,8 +138,10 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) {
}
pub unsafe fn bootstrap_mem(bootstrap: &crate::Bootstrap) -> &'static [u8] {
core::slice::from_raw_parts(
CurrentRmmArch::phys_to_virt(bootstrap.base.base()).data() as *const u8,
bootstrap.page_count * PAGE_SIZE,
)
unsafe {
core::slice::from_raw_parts(
CurrentRmmArch::phys_to_virt(bootstrap.base.base()).data() as *const u8,
bootstrap.page_count * PAGE_SIZE,
)
}
}
src/syscall/usercopy.rs
+13 -11
View File
@@ -108,19 +108,21 @@ impl<const WRITE: bool> UserSlice<true, WRITE> {
}
}
pub unsafe fn read_exact<T>(self) -> Result<T> {
let mut t: T = core::mem::zeroed();
let slice = unsafe {
core::slice::from_raw_parts_mut(
(&mut t as *mut T).cast::<u8>(),
core::mem::size_of::<T>(),
)
};
unsafe {
let mut t: T = core::mem::zeroed();
let slice = unsafe {
core::slice::from_raw_parts_mut(
(&mut t as *mut T).cast::<u8>(),
core::mem::size_of::<T>(),
)
};
self.limit(core::mem::size_of::<T>())
.ok_or(Error::new(EINVAL))?
.copy_to_slice(slice)?;
self.limit(core::mem::size_of::<T>())
.ok_or(Error::new(EINVAL))?
.copy_to_slice(slice)?;
Ok(t)
Ok(t)
}
}
pub fn copy_common_bytes_to_slice(self, slice: &mut [u8]) -> Result<usize> {
let min = core::cmp::min(self.len(), slice.len());