Red Bear OS kernel baseline

From release 0.1.0 pre-patched archive.
This includes all Red Bear modifications previously maintained
as patches in local/patches/kernel/.

src/acpi/gtdt.rs

@@ -0,0 +1,64 @@
+use alloc::boxed::Box;
+
+use super::{find_sdt, sdt::Sdt};
+use crate::{
+    arch::device::generic_timer::GenericTimer,
+    dtb::irqchip::{register_irq, IRQ_CHIP},
+};
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct Gtdt {
+    pub header: Sdt,
+    pub cnt_control_base: u64,
+    _reserved: u32,
+    pub secure_el1_timer_gsiv: u32,
+    pub secure_el1_timer_flags: u32,
+    pub non_secure_el1_timer_gsiv: u32,
+    pub non_secure_el1_timer_flags: u32,
+    pub virtual_el1_timer_gsiv: u32,
+    pub virtual_el1_timer_flags: u32,
+    pub el2_timer_gsiv: u32,
+    pub el2_timer_flags: u32,
+    pub cnt_read_base: u64,
+    pub platform_timer_count: u32,
+    pub platform_timer_offset: u32,
+    /*TODO: we don't need these yet, and they cause short tables to fail parsing
+    pub virtual_el2_timer_gsiv: u32,
+    pub virtual_el2_timer_flags: u32,
+    */
+    //TODO: platform timer structure (at platform timer offset, with platform timer count)
+}
+
+impl Gtdt {
+    pub fn init() {
+        let gtdt_sdt = find_sdt("GTDT");
+        let gtdt = if gtdt_sdt.len() == 1 {
+            match Gtdt::new(gtdt_sdt[0]) {
+                Some(gtdt) => gtdt,
+                None => {
+                    warn!("Failed to parse GTDT");
+                    return;
+                }
+            }
+        } else {
+            warn!("Unable to find GTDT");
+            return;
+        };
+
+        let gsiv = gtdt.non_secure_el1_timer_gsiv;
+        info!("generic_timer gsiv = {}", gsiv);
+        let mut timer = GenericTimer::new();
+        timer.init();
+        register_irq(gsiv, Box::new(timer));
+        unsafe { IRQ_CHIP.irq_enable(gsiv as u32) };
+    }
+
+    pub fn new(sdt: &'static Sdt) -> Option<&'static Gtdt> {
+        if &sdt.signature == b"GTDT" && sdt.length as usize >= size_of::<Gtdt>() {
+            Some(unsafe { &*((sdt as *const Sdt) as *const Gtdt) })
+        } else {
+            None
+        }
+    }
+}

src/acpi/hpet.rs

@@ -0,0 +1,121 @@
+use core::ptr::{self, read_volatile, write_volatile};
+
+#[cfg(not(target_arch = "x86"))]
+use crate::memory::{RmmA, RmmArch};
+use crate::{find_one_sdt, memory::PhysicalAddress};
+
+use super::{sdt::Sdt, GenericAddressStructure, ACPI_TABLE};
+
+#[repr(C, packed)]
+#[derive(Clone, Copy, Debug)]
+pub struct Hpet {
+    pub header: Sdt,
+
+    pub hw_rev_id: u8,
+    pub comparator_descriptor: u8,
+    pub pci_vendor_id: u16,
+
+    pub base_address: GenericAddressStructure,
+
+    pub hpet_number: u8,
+    pub min_periodic_clk_tick: u16,
+    pub oem_attribute: u8,
+}
+
+impl Hpet {
+    pub fn init() {
+        let hpet = Hpet::new(find_one_sdt!("HPET"));
+
+        if let Some(hpet) = hpet {
+            debug!("  HPET: {:X}", hpet.hpet_number);
+
+            let mut hpet_t = ACPI_TABLE.hpet.write();
+            *hpet_t = Some(hpet);
+        }
+    }
+
+    pub fn new(sdt: &'static Sdt) -> Option<Hpet> {
+        if &sdt.signature == b"HPET" && sdt.length as usize >= size_of::<Hpet>() {
+            let s = unsafe { ptr::read((sdt as *const Sdt) as *const Hpet) };
+            if s.base_address.address_space == 0 {
+                unsafe { s.map() };
+                Some(s)
+            } else {
+                warn!(
+                    "HPET has unsupported address space {}",
+                    s.base_address.address_space
+                );
+                None
+            }
+        } else {
+            None
+        }
+    }
+}
+
+//TODO: x86 use assumes only one HPET and only one GenericAddressStructure
+#[cfg(target_arch = "x86")]
+impl Hpet {
+    pub unsafe fn map(&self) {
+        unsafe {
+            use crate::memory::{Frame, KernelMapper, Page, PageFlags, VirtualAddress};
+
+            let frame = Frame::containing(PhysicalAddress::new(self.base_address.address as usize));
+            let page = Page::containing_address(VirtualAddress::new(crate::HPET_OFFSET));
+
+            KernelMapper::lock_rw()
+                .map_phys(
+                    page.start_address(),
+                    frame.base(),
+                    PageFlags::new().write(true).device_memory(true),
+                )
+                .expect("failed to map memory for GenericAddressStructure")
+                .flush();
+        }
+    }
+
+    pub unsafe fn read_u64(&self, offset: usize) -> u64 {
+        unsafe { read_volatile((crate::HPET_OFFSET + offset) as *const u64) }
+    }
+
+    pub unsafe fn write_u64(&mut self, offset: usize, value: u64) {
+        unsafe {
+            write_volatile((crate::HPET_OFFSET + offset) as *mut u64, value);
+        }
+    }
+}
+
+#[cfg(not(target_arch = "x86"))]
+impl Hpet {
+    pub unsafe fn map(&self) {
+        unsafe {
+            crate::memory::map_device_memory(
+                PhysicalAddress::new(self.base_address.address as usize),
+                crate::memory::PAGE_SIZE,
+            );
+        }
+    }
+
+    pub unsafe fn read_u64(&self, offset: usize) -> u64 {
+        unsafe {
+            read_volatile(
+                RmmA::phys_to_virt(PhysicalAddress::new(
+                    self.base_address.address as usize + offset,
+                ))
+                .data() as *const u64,
+            )
+        }
+    }
+
+    pub unsafe fn write_u64(&mut self, offset: usize, value: u64) {
+        unsafe {
+            write_volatile(
+                RmmA::phys_to_virt(PhysicalAddress::new(
+                    self.base_address.address as usize + offset,
+                ))
+                .data() as *mut u64,
+                value,
+            );
+        }
+    }
+}

src/acpi/madt/arch/aarch64.rs

@@ -0,0 +1,97 @@
+use alloc::{boxed::Box, vec::Vec};
+
+use super::{Madt, MadtEntry};
+use crate::{
+    arch::device::irqchip::{
+        gic::{GenericInterruptController, GicCpuIf, GicDistIf},
+        gicv3::{GicV3, GicV3CpuIf},
+    },
+    dtb::irqchip::{IrqChipItem, IRQ_CHIP},
+    memory::{map_device_memory, PhysicalAddress, PAGE_SIZE},
+};
+
+pub(super) fn init(madt: Madt) {
+    let mut gicd_opt = None;
+    let mut giccs = Vec::new();
+    for madt_entry in madt.iter() {
+        debug!("      {:#x?}", madt_entry);
+        match madt_entry {
+            MadtEntry::Gicc(gicc) => {
+                giccs.push(gicc);
+            }
+            MadtEntry::Gicd(gicd) => {
+                if gicd_opt.is_some() {
+                    warn!("Only one GICD should be present on a system, ignoring this one");
+                } else {
+                    gicd_opt = Some(gicd);
+                }
+            }
+            _ => {}
+        }
+    }
+    let Some(gicd) = gicd_opt else {
+        warn!("No GICD found");
+        return;
+    };
+    let mut gic_dist_if = GicDistIf::default();
+    unsafe {
+        let phys = PhysicalAddress::new(gicd.physical_base_address as usize);
+        let virt = map_device_memory(phys, PAGE_SIZE);
+        gic_dist_if.init(virt.data());
+    };
+    info!("{:#x?}", gic_dist_if);
+    match gicd.gic_version {
+        1 | 2 => {
+            for gicc in giccs {
+                let mut gic_cpu_if = GicCpuIf::default();
+                unsafe {
+                    let phys = PhysicalAddress::new(gicc.physical_base_address as usize);
+                    let virt = map_device_memory(phys, PAGE_SIZE);
+                    gic_cpu_if.init(virt.data())
+                };
+                info!("{:#x?}", gic_cpu_if);
+                let gic = GenericInterruptController {
+                    gic_dist_if,
+                    gic_cpu_if,
+                    irq_range: (0, 0),
+                };
+                let chip = IrqChipItem {
+                    phandle: 0,
+                    parents: Vec::new(),
+                    children: Vec::new(),
+                    ic: Box::new(gic),
+                };
+                unsafe { IRQ_CHIP.irq_chip_list.chips.push(chip) };
+                //TODO: support more GICCs
+                break;
+            }
+        }
+        3 => {
+            for gicc in giccs {
+                let mut gic_cpu_if = GicV3CpuIf;
+                unsafe { gic_cpu_if.init() };
+                info!("{:#x?}", gic_cpu_if);
+                let gic = GicV3 {
+                    gic_dist_if,
+                    gic_cpu_if,
+                    //TODO: get GICRs
+                    gicrs: Vec::new(),
+                    irq_range: (0, 0),
+                };
+                let chip = IrqChipItem {
+                    phandle: 0,
+                    parents: Vec::new(),
+                    children: Vec::new(),
+                    ic: Box::new(gic),
+                };
+                unsafe { IRQ_CHIP.irq_chip_list.chips.push(chip) };
+                //TODO: support more GICCs
+                break;
+            }
+        }
+        _ => {
+            warn!("unsupported GIC version {}", gicd.gic_version);
+        }
+    }
+    unsafe { IRQ_CHIP.init(None) };
+}

src/acpi/madt/arch/other.rs

@@ -0,0 +1,9 @@
+use super::Madt;
+
+pub(super) fn init(madt: Madt) {
+    for madt_entry in madt.iter() {
+        debug!("      {:#x?}", madt_entry);
+    }
+
+    warn!("MADT not yet handled on this platform");
+}

src/acpi/madt/arch/x86.rs

@@ -0,0 +1,160 @@
+use core::{
+    hint,
+    sync::atomic::{AtomicU8, Ordering},
+};
+
+use crate::{
+    arch::{
+        device::local_apic::the_local_apic,
+        start::{kstart_ap, KernelArgsAp},
+    },
+    cpu_set::LogicalCpuId,
+    memory::{
+        allocate_p2frame, Frame, KernelMapper, Page, PageFlags, PhysicalAddress, RmmA, RmmArch,
+        VirtualAddress, PAGE_SIZE,
+    },
+    startup::AP_READY,
+};
+
+use super::{Madt, MadtEntry};
+
+const TRAMPOLINE: usize = 0x8000;
+static TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trampoline"));
+
+pub(super) fn init(madt: Madt) {
+    let local_apic = unsafe { the_local_apic() };
+    let me = local_apic.id();
+
+    if local_apic.x2 {
+        debug!("    X2APIC {}", me.get());
+    } else {
+        debug!("    XAPIC {}: {:>08X}", me.get(), local_apic.address);
+    }
+
+    if cfg!(not(feature = "multi_core")) {
+        return;
+    }
+
+    // Map trampoline
+    let trampoline_frame = Frame::containing(PhysicalAddress::new(TRAMPOLINE));
+    let trampoline_page = Page::containing_address(VirtualAddress::new(TRAMPOLINE));
+    let (result, page_table_physaddr) = unsafe {
+        //TODO: do not have writable and executable!
+        let mut mapper = KernelMapper::lock_rw();
+
+        let result = mapper
+            .map_phys(
+                trampoline_page.start_address(),
+                trampoline_frame.base(),
+                PageFlags::new().execute(true).write(true),
+            )
+            .expect("failed to map trampoline");
+
+        (result, mapper.table().phys().data())
+    };
+    result.flush();
+
+    // Write trampoline, make sure TRAMPOLINE page is free for use
+    for (i, val) in TRAMPOLINE_DATA.iter().enumerate() {
+        unsafe {
+            (*((TRAMPOLINE as *mut u8).add(i) as *const AtomicU8)).store(*val, Ordering::SeqCst);
+        }
+    }
+
+    unsafe {
+        let preliminary_cpu_count = madt.iter().filter(|e| matches!(e, MadtEntry::LocalApic(entry) if u32::from(entry.id) == me.get() || entry.flags & 1 == 1)).count();
+        crate::profiling::allocate(preliminary_cpu_count as u32);
+    }
+
+    for madt_entry in madt.iter() {
+        debug!("      {:x?}", madt_entry);
+        if let MadtEntry::LocalApic(ap_local_apic) = madt_entry {
+            if u32::from(ap_local_apic.id) == me.get() {
+                debug!("        This is my local APIC");
+            } else if ap_local_apic.flags & 1 == 1 {
+                let cpu_id = LogicalCpuId::next();
+
+                // Allocate a stack
+                let stack_start = RmmA::phys_to_virt(
+                    allocate_p2frame(4)
+                        .expect("no more frames in acpi stack_start")
+                        .base(),
+                )
+                .data();
+                let stack_end = stack_start + (PAGE_SIZE << 4);
+
+                let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end);
+
+                let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id);
+
+                let args = KernelArgsAp {
+                    stack_end: stack_end as *mut u8,
+                    cpu_id,
+                    pcr_ptr,
+                    idt_ptr,
+                };
+
+                let ap_ready = (TRAMPOLINE + 8) as *mut u64;
+                let ap_args_ptr = unsafe { ap_ready.add(1) };
+                let ap_page_table = unsafe { ap_ready.add(2) };
+                let ap_code = unsafe { ap_ready.add(3) };
+
+                // Set the ap_ready to 0, volatile
+                unsafe {
+                    ap_ready.write(0);
+                    ap_args_ptr.write(&args as *const _ as u64);
+                    ap_page_table.write(page_table_physaddr as u64);
+                    #[expect(clippy::fn_to_numeric_cast)]
+                    ap_code.write(kstart_ap as u64);
+
+                    // TODO: Is this necessary (this fence)?
+                    core::arch::asm!("");
+                };
+                AP_READY.store(false, Ordering::SeqCst);
+
+                // Send INIT IPI
+                {
+                    let mut icr = 0x4500;
+                    if local_apic.x2 {
+                        icr |= u64::from(ap_local_apic.id) << 32;
+                    } else {
+                        icr |= u64::from(ap_local_apic.id) << 56;
+                    }
+                    local_apic.set_icr(icr);
+                }
+
+                // Send START IPI
+                {
+                    let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
+                    let mut icr = 0x4600 | ap_segment as u64;
+
+                    if local_apic.x2 {
+                        icr |= u64::from(ap_local_apic.id) << 32;
+                    } else {
+                        icr |= u64::from(ap_local_apic.id) << 56;
+                    }
+
+                    local_apic.set_icr(icr);
+                }
+
+                // Wait for trampoline ready
+                while unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } == 0 {
+                    hint::spin_loop();
+                }
+                while !AP_READY.load(Ordering::SeqCst) {
+                    hint::spin_loop();
+                }
+
+                RmmA::invalidate_all();
+            }
+        }
+    }
+
+    // Unmap trampoline
+    let (_frame, _, flush) = unsafe {
+        KernelMapper::lock_rw()
+            .unmap_phys(trampoline_page.start_address())
+            .expect("failed to unmap trampoline page")
+    };
+    flush.flush();
+}

src/acpi/madt/mod.rs

@@ -0,0 +1,240 @@
+use core::cell::SyncUnsafeCell;
+
+use super::sdt::Sdt;
+use crate::find_one_sdt;
+
+/// The Multiple APIC Descriptor Table
+#[derive(Clone, Copy, Debug)]
+pub struct Madt {
+    sdt: &'static Sdt,
+    pub local_address: u32,
+    pub flags: u32,
+}
+
+#[cfg(target_arch = "aarch64")]
+#[path = "arch/aarch64.rs"]
+mod arch;
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[path = "arch/x86.rs"]
+mod arch;
+
+#[cfg(not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")))]
+#[path = "arch/other.rs"]
+mod arch;
+
+static MADT: SyncUnsafeCell<Option<Madt>> = SyncUnsafeCell::new(None);
+pub fn madt() -> Option<&'static Madt> {
+    unsafe { &*MADT.get() }.as_ref()
+}
+pub const FLAG_PCAT: u32 = 1;
+
+impl Madt {
+    pub fn init() {
+        let madt = Madt::new(find_one_sdt!("APIC"));
+
+        if let Some(madt) = madt {
+            // safe because no APs have been started yet.
+            unsafe { MADT.get().write(Some(madt)) };
+
+            debug!("  APIC: {:>08X}: {}", madt.local_address, madt.flags);
+
+            arch::init(madt);
+        }
+    }
+
+    pub fn new(sdt: &'static Sdt) -> Option<Madt> {
+        if &sdt.signature == b"APIC" && sdt.data_len() >= 8 {
+            //Not valid if no local address and flags
+            let local_address = unsafe { (sdt.data_address() as *const u32).read_unaligned() };
+            let flags = unsafe {
+                (sdt.data_address() as *const u32)
+                    .offset(1)
+                    .read_unaligned()
+            };
+
+            Some(Madt {
+                sdt,
+                local_address,
+                flags,
+            })
+        } else {
+            None
+        }
+    }
+
+    pub fn iter(&self) -> MadtIter {
+        MadtIter {
+            sdt: self.sdt,
+            i: 8, // Skip local controller address and flags
+        }
+    }
+}
+
+/// MADT Local APIC
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct MadtLocalApic {
+    /// Processor ID
+    pub processor: u8,
+    /// Local APIC ID
+    pub id: u8,
+    /// Flags. 1 means that the processor is enabled
+    pub flags: u32,
+}
+
+/// MADT I/O APIC
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct MadtIoApic {
+    /// I/O APIC ID
+    pub id: u8,
+    /// reserved
+    _reserved: u8,
+    /// I/O APIC address
+    pub address: u32,
+    /// Global system interrupt base
+    pub gsi_base: u32,
+}
+
+/// MADT Interrupt Source Override
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct MadtIntSrcOverride {
+    /// Bus Source
+    pub bus_source: u8,
+    /// IRQ Source
+    pub irq_source: u8,
+    /// Global system interrupt base
+    pub gsi_base: u32,
+    /// Flags
+    pub flags: u16,
+}
+
+/// MADT GICC
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct MadtGicc {
+    _reserved: u16,
+    pub cpu_interface_number: u32,
+    pub acpi_processor_uid: u32,
+    pub flags: u32,
+    pub parking_protocol_version: u32,
+    pub performance_interrupt_gsiv: u32,
+    pub parked_address: u64,
+    pub physical_base_address: u64,
+    pub gicv: u64,
+    pub gich: u64,
+    pub vgic_maintenance_interrupt: u32,
+    pub gicr_base_address: u64,
+    pub mpidr: u64,
+    pub processor_power_efficiency_class: u8,
+    _reserved2: u8,
+    pub spe_overflow_interrupt: u16,
+    //TODO: optional field introduced in ACPI 6.5: pub trbe_interrupt: u16,
+}
+
+/// MADT GICD
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct MadtGicd {
+    _reserved: u16,
+    pub gic_id: u32,
+    pub physical_base_address: u64,
+    pub system_vector_base: u32,
+    pub gic_version: u8,
+    _reserved2: [u8; 3],
+}
+
+/// MADT Entries
+#[derive(Debug)]
+#[allow(dead_code)]
+pub enum MadtEntry {
+    LocalApic(&'static MadtLocalApic),
+    InvalidLocalApic(usize),
+    IoApic(&'static MadtIoApic),
+    InvalidIoApic(usize),
+    IntSrcOverride(&'static MadtIntSrcOverride),
+    InvalidIntSrcOverride(usize),
+    Gicc(&'static MadtGicc),
+    InvalidGicc(usize),
+    Gicd(&'static MadtGicd),
+    InvalidGicd(usize),
+    Unknown(u8),
+}
+
+pub struct MadtIter {
+    sdt: &'static Sdt,
+    i: usize,
+}
+
+impl Iterator for MadtIter {
+    type Item = MadtEntry;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.i + 1 < self.sdt.data_len() {
+            let entry_type = unsafe { *(self.sdt.data_address() as *const u8).add(self.i) };
+            let entry_len =
+                unsafe { *(self.sdt.data_address() as *const u8).add(self.i + 1) } as usize;
+
+            if self.i + entry_len <= self.sdt.data_len() {
+                let item = match entry_type {
+                    0x0 => {
+                        if entry_len == size_of::<MadtLocalApic>() + 2 {
+                            MadtEntry::LocalApic(unsafe {
+                                &*((self.sdt.data_address() + self.i + 2) as *const MadtLocalApic)
+                            })
+                        } else {
+                            MadtEntry::InvalidLocalApic(entry_len)
+                        }
+                    }
+                    0x1 => {
+                        if entry_len == size_of::<MadtIoApic>() + 2 {
+                            MadtEntry::IoApic(unsafe {
+                                &*((self.sdt.data_address() + self.i + 2) as *const MadtIoApic)
+                            })
+                        } else {
+                            MadtEntry::InvalidIoApic(entry_len)
+                        }
+                    }
+                    0x2 => {
+                        if entry_len == size_of::<MadtIntSrcOverride>() + 2 {
+                            MadtEntry::IntSrcOverride(unsafe {
+                                &*((self.sdt.data_address() + self.i + 2)
+                                    as *const MadtIntSrcOverride)
+                            })
+                        } else {
+                            MadtEntry::InvalidIntSrcOverride(entry_len)
+                        }
+                    }
+                    0xB => {
+                        if entry_len >= size_of::<MadtGicc>() + 2 {
+                            MadtEntry::Gicc(unsafe {
+                                &*((self.sdt.data_address() + self.i + 2) as *const MadtGicc)
+                            })
+                        } else {
+                            MadtEntry::InvalidGicc(entry_len)
+                        }
+                    }
+                    0xC => {
+                        if entry_len >= size_of::<MadtGicd>() + 2 {
+                            MadtEntry::Gicd(unsafe {
+                                &*((self.sdt.data_address() + self.i + 2) as *const MadtGicd)
+                            })
+                        } else {
+                            MadtEntry::InvalidGicd(entry_len)
+                        }
+                    }
+                    _ => MadtEntry::Unknown(entry_type),
+                };
+
+                self.i += entry_len;
+
+                Some(item)
+            } else {
+                None
+            }
+        } else {
+            None
+        }
+    }
+}

src/acpi/mod.rs

@@ -0,0 +1,212 @@
+//! # ACPI
+//! Code to parse the ACPI tables
+
+use alloc::{boxed::Box, string::String, vec::Vec};
+
+use hashbrown::HashMap;
+use spin::{Once, RwLock};
+
+use crate::memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch};
+
+use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt};
+
+#[cfg(target_arch = "aarch64")]
+mod gtdt;
+pub mod hpet;
+pub mod madt;
+mod rsdp;
+mod rsdt;
+mod rxsdt;
+pub mod sdt;
+#[cfg(target_arch = "aarch64")]
+mod spcr;
+mod xsdt;
+
+unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) {
+    unsafe {
+        let base = PhysicalAddress::new(crate::memory::round_down_pages(addr.data()));
+        let aligned_len = crate::memory::round_up_pages(len + (addr.data() - base.data()));
+
+        for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE {
+            let (_, flush) = mapper
+                .map_linearly(
+                    base.add(page_idx * crate::memory::PAGE_SIZE),
+                    PageFlags::new(),
+                )
+                .expect("failed to linearly map SDT");
+            flush.flush();
+        }
+    }
+}
+
+pub fn get_sdt(sdt_address: PhysicalAddress, mapper: &mut KernelMapper<true>) -> &'static Sdt {
+    let sdt;
+
+    unsafe {
+        const SDT_SIZE: usize = size_of::<Sdt>();
+        map_linearly(sdt_address, SDT_SIZE, mapper);
+
+        sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt);
+
+        map_linearly(
+            sdt_address.add(SDT_SIZE),
+            sdt.length as usize - SDT_SIZE,
+            mapper,
+        );
+    }
+    sdt
+}
+
+#[repr(C, packed)]
+#[derive(Clone, Copy, Debug, Default)]
+pub struct GenericAddressStructure {
+    pub address_space: u8,
+    pub bit_width: u8,
+    pub bit_offset: u8,
+    pub access_size: u8,
+    pub address: u64,
+}
+
+pub enum RxsdtEnum {
+    Rsdt(Rsdt),
+    Xsdt(Xsdt),
+}
+impl Rxsdt for RxsdtEnum {
+    fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
+        match self {
+            Self::Rsdt(rsdt) => <Rsdt as Rxsdt>::iter(rsdt),
+            Self::Xsdt(xsdt) => <Xsdt as Rxsdt>::iter(xsdt),
+        }
+    }
+}
+
+pub static RXSDT_ENUM: Once<RxsdtEnum> = Once::new();
+
+/// Parse the ACPI tables to gather CPU, interrupt, and timer information
+pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
+    unsafe {
+        {
+            let mut sdt_ptrs = SDT_POINTERS.write();
+            *sdt_ptrs = Some(HashMap::new());
+        }
+
+        // Search for RSDP
+        let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp);
+
+        if let Some(rsdp) = rsdp_opt {
+            debug!("SDT address: {:#x}", rsdp.sdt_address().data());
+            let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw());
+
+            let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) {
+                let mut initialized = false;
+
+                let rsdt = RXSDT_ENUM.call_once(|| {
+                    initialized = true;
+
+                    RxsdtEnum::Rsdt(rsdt)
+                });
+
+                if !initialized {
+                    error!("RXSDT_ENUM already initialized");
+                }
+
+                rsdt
+            } else if let Some(xsdt) = Xsdt::new(rxsdt) {
+                let mut initialized = false;
+
+                let xsdt = RXSDT_ENUM.call_once(|| {
+                    initialized = true;
+
+                    RxsdtEnum::Xsdt(xsdt)
+                });
+                if !initialized {
+                    error!("RXSDT_ENUM already initialized");
+                }
+
+                xsdt
+            } else {
+                warn!("UNKNOWN RSDT OR XSDT SIGNATURE");
+                return;
+            };
+
+            // TODO: Don't touch ACPI tables in kernel?
+
+            for sdt in rxsdt.iter() {
+                get_sdt(sdt, &mut KernelMapper::lock_rw());
+            }
+
+            for sdt_address in rxsdt.iter() {
+                let sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt);
+
+                let signature = get_sdt_signature(sdt);
+                if let Some(ref mut ptrs) = *(SDT_POINTERS.write()) {
+                    ptrs.insert(signature, sdt);
+                }
+            }
+
+            // TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface
+            // to initialize enumerated processors to userspace?
+            Madt::init();
+            //TODO: support this on any arch
+            // SPCR must be initialized after MADT for interrupt controllers
+            #[cfg(target_arch = "aarch64")]
+            spcr::Spcr::init();
+            // TODO: Let userspace setup HPET, and then provide an interface to specify which timer to
+            // use?
+            Hpet::init();
+            #[cfg(target_arch = "aarch64")]
+            gtdt::Gtdt::init();
+        } else {
+            error!("NO RSDP FOUND");
+        }
+    }
+}
+
+pub type SdtSignature = (String, [u8; 6], [u8; 8]);
+pub static SDT_POINTERS: RwLock<Option<HashMap<SdtSignature, &'static Sdt>>> = RwLock::new(None);
+
+pub fn find_sdt(name: &str) -> Vec<&'static Sdt> {
+    let mut sdts: Vec<&'static Sdt> = vec![];
+
+    if let Some(ref ptrs) = *(SDT_POINTERS.read()) {
+        for (signature, sdt) in ptrs {
+            if signature.0 == name {
+                sdts.push(sdt);
+            }
+        }
+    }
+
+    sdts
+}
+
+#[macro_export]
+macro_rules! find_one_sdt {
+    ($name:expr) => {{
+        use $crate::acpi::find_sdt;
+        match find_sdt($name).as_slice() {
+            [] => {
+                println!("Unable to find {}", $name);
+                return;
+            }
+            [x] => *x,
+            x => {
+                println!("{} {} found, expected 1", x.len(), $name);
+                return;
+            }
+        }
+    }};
+}
+
+pub fn get_sdt_signature(sdt: &'static Sdt) -> SdtSignature {
+    let signature =
+        String::from_utf8(sdt.signature.to_vec()).expect("Error converting signature to string");
+    (signature, sdt.oem_id, sdt.oem_table_id)
+}
+
+pub struct Acpi {
+    pub hpet: RwLock<Option<Hpet>>,
+}
+
+pub static ACPI_TABLE: Acpi = Acpi {
+    hpet: RwLock::new(None),
+};

src/acpi/rsdp.rs

@@ -0,0 +1,34 @@
+use rmm::PhysicalAddress;
+
+/// RSDP
+#[derive(Copy, Clone, Debug)]
+#[repr(C, packed)]
+pub struct Rsdp {
+    signature: [u8; 8],
+    _checksum: u8,
+    _oemid: [u8; 6],
+    revision: u8,
+    rsdt_address: u32,
+    _length: u32,
+    xsdt_address: u64,
+    _extended_checksum: u8,
+    _reserved: [u8; 3],
+}
+
+impl Rsdp {
+    pub unsafe fn get_rsdp(already_supplied_rsdp: Option<*const u8>) -> Option<Rsdp> {
+        already_supplied_rsdp.map(|rsdp_ptr| {
+            // TODO: Validate
+            unsafe { *(rsdp_ptr as *const Rsdp) }
+        })
+    }
+
+    /// Get the RSDT or XSDT address
+    pub fn sdt_address(&self) -> PhysicalAddress {
+        PhysicalAddress::new(if self.revision >= 2 {
+            self.xsdt_address as usize
+        } else {
+            self.rsdt_address as usize
+        })
+    }
+}

src/acpi/rsdt.rs

@@ -0,0 +1,52 @@
+use alloc::boxed::Box;
+use core::convert::TryFrom;
+use rmm::PhysicalAddress;
+
+use super::{rxsdt::Rxsdt, sdt::Sdt};
+
+#[derive(Debug)]
+pub struct Rsdt(&'static Sdt);
+
+impl Rsdt {
+    pub fn new(sdt: &'static Sdt) -> Option<Rsdt> {
+        if &sdt.signature == b"RSDT" {
+            Some(Rsdt(sdt))
+        } else {
+            None
+        }
+    }
+    pub fn as_slice(&self) -> &[u8] {
+        let length =
+            usize::try_from(self.0.length).expect("expected 32-bit length to fit within usize");
+
+        unsafe { core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) }
+    }
+}
+
+impl Rxsdt for Rsdt {
+    fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
+        Box::new(RsdtIter { sdt: self.0, i: 0 })
+    }
+}
+
+pub struct RsdtIter {
+    sdt: &'static Sdt,
+    i: usize,
+}
+
+impl Iterator for RsdtIter {
+    type Item = PhysicalAddress;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.i < self.sdt.data_len() / size_of::<u32>() {
+            let item = unsafe {
+                (self.sdt.data_address() as *const u32)
+                    .add(self.i)
+                    .read_unaligned()
+            };
+            self.i += 1;
+            Some(PhysicalAddress::new(item as usize))
+        } else {
+            None
+        }
+    }
+}

src/acpi/rxsdt.rs

@@ -0,0 +1,6 @@
+use alloc::boxed::Box;
+use rmm::PhysicalAddress;
+
+pub trait Rxsdt {
+    fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>>;
+}

src/acpi/sdt.rs

@@ -0,0 +1,27 @@
+#[derive(Copy, Clone, Debug)]
+#[repr(C, packed)]
+pub struct Sdt {
+    pub signature: [u8; 4],
+    pub length: u32,
+    pub revision: u8,
+    pub checksum: u8,
+    pub oem_id: [u8; 6],
+    pub oem_table_id: [u8; 8],
+    pub oem_revision: u32,
+    pub creator_id: u32,
+    pub creator_revision: u32,
+}
+
+impl Sdt {
+    /// Get the address of this tables data
+    pub fn data_address(&self) -> usize {
+        self as *const _ as usize + size_of::<Sdt>()
+    }
+
+    /// Get the length of this tables data
+    pub fn data_len(&self) -> usize {
+        let total_size = self.length as usize;
+        let header_size = size_of::<Sdt>();
+        total_size.saturating_sub(header_size)
+    }
+}

src/acpi/spcr.rs

@@ -0,0 +1,140 @@
+use super::{find_sdt, sdt::Sdt, GenericAddressStructure};
+use crate::{
+    arch::device::serial::COM1,
+    devices::{serial::SerialKind, uart_pl011},
+    log::LOG,
+    memory::{map_device_memory, PhysicalAddress, PAGE_SIZE},
+};
+
+const INTERRUPT_TYPE_8259: u8 = 1 << 0;
+const INTERRUPT_TYPE_APIC: u8 = 1 << 1;
+const INTERRUPT_TYPE_SAPIC: u8 = 1 << 2;
+const INTERRUPT_TYPE_GIC: u8 = 1 << 3;
+const INTERRUPT_TYPE_PLIC: u8 = 1 << 4;
+
+#[derive(Clone, Copy, Debug)]
+#[repr(C, packed)]
+pub struct Spcr {
+    pub header: Sdt,
+    pub interface_type: u8,
+    _reserved: [u8; 3],
+    pub base_address: GenericAddressStructure,
+    pub interrupt_type: u8,
+    pub irq: u8,
+    pub gsiv: u32,
+    pub configured_baud_rate: u8,
+    pub parity: u8,
+    pub stop_bits: u8,
+    pub flow_control: u8,
+    pub terminal_type: u8,
+    pub language: u8,
+    pub pci_device_id: u16,
+    pub pci_vendor_id: u16,
+    pub pci_bus: u8,
+    pub pci_device: u8,
+    pub pci_function: u8,
+    pub pci_flags: u32,
+    pub pci_segment: u8,
+    /*TODO: these fields are optional based on the table revision
+    pub uart_clock_frequency: u32,
+    pub precise_baud_rate: u32,
+    pub namespace_string_length: u16,
+    pub namespace_string_offset: u16,
+    */
+    // namespace_string
+}
+
+impl Spcr {
+    pub fn init() {
+        let spcr_sdt = find_sdt("SPCR");
+        let spcr = if spcr_sdt.len() == 1 {
+            match Spcr::new(spcr_sdt[0]) {
+                Some(spcr) => spcr,
+                None => {
+                    warn!("Failed to parse SPCR");
+                    return;
+                }
+            }
+        } else {
+            warn!("Unable to find SPCR");
+            return;
+        };
+
+        if spcr.base_address.address == 0 {
+            // Serial disabled
+            return;
+        }
+
+        let serial_was_empty = !matches!(*COM1.lock(), SerialKind::NotPresent);
+        if spcr.header.revision >= 2 {
+            match spcr.interface_type {
+                3 => {
+                    // PL011
+                    if spcr.base_address.address_space == 0
+                        && spcr.base_address.bit_width == 32
+                        && spcr.base_address.bit_offset == 0
+                        && spcr.base_address.access_size == 3
+                    {
+                        let virt = unsafe {
+                            map_device_memory(
+                                PhysicalAddress::new(spcr.base_address.address as usize),
+                                PAGE_SIZE,
+                            )
+                        };
+                        let serial_port = uart_pl011::SerialPort::new(virt.data(), false);
+                        *COM1.lock() = SerialKind::Pl011(serial_port);
+                        //TODO: enable IRQ on more platforms and interrupt types
+                        if (spcr.interrupt_type & INTERRUPT_TYPE_GIC) == INTERRUPT_TYPE_GIC {
+                            #[cfg(target_arch = "aarch64")]
+                            unsafe {
+                                crate::arch::device::serial::init_acpi(spcr.gsiv);
+                            }
+                        }
+                    } else {
+                        warn!(
+                            "SPCR unsuppoted address for PL011 {:#x?}",
+                            spcr.base_address
+                        );
+                    }
+                }
+                //TODO: support more types!
+                unsupported => {
+                    warn!(
+                        "SPCR revision {} unsupported interface type {}",
+                        spcr.header.revision, unsupported
+                    );
+                }
+            }
+        } else if spcr.header.revision == 1 {
+            match spcr.interface_type {
+                //TODO: support more types!
+                unsupported => {
+                    warn!("SPCR revision 1 unsupported interface type {}", unsupported);
+                }
+            }
+        } else {
+            warn!("SPCR unsupported revision {}", spcr.header.revision);
+        }
+        let mut serial_port = COM1.lock();
+        if serial_was_empty && !matches!(*serial_port, SerialKind::NotPresent) {
+            // backfill logs since the heap is loaded
+            if let Some(ref mut early_log) = *LOG.lock() {
+                let (s1, s2) = early_log.read();
+                if !s1.is_empty() {
+                    serial_port.write(s1);
+                }
+                if !s2.is_empty() {
+                    serial_port.write(s2);
+                }
+            }
+        }
+    }
+
+    pub fn new(sdt: &'static Sdt) -> Option<&'static Spcr> {
+        if &sdt.signature == b"SPCR" && sdt.length as usize >= size_of::<Spcr>() {
+            Some(unsafe { &*((sdt as *const Sdt) as *const Spcr) })
+        } else {
+            None
+        }
+    }
+}

src/acpi/xsdt.rs

@@ -0,0 +1,50 @@
+use alloc::boxed::Box;
+use core::convert::TryFrom;
+use rmm::PhysicalAddress;
+
+use super::{rxsdt::Rxsdt, sdt::Sdt};
+
+#[derive(Debug)]
+pub struct Xsdt(&'static Sdt);
+
+impl Xsdt {
+    pub fn new(sdt: &'static Sdt) -> Option<Xsdt> {
+        if &sdt.signature == b"XSDT" {
+            Some(Xsdt(sdt))
+        } else {
+            None
+        }
+    }
+    pub fn as_slice(&self) -> &[u8] {
+        let length =
+            usize::try_from(self.0.length).expect("expected 32-bit length to fit within usize");
+
+        unsafe { core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) }
+    }
+}
+
+impl Rxsdt for Xsdt {
+    fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
+        Box::new(XsdtIter { sdt: self.0, i: 0 })
+    }
+}
+
+pub struct XsdtIter {
+    sdt: &'static Sdt,
+    i: usize,
+}
+
+impl Iterator for XsdtIter {
+    type Item = PhysicalAddress;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.i < self.sdt.data_len() / size_of::<u64>() {
+            let item = unsafe {
+                core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i))
+            };
+            self.i += 1;
+            Some(PhysicalAddress::new(item as usize))
+        } else {
+            None
+        }
+    }
+}

src/allocator/linked_list.rs

@@ -0,0 +1,50 @@
+use crate::memory::KernelMapper;
+use core::{
+    alloc::{GlobalAlloc, Layout},
+    ptr::NonNull,
+};
+use linked_list_allocator::Heap;
+use spin::Mutex;
+
+static HEAP: Mutex<Option<Heap>> = Mutex::new(None);
+
+pub struct Allocator;
+
+impl Allocator {
+    pub unsafe fn init(offset: usize, size: usize) {
+        unsafe {
+            *HEAP.lock() = Some(Heap::new(offset, size));
+        }
+    }
+}
+
+unsafe impl GlobalAlloc for Allocator {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        unsafe {
+            while let Some(ref mut heap) = *HEAP.lock() {
+                match heap.allocate_first_fit(layout) {
+                    Ok(ptr) => return ptr.as_ptr(),
+                    Err(()) => {
+                        let size = heap.size();
+                        super::map_heap(
+                            &mut KernelMapper::lock_rw(),
+                            crate::kernel_heap_offset() + size,
+                            super::KERNEL_HEAP_SIZE,
+                        );
+                        heap.extend(super::KERNEL_HEAP_SIZE);
+                    }
+                }
+            }
+            panic!("__rust_allocate: heap not initialized");
+        }
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        unsafe {
+            HEAP.lock()
+                .as_mut()
+                .expect("heap not initialized")
+                .deallocate(NonNull::new_unchecked(ptr), layout)
+        }
+    }
+}

src/allocator/mod.rs

@@ -0,0 +1,48 @@
+use crate::memory::{KernelMapper, Page, PageFlags, VirtualAddress};
+use rmm::{Flusher, FrameAllocator, PageFlushAll};
+
+pub use self::linked_list::Allocator;
+mod linked_list;
+
+/// Size of kernel heap
+const KERNEL_HEAP_SIZE: usize = ::rmm::MEGABYTE;
+
+unsafe fn map_heap(mapper: &mut KernelMapper<true>, offset: usize, size: usize) {
+    let mut flush_all = PageFlushAll::new();
+
+    let heap_start_page = Page::containing_address(VirtualAddress::new(offset));
+    let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1));
+    for page in Page::range_inclusive(heap_start_page, heap_end_page) {
+        let phys = mapper
+            .allocator_mut()
+            .allocate_one()
+            .expect("failed to allocate kernel heap");
+        let flush = unsafe {
+            mapper
+                .map_phys(
+                    page.start_address(),
+                    phys,
+                    PageFlags::new()
+                        .write(true)
+                        .global(cfg!(not(feature = "pti"))),
+                )
+                .expect("failed to map kernel heap")
+        };
+        flush_all.consume(flush);
+    }
+
+    flush_all.flush();
+}
+
+pub unsafe fn init() {
+    unsafe {
+        let offset = crate::kernel_heap_offset();
+        let size = KERNEL_HEAP_SIZE;
+
+        // Map heap pages
+        map_heap(&mut KernelMapper::lock_rw(), offset, size);
+
+        // Initialize global heap
+        Allocator::init(offset, size);
+    }
+}

src/arch/aarch64/consts.rs

@@ -0,0 +1,15 @@
+// Because the memory map is so important to not be aliased, it is defined here, in one place
+// The lower 256 PML4 entries are reserved for userspace
+// Each PML4 entry references up to 512 GB of memory
+// The second from the top (510) PML4 is reserved for the kernel
+/// The size of a single PML4
+pub const PML4_SIZE: usize = 0x0000_0080_0000_0000;
+
+/// Offset to kernel heap
+#[inline(always)]
+pub fn kernel_heap_offset() -> usize {
+    crate::kernel_executable_offsets::KERNEL_OFFSET() - PML4_SIZE
+}
+
+/// End offset of the user image, i.e. kernel start
+pub const USER_END_OFFSET: usize = 256 * PML4_SIZE;

src/arch/aarch64/debug.rs

@@ -0,0 +1,19 @@
+use spin::MutexGuard;
+
+use crate::{arch::device::serial::COM1, devices::serial::SerialKind};
+
+pub struct Writer<'a> {
+    serial: MutexGuard<'a, SerialKind>,
+}
+
+impl<'a> Writer<'a> {
+    pub fn new() -> Writer<'a> {
+        Writer {
+            serial: COM1.lock(),
+        }
+    }
+
+    pub fn write(&mut self, buf: &[u8]) {
+        self.serial.write(buf);
+    }
+}

src/arch/aarch64/device/cpu/mod.rs

@@ -0,0 +1,277 @@
+use core::fmt::{Result, Write};
+
+use crate::arch::device::cpu::registers::{control_regs, id_regs};
+
+pub mod registers;
+
+bitfield::bitfield! {
+    pub struct MachineId(u32);
+    get_implementer, _: 31, 24;
+    get_variant, _: 23, 20;
+    get_architecture, _: 19, 16;
+    get_part_number, _: 15, 4;
+    get_revision, _: 3, 0;
+}
+
+enum ImplementerID {
+    Unknown,
+    Arm,
+    Broadcom,
+    Cavium,
+    Digital,
+    Fujitsu,
+    Infineon,
+    Motorola,
+    Nvidia,
+    AMCC,
+    Qualcomm,
+    Marvell,
+    Intel,
+    Ampere,
+}
+
+const IMPLEMENTERS: [&'static str; 14] = [
+    "Unknown", "Arm", "Broadcom", "Cavium", "Digital", "Fujitsu", "Infineon", "Motorola", "Nvidia",
+    "AMCC", "Qualcomm", "Marvell", "Intel", "Ampere",
+];
+
+enum VariantID {
+    Unknown,
+}
+
+const VARIANTS: [&'static str; 1] = ["Unknown"];
+
+enum ArchitectureID {
+    Unknown,
+    V4,
+    V4T,
+    V5,
+    V5T,
+    V5TE,
+    V5TEJ,
+    V6,
+}
+
+const ARCHITECTURES: [&'static str; 8] =
+    ["Unknown", "v4", "v4T", "v5", "v5T", "v5TE", "v5TEJ", "v6"];
+
+enum PartNumberID {
+    Unknown,
+    Thunder,
+    Foundation,
+    CortexA35,
+    CortexA53,
+    CortexA55,
+    CortexA57,
+    CortexA72,
+    CortexA73,
+    CortexA75,
+}
+
+const PART_NUMBERS: [&'static str; 10] = [
+    "Unknown",
+    "Thunder",
+    "Foundation",
+    "Cortex-A35",
+    "Cortex-A53",
+    "Cortex-A55",
+    "Cortex-A57",
+    "Cortex-A72",
+    "Cortex-A73",
+    "Cortex-A75",
+];
+
+enum RevisionID {
+    Unknown,
+    Thunder1_0,
+    Thunder1_1,
+}
+
+const REVISIONS: [&'static str; 3] = ["Unknown", "Thunder-1.0", "Thunder-1.1"];
+
+struct CpuInfo {
+    implementer: &'static str,
+    variant: &'static str,
+    architecture: &'static str,
+    part_number: &'static str,
+    revision: &'static str,
+    aa64isar0: id_regs::AA64Isar0,
+    aa64isar1: id_regs::AA64Isar1,
+}
+
+impl CpuInfo {
+    fn new() -> CpuInfo {
+        let midr = unsafe { control_regs::midr() };
+        let midr = MachineId(midr);
+
+        let implementer = match midr.get_implementer() {
+            0x41 => IMPLEMENTERS[ImplementerID::Arm as usize],
+            0x42 => IMPLEMENTERS[ImplementerID::Broadcom as usize],
+            0x43 => IMPLEMENTERS[ImplementerID::Cavium as usize],
+            0x44 => IMPLEMENTERS[ImplementerID::Digital as usize],
+            0x46 => IMPLEMENTERS[ImplementerID::Fujitsu as usize],
+            0x49 => IMPLEMENTERS[ImplementerID::Infineon as usize],
+            0x4d => IMPLEMENTERS[ImplementerID::Motorola as usize],
+            0x4e => IMPLEMENTERS[ImplementerID::Nvidia as usize],
+            0x50 => IMPLEMENTERS[ImplementerID::AMCC as usize],
+            0x51 => IMPLEMENTERS[ImplementerID::Qualcomm as usize],
+            0x56 => IMPLEMENTERS[ImplementerID::Marvell as usize],
+            0x69 => IMPLEMENTERS[ImplementerID::Intel as usize],
+            0xc0 => IMPLEMENTERS[ImplementerID::Ampere as usize],
+            _ => IMPLEMENTERS[ImplementerID::Unknown as usize],
+        };
+
+        let variant = match midr.get_variant() {
+            _ => VARIANTS[VariantID::Unknown as usize],
+        };
+
+        let architecture = match midr.get_architecture() {
+            0b0001 => ARCHITECTURES[ArchitectureID::V4 as usize],
+            0b0010 => ARCHITECTURES[ArchitectureID::V4T as usize],
+            0b0011 => ARCHITECTURES[ArchitectureID::V5 as usize],
+            0b0100 => ARCHITECTURES[ArchitectureID::V5T as usize],
+            0b0101 => ARCHITECTURES[ArchitectureID::V5TE as usize],
+            0b0110 => ARCHITECTURES[ArchitectureID::V5TEJ as usize],
+            0b0111 => ARCHITECTURES[ArchitectureID::V6 as usize],
+            _ => ARCHITECTURES[ArchitectureID::Unknown as usize],
+        };
+
+        let part_number = match midr.get_part_number() {
+            0x0a1 => PART_NUMBERS[PartNumberID::Thunder as usize],
+            0xd00 => PART_NUMBERS[PartNumberID::Foundation as usize],
+            0xd04 => PART_NUMBERS[PartNumberID::CortexA35 as usize],
+            0xd03 => PART_NUMBERS[PartNumberID::CortexA53 as usize],
+            0xd05 => PART_NUMBERS[PartNumberID::CortexA55 as usize],
+            0xd07 => PART_NUMBERS[PartNumberID::CortexA57 as usize],
+            0xd08 => PART_NUMBERS[PartNumberID::CortexA72 as usize],
+            0xd09 => PART_NUMBERS[PartNumberID::CortexA73 as usize],
+            0xd0a => PART_NUMBERS[PartNumberID::CortexA75 as usize],
+            _ => PART_NUMBERS[PartNumberID::Unknown as usize],
+        };
+
+        let revision = match part_number {
+            "Thunder" => {
+                let val = match midr.get_revision() {
+                    0x00 => REVISIONS[RevisionID::Thunder1_0 as usize],
+                    0x01 => REVISIONS[RevisionID::Thunder1_1 as usize],
+                    _ => REVISIONS[RevisionID::Unknown as usize],
+                };
+                val
+            }
+            _ => REVISIONS[RevisionID::Unknown as usize],
+        };
+
+        let aa64isar0 = id_regs::aa64isar0();
+        let aa64isar1 = id_regs::aa64isar1();
+
+        CpuInfo {
+            implementer,
+            variant,
+            architecture,
+            part_number,
+            revision,
+            aa64isar0,
+            aa64isar1,
+        }
+    }
+}
+
+pub fn cpu_info<W: Write>(w: &mut W) -> Result {
+    let cpuinfo = CpuInfo::new();
+
+    writeln!(w, "Implementer: {}", cpuinfo.implementer)?;
+    writeln!(w, "Variant: {}", cpuinfo.variant)?;
+    writeln!(w, "Architecture version: {}", cpuinfo.architecture)?;
+    writeln!(w, "Part Number: {}", cpuinfo.part_number)?;
+    writeln!(w, "Revision: {}", cpuinfo.revision)?;
+
+    // Print detected CPU features.
+    // Follow the naming convention estabilished by `std::arch::is_aarch64_feature_detected`.
+    write!(w, "Features:")?;
+
+    // ID_AA64ISAR0_EL1
+    if cpuinfo.aa64isar0.has_feat_rng() {
+        write!(w, " rand")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_flagm() {
+        write!(w, " flagm")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_flagm2() {
+        write!(w, " flagm2")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_fhm() {
+        write!(w, " fhm")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_dotprod() {
+        write!(w, " dotprod")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_sm3() && cpuinfo.aa64isar0.has_feat_sm4() {
+        write!(w, " sm4")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_sha512() && cpuinfo.aa64isar0.has_feat_sha3() {
+        write!(w, " sha3")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_rdm() {
+        write!(w, " rdm")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_lse() {
+        write!(w, " lse")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_lse128() {
+        write!(w, " lse128")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_crc() {
+        write!(w, " crc")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_sha1() && cpuinfo.aa64isar0.has_feat_sha256() {
+        write!(w, " sha2")?;
+    }
+    if cpuinfo.aa64isar0.has_feat_aes() && cpuinfo.aa64isar0.has_feat_pmull() {
+        write!(w, " aes")?;
+    }
+
+    // ID_AA64ISAR1_EL1
+    if cpuinfo.aa64isar1.has_feat_i8mm() {
+        write!(w, " i8mm")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_bf16() {
+        write!(w, " bf16")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_sb() {
+        write!(w, " sb")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_frintts() {
+        write!(w, " frintts")?;
+    }
+    if cpuinfo.aa64isar1.gpi() != 0 || cpuinfo.aa64isar1.gpa() != 0 {
+        write!(w, " pacg")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_lrcpc() {
+        write!(w, " rcpc")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_lrcpc2() {
+        write!(w, " rcpc2")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_lrcpc3() {
+        write!(w, " rcpc3")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_fcma() {
+        write!(w, " fcma")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_jscvt() {
+        write!(w, " jsconv")?;
+    }
+    if cpuinfo.aa64isar1.api() != 0 || cpuinfo.aa64isar1.apa() != 0 {
+        write!(w, " paca")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_dpb() {
+        write!(w, " dpb")?;
+    }
+    if cpuinfo.aa64isar1.has_feat_dpb2() {
+        write!(w, " dpb2")?;
+    }
+
+    writeln!(w)?;
+
+    Ok(())
+}

src/arch/aarch64/device/cpu/registers/control_regs.rs

@@ -0,0 +1,167 @@
+#![allow(unused)]
+
+//! Functions to read and write control registers.
+
+use core::arch::asm;
+
+pub unsafe fn ttbr0_el1() -> u64 {
+    unsafe {
+        let ret: u64;
+        asm!("mrs {}, ttbr0_el1", out(reg) ret);
+        ret
+    }
+}
+
+pub unsafe fn ttbr0_el1_write(val: u64) {
+    unsafe {
+        asm!("msr ttbr0_el1, {}", in(reg) val);
+    }
+}
+
+pub unsafe fn ttbr1_el1() -> u64 {
+    unsafe {
+        let ret: u64;
+        asm!("mrs {}, ttbr1_el1", out(reg) ret);
+        ret
+    }
+}
+
+pub unsafe fn ttbr1_el1_write(val: u64) {
+    unsafe {
+        asm!("msr ttbr1_el1, {}", in(reg) val);
+    }
+}
+
+pub unsafe fn tpidr_el0() -> u64 {
+    unsafe {
+        let ret: u64;
+        asm!("mrs {}, tpidr_el0", out(reg) ret);
+        ret
+    }
+}
+
+pub unsafe fn tpidr_el0_write(val: u64) {
+    unsafe {
+        asm!("msr tpidr_el0, {}", in(reg) val);
+    }
+}
+
+pub unsafe fn tpidr_el1() -> u64 {
+    unsafe {
+        let ret: u64;
+        asm!("mrs {}, tpidr_el1", out(reg) ret);
+        ret
+    }
+}
+
+pub unsafe fn tpidr_el1_write(val: u64) {
+    unsafe {
+        asm!("msr tpidr_el1, {}", in(reg) val);
+    }
+}
+
+pub unsafe fn tpidrro_el0() -> u64 {
+    unsafe {
+        let ret: u64;
+        asm!("mrs {}, tpidrro_el0", out(reg) ret);
+        ret
+    }
+}
+
+pub unsafe fn tpidrro_el0_write(val: u64) {
+    unsafe {
+        asm!("msr tpidrro_el0, {}", in(reg) val);
+    }
+}
+
+pub unsafe fn esr_el1() -> u32 {
+    unsafe {
+        let ret: u32;
+        asm!("mrs {0:w}, esr_el1", out(reg) ret);
+        ret
+    }
+}
+
+pub unsafe fn vhe_present() -> bool {
+    unsafe {
+        let mut mmfr1: u64;
+        asm!("mrs {}, id_aa64mmfr1_el1", out(reg) mmfr1);
+
+        // The VHE (Virtualization Host Extensions) field is in bits [7:4].
+        let vhe_field = (mmfr1 >> 4) & 0b1111;
+
+        vhe_field != 0
+    }
+}
+
+pub unsafe fn cntfrq_el0() -> u32 {
+    unsafe {
+        let ret: usize;
+        asm!("mrs {}, cntfrq_el0", out(reg) ret);
+        ret as u32
+    }
+}
+
+pub unsafe fn ptmr_ctrl() -> u32 {
+    unsafe {
+        let ret: usize;
+        asm!("mrs {}, cntp_ctl_el0", out(reg) ret);
+        ret as u32
+    }
+}
+
+pub unsafe fn ptmr_ctrl_write(val: u32) {
+    unsafe {
+        asm!("msr cntp_ctl_el0, {}", in(reg) val as usize);
+    }
+}
+
+pub unsafe fn ptmr_tval() -> u32 {
+    unsafe {
+        let ret: usize;
+        asm!("mrs {0}, cntp_tval_el0", out(reg) ret);
+        ret as u32
+    }
+}
+
+pub unsafe fn ptmr_tval_write(val: u32) {
+    unsafe {
+        asm!("msr cntp_tval_el0, {}", in(reg) val as usize);
+    }
+}
+
+pub unsafe fn vtmr_ctrl() -> u32 {
+    unsafe {
+        let ret: usize;
+        asm!("mrs {}, cntv_ctl_el0", out(reg) ret);
+        ret as u32
+    }
+}
+
+pub unsafe fn vtmr_ctrl_write(val: u32) {
+    unsafe {
+        asm!("msr cntv_ctl_el0, {}", in(reg) val as usize);
+    }
+}
+
+pub unsafe fn vtmr_tval() -> u32 {
+    unsafe {
+        let ret: usize;
+        asm!("mrs {0}, cntv_tval_el0", out(reg) ret);
+        ret as u32
+    }
+}
+
+pub unsafe fn vtmr_tval_write(val: u32) {
+    unsafe {
+        asm!("msr cntv_tval_el0, {}", in(reg) val as usize);
+    }
+}
+
+pub unsafe fn midr() -> u32 {
+    unsafe {
+        let ret: usize;
+        asm!("mrs {}, midr_el1", out(reg) ret);
+        ret as u32
+    }
+}

src/arch/aarch64/device/cpu/registers/id_regs.rs

@@ -0,0 +1,151 @@
+//! Functions and bitfield definitions for `ID_AA64*` system registers. (e.g. `ID_AA64ISAR0_EL1`)
+
+use core::arch::asm;
+
+bitfield::bitfield! {
+    pub struct AA64Isar0(u64);
+    impl Debug;
+    pub rndr, _: 63, 60;
+    pub tlb, _: 59, 56;
+    pub ts, _: 55, 52;
+    pub fhm, _: 51, 48;
+    pub dp, _: 47, 44;
+    pub sm4, _: 43, 40;
+    pub sm3, _: 39, 36;
+    pub sha3, _: 35, 32;
+    pub rdm, _: 31, 28;
+    pub atomic, _: 23, 20;
+    pub crc32, _: 19, 16;
+    pub sha2, _: 15, 12;
+    pub sha1, _: 11, 8;
+    pub aes, _: 7, 4;
+}
+
+bitfield::bitfield! {
+    pub struct AA64Isar1(u64);
+    impl Debug;
+    pub ls64, _: 63, 60;
+    pub xs, _: 59, 56;
+    pub i8mm, _: 55, 52;
+    pub dgh, _: 51, 48;
+    pub bf16, _: 47, 44;
+    pub specres, _: 43, 40;
+    pub sb, _: 39, 36;
+    pub frintts, _: 35, 32;
+    pub gpi, _: 31, 28;
+    pub gpa, _: 27, 24;
+    pub lrcpc, _: 23, 20;
+    pub fcma, _: 19, 16;
+    pub jscvt, _: 15, 12;
+    pub api, _: 11, 8;
+    pub apa, _: 7, 4;
+    pub dpb, _: 3, 0;
+}
+
+impl AA64Isar0 {
+    pub fn has_feat_rng(&self) -> bool {
+        self.rndr() == 0b0001
+    }
+    pub fn has_feat_flagm(&self) -> bool {
+        self.ts() == 0b0001
+    }
+    pub fn has_feat_flagm2(&self) -> bool {
+        self.ts() == 0b0010
+    }
+    pub fn has_feat_fhm(&self) -> bool {
+        self.fhm() == 0b0001
+    }
+    pub fn has_feat_dotprod(&self) -> bool {
+        self.dp() == 0b0001
+    }
+    pub fn has_feat_sm4(&self) -> bool {
+        self.sm4() == 0b0001
+    }
+    pub fn has_feat_sm3(&self) -> bool {
+        self.sm3() == 0b0001
+    }
+    pub fn has_feat_sha3(&self) -> bool {
+        self.sha3() == 0b0001
+    }
+    pub fn has_feat_rdm(&self) -> bool {
+        self.rdm() == 0b0001
+    }
+    pub fn has_feat_lse(&self) -> bool {
+        self.atomic() == 0b0010
+    }
+    pub fn has_feat_lse128(&self) -> bool {
+        self.atomic() == 0b0011
+    }
+    /// The current Arm Architecture Registers Manual calls it FEAT_CRC32,
+    /// but everyone else seems to call it FEAT_CRC.
+    pub fn has_feat_crc(&self) -> bool {
+        self.crc32() == 0b0001
+    }
+    pub fn has_feat_sha256(&self) -> bool {
+        self.sha2() == 0b0001
+    }
+    pub fn has_feat_sha512(&self) -> bool {
+        self.sha2() == 0b0010
+    }
+    pub fn has_feat_sha1(&self) -> bool {
+        self.sha1() == 0b0001
+    }
+    pub fn has_feat_aes(&self) -> bool {
+        self.aes() == 0b0001
+    }
+    pub fn has_feat_pmull(&self) -> bool {
+        self.aes() == 0b0010
+    }
+}
+
+impl AA64Isar1 {
+    pub fn has_feat_i8mm(&self) -> bool {
+        self.i8mm() == 0b0001
+    }
+    pub fn has_feat_bf16(&self) -> bool {
+        self.bf16() == 0b0001
+    }
+    pub fn has_feat_sb(&self) -> bool {
+        self.sb() == 0b0001
+    }
+    pub fn has_feat_frintts(&self) -> bool {
+        self.frintts() == 0b0001
+    }
+    pub fn has_feat_lrcpc(&self) -> bool {
+        self.lrcpc() == 0b0001
+    }
+    pub fn has_feat_lrcpc2(&self) -> bool {
+        self.lrcpc() == 0b0010
+    }
+    pub fn has_feat_lrcpc3(&self) -> bool {
+        self.lrcpc() == 0b0011
+    }
+    pub fn has_feat_fcma(&self) -> bool {
+        self.fcma() == 0b0001
+    }
+    pub fn has_feat_jscvt(&self) -> bool {
+        self.jscvt() == 0b0011
+    }
+    pub fn has_feat_dpb(&self) -> bool {
+        self.dpb() == 0b0001
+    }
+    pub fn has_feat_dpb2(&self) -> bool {
+        self.dpb() == 0b0010
+    }
+}
+
+pub fn aa64isar0() -> AA64Isar0 {
+    let ret: u64;
+    unsafe {
+        asm!("mrs {}, ID_AA64ISAR0_EL1", out(reg) ret);
+    }
+    AA64Isar0(ret)
+}
+
+pub fn aa64isar1() -> AA64Isar1 {
+    let ret: u64;
+    unsafe {
+        asm!("mrs {}, ID_AA64ISAR1_EL1", out(reg) ret);
+    }
+    AA64Isar1(ret)
+}

src/arch/aarch64/device/cpu/registers/mod.rs

@@ -0,0 +1,2 @@
+pub mod control_regs;
+pub mod id_regs;

src/arch/aarch64/device/generic_timer.rs

@@ -0,0 +1,145 @@
+use alloc::boxed::Box;
+
+use super::ic_for_chip;
+use crate::{
+    arch::device::cpu::registers::control_regs,
+    context::{self, timeout},
+    dtb::{
+        get_interrupt,
+        irqchip::{register_irq, InterruptHandler, IRQ_CHIP},
+    },
+    scheme::irq::irq_trigger,
+    sync::CleanLockToken,
+    time,
+};
+use fdt::Fdt;
+
+bitflags! {
+    struct TimerCtrlFlags: u32 {
+        const ENABLE = 1 << 0;
+        const IMASK = 1 << 1;
+        const ISTATUS = 1 << 2;
+    }
+}
+
+pub unsafe fn init(fdt: &Fdt) {
+    unsafe {
+        let mut timer = GenericTimer::new();
+        timer.init();
+        if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) {
+            let irq = get_interrupt(fdt, &node, 1).unwrap();
+            debug!("irq = {:?}", irq);
+            if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
+                //PHYS_NONSECURE_PPI only
+                let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
+                    .ic
+                    .irq_xlate(irq)
+                    .unwrap();
+                info!("generic_timer virq = {}", virq);
+                register_irq(virq as u32, Box::new(timer));
+                IRQ_CHIP.irq_enable(virq as u32);
+            } else {
+                error!("Failed to find irq parent for generic timer");
+            }
+        }
+    }
+}
+
+pub struct GenericTimer {
+    pub use_virtual_timer: bool,
+    pub clk_freq: u32,
+    pub reload_count: u32,
+}
+
+impl GenericTimer {
+    pub fn new() -> Self {
+        Self {
+            use_virtual_timer: false,
+            clk_freq: 0,
+            reload_count: 0,
+        }
+    }
+    pub fn init(&mut self) {
+        self.use_virtual_timer = unsafe { !control_regs::vhe_present() };
+        debug!(
+            "generic_timer use_virtual_timer = {:?}",
+            self.use_virtual_timer
+        );
+        let clk_freq = unsafe { control_regs::cntfrq_el0() };
+        self.clk_freq = clk_freq;
+        self.reload_count = clk_freq / 100;
+        self.reload_count();
+    }
+
+    fn read_tmr_ctrl(&self) -> TimerCtrlFlags {
+        TimerCtrlFlags::from_bits_truncate(if self.use_virtual_timer {
+            unsafe { control_regs::vtmr_ctrl() }
+        } else {
+            unsafe { control_regs::ptmr_ctrl() }
+        })
+    }
+
+    fn write_tmr_ctrl(&self, ctrl: TimerCtrlFlags) {
+        if self.use_virtual_timer {
+            unsafe { control_regs::vtmr_ctrl_write(ctrl.bits()) };
+        } else {
+            unsafe { control_regs::ptmr_ctrl_write(ctrl.bits()) };
+        }
+    }
+
+    #[allow(unused)]
+    fn disable(&self) {
+        let mut ctrl = self.read_tmr_ctrl();
+        ctrl.remove(TimerCtrlFlags::ENABLE);
+        self.write_tmr_ctrl(ctrl);
+    }
+
+    #[allow(unused)]
+    pub fn set_irq(&mut self) {
+        let mut ctrl = self.read_tmr_ctrl();
+        ctrl.remove(TimerCtrlFlags::IMASK);
+        self.write_tmr_ctrl(ctrl);
+    }
+
+    pub fn clear_irq(&mut self) {
+        let mut ctrl = self.read_tmr_ctrl();
+
+        if ctrl.contains(TimerCtrlFlags::ISTATUS) {
+            ctrl.insert(TimerCtrlFlags::IMASK);
+            self.write_tmr_ctrl(ctrl);
+        }
+    }
+
+    pub fn reload_count(&mut self) {
+        if self.use_virtual_timer {
+            unsafe { control_regs::vtmr_tval_write(self.reload_count) };
+        } else {
+            unsafe { control_regs::ptmr_tval_write(self.reload_count) };
+        }
+        let mut ctrl = self.read_tmr_ctrl();
+        ctrl.insert(TimerCtrlFlags::ENABLE);
+        ctrl.remove(TimerCtrlFlags::IMASK);
+        self.write_tmr_ctrl(ctrl);
+    }
+}
+
+impl InterruptHandler for GenericTimer {
+    fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) {
+        self.clear_irq();
+        {
+            *time::OFFSET.write(token.token()) += self.clk_freq as u128;
+        }
+
+        timeout::trigger(token);
+        context::switch::tick(token);
+
+        unsafe {
+            // FIXME add_irq accepts a u8 as irq number
+            // PercpuBlock::current().stats.add_irq(irq);
+
+            irq_trigger(irq.try_into().unwrap(), token);
+            IRQ_CHIP.irq_eoi(irq);
+        }
+        self.reload_count();
+    }
+}

src/arch/aarch64/device/irqchip/gic.rs

@@ -0,0 +1,288 @@
+use super::InterruptController;
+use crate::{
+    dtb::{
+        get_mmio_address,
+        irqchip::{InterruptHandler, IrqCell, IrqDesc},
+    },
+    sync::CleanLockToken,
+};
+use core::ptr::{read_volatile, write_volatile};
+use fdt::{node::FdtNode, Fdt};
+use syscall::{
+    error::{Error, EINVAL},
+    Result,
+};
+
+static GICD_CTLR: u32 = 0x000;
+static GICD_TYPER: u32 = 0x004;
+static GICD_ISENABLER: u32 = 0x100;
+static GICD_ICENABLER: u32 = 0x180;
+static GICD_IPRIORITY: u32 = 0x400;
+static GICD_ITARGETSR: u32 = 0x800;
+static GICD_ICFGR: u32 = 0xc00;
+
+static GICC_EOIR: u32 = 0x0010;
+static GICC_IAR: u32 = 0x000c;
+static GICC_CTLR: u32 = 0x0000;
+static GICC_PMR: u32 = 0x0004;
+
+pub struct GenericInterruptController {
+    pub gic_dist_if: GicDistIf,
+    pub gic_cpu_if: GicCpuIf,
+    pub irq_range: (usize, usize),
+}
+
+impl GenericInterruptController {
+    pub fn new() -> Self {
+        let gic_dist_if = GicDistIf::default();
+        let gic_cpu_if = GicCpuIf::default();
+
+        GenericInterruptController {
+            gic_dist_if,
+            gic_cpu_if,
+            irq_range: (0, 0),
+        }
+    }
+    pub fn parse(fdt: &Fdt) -> Result<(usize, usize, usize, usize)> {
+        if let Some(node) = fdt.find_compatible(&["arm,cortex-a15-gic", "arm,gic-400"]) {
+            return GenericInterruptController::parse_inner(fdt, &node);
+        } else {
+            return Err(Error::new(EINVAL));
+        }
+    }
+    fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize, usize, usize)> {
+        //assert address_cells == 0x2, size_cells == 0x2
+        let reg = node.reg().unwrap();
+        let mut regs = (0, 0, 0, 0);
+        let mut idx = 0;
+
+        for chunk in reg {
+            if chunk.size.is_none() {
+                break;
+            }
+            let addr = get_mmio_address(fdt, node, &chunk).unwrap();
+            match idx {
+                0 => (regs.0, regs.1) = (addr, chunk.size.unwrap()),
+                2 => (regs.2, regs.3) = (addr, chunk.size.unwrap()),
+                _ => break,
+            }
+            idx += 2;
+        }
+
+        if idx == 4 {
+            Ok(regs)
+        } else {
+            Err(Error::new(EINVAL))
+        }
+    }
+}
+
+impl InterruptHandler for GenericInterruptController {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
+}
+
+impl InterruptController for GenericInterruptController {
+    fn irq_init(
+        &mut self,
+        fdt_opt: Option<&Fdt>,
+        irq_desc: &mut [IrqDesc; 1024],
+        ic_idx: usize,
+        irq_idx: &mut usize,
+    ) -> Result<()> {
+        if let Some(fdt) = fdt_opt {
+            let (dist_addr, _dist_size, cpu_addr, _cpu_size) =
+                match GenericInterruptController::parse(fdt) {
+                    Ok(regs) => regs,
+                    Err(err) => return Err(err),
+                };
+
+            unsafe {
+                self.gic_dist_if.init(crate::PHYS_OFFSET + dist_addr);
+                self.gic_cpu_if.init(crate::PHYS_OFFSET + cpu_addr);
+            }
+        }
+        let idx = *irq_idx;
+        let cnt = if self.gic_dist_if.nirqs > 1024 {
+            1024
+        } else {
+            self.gic_dist_if.nirqs as usize
+        };
+        let mut i: usize = 0;
+        //only support linear irq map now.
+        while i < cnt && (idx + i < 1024) {
+            irq_desc[idx + i].basic.ic_idx = ic_idx;
+            irq_desc[idx + i].basic.ic_irq = i as u32;
+            irq_desc[idx + i].basic.used = true;
+
+            i += 1;
+        }
+
+        info!("gic irq_range = ({}, {})", idx, idx + cnt);
+        self.irq_range = (idx, idx + cnt);
+        *irq_idx = idx + cnt;
+        Ok(())
+    }
+    fn irq_ack(&mut self) -> u32 {
+        unsafe { self.gic_cpu_if.irq_ack() }
+    }
+    fn irq_eoi(&mut self, irq_num: u32) {
+        unsafe { self.gic_cpu_if.irq_eoi(irq_num) }
+    }
+    fn irq_enable(&mut self, irq_num: u32) {
+        unsafe { self.gic_dist_if.irq_enable(irq_num) }
+    }
+    fn irq_disable(&mut self, irq_num: u32) {
+        unsafe { self.gic_dist_if.irq_disable(irq_num) }
+    }
+    fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
+        let off = match irq_data {
+            IrqCell::L3(0, irq, _flags) => irq as usize + 32, // SPI
+            IrqCell::L3(1, irq, _flags) => irq as usize + 16, // PPI
+            _ => return Err(Error::new(EINVAL)),
+        };
+        return Ok(off + self.irq_range.0);
+    }
+    fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
+        if hwirq >= self.gic_dist_if.nirqs {
+            None
+        } else {
+            Some(self.irq_range.0 + hwirq as usize)
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct GicDistIf {
+    pub address: usize,
+    pub ncpus: u32,
+    pub nirqs: u32,
+}
+
+impl GicDistIf {
+    pub unsafe fn init(&mut self, addr: usize) {
+        unsafe {
+            self.address = addr;
+
+            // Disable IRQ Distribution
+            self.write(GICD_CTLR, 0);
+
+            let typer = self.read(GICD_TYPER);
+            self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1;
+            self.nirqs = ((typer & 0x1f) + 1) * 32;
+            info!(
+                "gic: Distributor supports {:?} CPUs and {:?} IRQs",
+                self.ncpus, self.nirqs
+            );
+
+            // Set all SPIs to level triggered
+            for irq in (32..self.nirqs).step_by(16) {
+                self.write(GICD_ICFGR + ((irq / 16) * 4), 0);
+            }
+
+            // Disable all SPIs
+            for irq in (32..self.nirqs).step_by(32) {
+                self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff);
+            }
+
+            // Affine all SPIs to CPU0 and set priorities for all IRQs
+            for irq in 0..self.nirqs {
+                if irq > 31 {
+                    let ext_offset = GICD_ITARGETSR + (4 * (irq / 4));
+                    let int_offset = irq % 4;
+                    let mut val = self.read(ext_offset);
+                    val |= 0b0000_0001 << (8 * int_offset);
+                    self.write(ext_offset, val);
+                }
+
+                let ext_offset = GICD_IPRIORITY + (4 * (irq / 4));
+                let int_offset = irq % 4;
+                let mut val = self.read(ext_offset);
+                val |= 0b0000_0000 << (8 * int_offset);
+                self.write(ext_offset, val);
+            }
+
+            // Enable IRQ group 0 and group 1 non-secure distribution
+            self.write(GICD_CTLR, 0x3);
+        }
+    }
+
+    pub unsafe fn irq_enable(&mut self, irq: u32) {
+        unsafe {
+            let offset = GICD_ISENABLER + (4 * (irq / 32));
+            let shift = 1 << (irq % 32);
+            let mut val = self.read(offset);
+            val |= shift;
+            self.write(offset, val);
+        }
+    }
+
+    pub unsafe fn irq_disable(&mut self, irq: u32) {
+        unsafe {
+            let offset = GICD_ICENABLER + (4 * (irq / 32));
+            let shift = 1 << (irq % 32);
+            let mut val = self.read(offset);
+            val |= shift;
+            self.write(offset, val);
+        }
+    }
+
+    unsafe fn read(&self, reg: u32) -> u32 {
+        unsafe {
+            let val = read_volatile((self.address + reg as usize) as *const u32);
+            val
+        }
+    }
+
+    unsafe fn write(&mut self, reg: u32, value: u32) {
+        unsafe {
+            write_volatile((self.address + reg as usize) as *mut u32, value);
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct GicCpuIf {
+    pub address: usize,
+}
+
+impl GicCpuIf {
+    pub unsafe fn init(&mut self, addr: usize) {
+        unsafe {
+            self.address = addr;
+
+            // Enable CPU0's GIC interface
+            self.write(GICC_CTLR, 1);
+            // Set CPU0's Interrupt Priority Mask
+            self.write(GICC_PMR, 0xff);
+        }
+    }
+
+    unsafe fn irq_ack(&mut self) -> u32 {
+        unsafe {
+            let irq = self.read(GICC_IAR) & 0x1ff;
+            if irq == 1023 {
+                panic!("irq_ack: got ID 1023!!!");
+            }
+            irq
+        }
+    }
+
+    unsafe fn irq_eoi(&mut self, irq: u32) {
+        unsafe {
+            self.write(GICC_EOIR, irq);
+        }
+    }
+
+    unsafe fn read(&self, reg: u32) -> u32 {
+        unsafe {
+            let val = read_volatile((self.address + reg as usize) as *const u32);
+            val
+        }
+    }
+
+    unsafe fn write(&mut self, reg: u32, value: u32) {
+        unsafe {
+            write_volatile((self.address + reg as usize) as *mut u32, value);
+        }
+    }
+}

src/arch/aarch64/device/irqchip/gicv3.rs

@@ -0,0 +1,196 @@
+use alloc::vec::Vec;
+use core::arch::asm;
+use fdt::{node::NodeProperty, Fdt};
+
+use super::{gic::GicDistIf, InterruptController};
+use crate::{
+    dtb::{
+        get_mmio_address,
+        irqchip::{InterruptHandler, IrqCell, IrqDesc},
+    },
+    sync::CleanLockToken,
+};
+use syscall::{
+    error::{Error, EINVAL},
+    Result,
+};
+
+#[derive(Debug)]
+pub struct GicV3 {
+    pub gic_dist_if: GicDistIf,
+    pub gic_cpu_if: GicV3CpuIf,
+    pub gicrs: Vec<(usize, usize)>,
+    //TODO: GICC, GICH, GICV?
+    pub irq_range: (usize, usize),
+}
+
+impl GicV3 {
+    pub fn new() -> Self {
+        GicV3 {
+            gic_dist_if: GicDistIf::default(),
+            gic_cpu_if: GicV3CpuIf,
+            gicrs: Vec::new(),
+            irq_range: (0, 0),
+        }
+    }
+
+    pub fn parse(&mut self, fdt: &Fdt) -> Result<()> {
+        let Some(node) = fdt.find_compatible(&["arm,gic-v3"]) else {
+            return Err(Error::new(EINVAL));
+        };
+
+        // Clear current registers
+        //TODO: deinit?
+        self.gic_dist_if.address = 0;
+        self.gicrs.clear();
+
+        // Get number of GICRs
+        let gicrs = node
+            .property("#redistributor-regions")
+            .and_then(NodeProperty::as_usize)
+            .unwrap_or(1);
+
+        // Read registers
+        let mut chunks = node.reg().unwrap();
+        if let Some(gicd) = chunks.next()
+            && let Some(addr) = get_mmio_address(fdt, &node, &gicd)
+        {
+            unsafe {
+                self.gic_dist_if.init(crate::PHYS_OFFSET + addr);
+            }
+        }
+        for _ in 0..gicrs {
+            if let Some(gicr) = chunks.next() {
+                self.gicrs.push((
+                    get_mmio_address(fdt, &node, &gicr).unwrap(),
+                    gicr.size.unwrap(),
+                ));
+            }
+        }
+
+        if self.gic_dist_if.address == 0 || self.gicrs.is_empty() {
+            Err(Error::new(EINVAL))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl InterruptHandler for GicV3 {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
+}
+
+impl InterruptController for GicV3 {
+    fn irq_init(
+        &mut self,
+        fdt_opt: Option<&Fdt>,
+        irq_desc: &mut [IrqDesc; 1024],
+        ic_idx: usize,
+        irq_idx: &mut usize,
+    ) -> Result<()> {
+        if let Some(fdt) = fdt_opt {
+            self.parse(fdt)?;
+        }
+        info!("{:X?}", self);
+
+        unsafe {
+            self.gic_cpu_if.init();
+        }
+        let idx = *irq_idx;
+        let cnt = if self.gic_dist_if.nirqs > 1024 {
+            1024
+        } else {
+            self.gic_dist_if.nirqs as usize
+        };
+        let mut i: usize = 0;
+        //only support linear irq map now.
+        while i < cnt && (idx + i < 1024) {
+            irq_desc[idx + i].basic.ic_idx = ic_idx;
+            irq_desc[idx + i].basic.ic_irq = i as u32;
+            irq_desc[idx + i].basic.used = true;
+
+            i += 1;
+        }
+
+        info!("gic irq_range = ({}, {})", idx, idx + cnt);
+        self.irq_range = (idx, idx + cnt);
+        *irq_idx = idx + cnt;
+        Ok(())
+    }
+    fn irq_ack(&mut self) -> u32 {
+        let irq_num = unsafe { self.gic_cpu_if.irq_ack() };
+        irq_num
+    }
+    fn irq_eoi(&mut self, irq_num: u32) {
+        unsafe { self.gic_cpu_if.irq_eoi(irq_num) }
+    }
+    fn irq_enable(&mut self, irq_num: u32) {
+        unsafe { self.gic_dist_if.irq_enable(irq_num) }
+    }
+    fn irq_disable(&mut self, irq_num: u32) {
+        unsafe { self.gic_dist_if.irq_disable(irq_num) }
+    }
+    fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
+        let off = match irq_data {
+            IrqCell::L3(0, irq, _flags) => irq as usize + 32, // SPI
+            IrqCell::L3(1, irq, _flags) => irq as usize + 16, // PPI
+            _ => return Err(Error::new(EINVAL)),
+        };
+        return Ok(off + self.irq_range.0);
+    }
+    fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
+        if hwirq >= self.gic_dist_if.nirqs {
+            None
+        } else {
+            Some(self.irq_range.0 + hwirq as usize)
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct GicV3CpuIf;
+
+impl GicV3CpuIf {
+    pub unsafe fn init(&mut self) {
+        unsafe {
+            // Enable system register access
+            {
+                let value = 1_usize;
+                asm!("msr icc_sre_el1, {}", in(reg) value);
+            }
+            // Set control register
+            {
+                let value = 0_usize;
+                asm!("msr icc_ctlr_el1, {}", in(reg) value);
+            }
+            // Enable non-secure group 1
+            {
+                let value = 1_usize;
+                asm!("msr icc_igrpen1_el1, {}", in(reg) value);
+            }
+            // Set CPU0's Interrupt Priority Mask
+            {
+                let value = 0xFF_usize;
+                asm!("msr icc_pmr_el1, {}", in(reg) value);
+            }
+        }
+    }
+
+    unsafe fn irq_ack(&mut self) -> u32 {
+        unsafe {
+            let mut irq: usize;
+            asm!("mrs {}, icc_iar1_el1", out(reg) irq);
+            irq &= 0x1ff;
+            if irq == 1023 {
+                panic!("irq_ack: got ID 1023!!!");
+            }
+            irq as u32
+        }
+    }
+
+    unsafe fn irq_eoi(&mut self, irq: u32) {
+        unsafe {
+            asm!("msr icc_eoir1_el1, {}", in(reg) irq as usize);
+        }
+    }
+}

src/arch/aarch64/device/irqchip/irq_bcm2835.rs

@@ -0,0 +1,299 @@
+use core::ptr::{read_volatile, write_volatile};
+use fdt::{node::FdtNode, Fdt};
+
+use super::InterruptController;
+use crate::{
+    dtb::{
+        get_interrupt, get_mmio_address,
+        irqchip::{InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP},
+    },
+    sync::CleanLockToken,
+};
+use syscall::{
+    error::{Error, EINVAL},
+    Result,
+};
+
+#[inline(always)]
+fn ffs(num: u32) -> u32 {
+    let mut x = num;
+    if x == 0 {
+        return 0;
+    }
+    let mut r = 1;
+    if (x & 0xffff) == 0 {
+        x >>= 16;
+        r += 16;
+    }
+    if (x & 0xff) == 0 {
+        x >>= 8;
+        r += 8;
+    }
+    if (x & 0xf) == 0 {
+        x >>= 4;
+        r += 4;
+    }
+    if (x & 0x3) == 0 {
+        x >>= 2;
+        r += 2;
+    }
+    if (x & 0x1) == 0 {
+        r += 1;
+    }
+
+    r
+}
+
+const PENDING_0: u32 = 0x0;
+const PENDING_1: u32 = 0x4;
+const PENDING_2: u32 = 0x8;
+const ENABLE_0: u32 = 0x18;
+const ENABLE_1: u32 = 0x10;
+const ENABLE_2: u32 = 0x14;
+const DISABLE_0: u32 = 0x24;
+const DISABLE_1: u32 = 0x1c;
+const DISABLE_2: u32 = 0x20;
+
+pub struct Bcm2835ArmInterruptController {
+    pub address: usize,
+    pub irq_range: (usize, usize),
+}
+
+impl Bcm2835ArmInterruptController {
+    pub fn new() -> Self {
+        Bcm2835ArmInterruptController {
+            address: 0,
+            irq_range: (0, 0),
+        }
+    }
+    pub fn parse(fdt: &Fdt) -> Result<(usize, usize, Option<usize>)> {
+        if let Some(node) = fdt.find_compatible(&["brcm,bcm2836-armctrl-ic"]) {
+            return unsafe { Bcm2835ArmInterruptController::parse_inner(fdt, &node) };
+        } else {
+            return Err(Error::new(EINVAL));
+        }
+    }
+    unsafe fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize, Option<usize>)> {
+        unsafe {
+            //assert address_cells == 0x1, size_cells == 0x1
+            let mem = node.reg().unwrap().nth(0).unwrap();
+            let base = get_mmio_address(fdt, node, &mem).unwrap();
+            let size = mem.size.unwrap() as u32;
+            let mut ret_virq = None;
+
+            if let Some(interrupt_parent) = node.property("interrupt-parent") {
+                let phandle = interrupt_parent.as_usize().unwrap() as u32;
+                let irq = get_interrupt(fdt, node, 0).unwrap();
+                let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap();
+                //PHYS_NONSECURE_PPI only
+                let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
+                    .ic
+                    .irq_xlate(irq)
+                    .unwrap();
+                info!(
+                    "register bcm2835arm_ctrl as ic_idx {}'s child  virq = {}",
+                    ic_idx, virq
+                );
+                ret_virq = Some(virq);
+            }
+            Ok((base as usize, size as usize, ret_virq))
+        }
+    }
+
+    unsafe fn init(&mut self) {
+        unsafe {
+            debug!("IRQ BCM2835 INIT");
+            //disable all interrupt
+            self.write(DISABLE_0, 0xffff_ffff);
+            self.write(DISABLE_1, 0xffff_ffff);
+            self.write(DISABLE_2, 0xffff_ffff);
+
+            debug!("IRQ BCM2835 END");
+        }
+    }
+
+    unsafe fn read(&self, reg: u32) -> u32 {
+        unsafe {
+            let val = read_volatile((self.address + reg as usize) as *const u32);
+            val
+        }
+    }
+
+    unsafe fn write(&mut self, reg: u32, value: u32) {
+        unsafe {
+            write_volatile((self.address + reg as usize) as *mut u32, value);
+        }
+    }
+}
+
+impl InterruptController for Bcm2835ArmInterruptController {
+    fn irq_init(
+        &mut self,
+        fdt_opt: Option<&Fdt>,
+        irq_desc: &mut [IrqDesc; 1024],
+        ic_idx: usize,
+        irq_idx: &mut usize,
+    ) -> Result<()> {
+        let (base, _size, _virq) = match Bcm2835ArmInterruptController::parse(fdt_opt.unwrap()) {
+            Ok((a, b, c)) => (a, b, c),
+            Err(_) => return Err(Error::new(EINVAL)),
+        };
+        unsafe {
+            self.address = base + crate::PHYS_OFFSET;
+
+            self.init();
+            let idx = *irq_idx;
+            let cnt = 3 << 5; //3 * 32 irqs, basic == 8, reg1 = 32, reg2 = 32
+            let mut i: usize = 0;
+            //only support linear irq map now.
+            while i < cnt && (idx + i < 1024) {
+                irq_desc[idx + i].basic.ic_idx = ic_idx;
+                irq_desc[idx + i].basic.ic_irq = i as u32;
+                irq_desc[idx + i].basic.used = true;
+
+                i += 1;
+            }
+
+            info!("bcm2835 irq_range = ({}, {})", idx, idx + cnt);
+            self.irq_range = (idx, idx + cnt);
+            *irq_idx = idx + cnt;
+        }
+
+        Ok(())
+    }
+
+    fn irq_ack(&mut self) -> u32 {
+        //TODO: support smp self.read(LOCAL_IRQ_PENDING + 4 * cpu)
+        let sources = unsafe { self.read(PENDING_0) };
+        let pending_num = ffs(sources) - 1;
+        let fast_irq = [
+            7 + 32,
+            9 + 32,
+            10 + 32,
+            18 + 32,
+            19 + 32,
+            21 + 64,
+            22 + 64,
+            23 + 64,
+            24 + 64,
+            25 + 64,
+            30 + 64,
+        ];
+
+        //fast irq
+        if pending_num >= 10 && pending_num <= 20 {
+            return fast_irq[(pending_num - 10) as usize];
+        }
+
+        let pending_num = ffs(sources & 0x3ff) - 1;
+        match pending_num {
+            num @ 0..=7 => return num,
+            8 => {
+                let sources1 = unsafe { self.read(PENDING_1) };
+                let irq_0_31 = ffs(sources1) - 1;
+                return irq_0_31 + 32;
+            }
+            9 => {
+                let sources2 = unsafe { self.read(PENDING_2) };
+                let irq_32_63 = ffs(sources2) - 1;
+                return irq_32_63 + 64;
+            }
+            num => {
+                error!(
+                    "unexpected irq pending in BASIC PENDING: 0x{}, sources = 0x{:08x}",
+                    num, sources
+                );
+                return num;
+            }
+        }
+    }
+
+    fn irq_eoi(&mut self, _irq_num: u32) {}
+
+    fn irq_enable(&mut self, irq_num: u32) {
+        debug!("bcm2835 enable {} {}", irq_num, irq_num & 0x1f);
+        match irq_num {
+            num @ 0..=31 => {
+                let val = 1 << num;
+                unsafe {
+                    self.write(ENABLE_0, val);
+                }
+            }
+            num @ 32..=63 => {
+                let val = 1 << (num & 0x1f);
+                unsafe {
+                    self.write(ENABLE_1, val);
+                }
+            }
+            num @ 64..=95 => {
+                let val = 1 << (num & 0x1f);
+                unsafe {
+                    self.write(ENABLE_2, val);
+                }
+            }
+            _ => return,
+        }
+    }
+
+    fn irq_disable(&mut self, irq_num: u32) {
+        match irq_num {
+            num @ 0..=31 => {
+                let val = 1 << num;
+                unsafe {
+                    self.write(DISABLE_0, val);
+                }
+            }
+            num @ 32..=63 => {
+                let val = 1 << (num & 0x1f);
+                unsafe {
+                    self.write(DISABLE_1, val);
+                }
+            }
+            num @ 64..=95 => {
+                let val = 1 << (num & 0x1f);
+                unsafe {
+                    self.write(DISABLE_2, val);
+                }
+            }
+            _ => return,
+        }
+    }
+    fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
+        //assert interrupt-cells == 0x2
+        match irq_data {
+            IrqCell::L2(bank, irq) => {
+                //TODO: check bank && irq
+                let hwirq = (bank as usize) << 5 | (irq as usize);
+                let off = hwirq + self.irq_range.0;
+                Ok(off)
+            }
+            _ => Err(Error::new(EINVAL)),
+        }
+    }
+
+    fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
+        if hwirq > 95 {
+            None
+        } else {
+            Some(self.irq_range.0 + hwirq as usize)
+        }
+    }
+}
+
+impl InterruptHandler for Bcm2835ArmInterruptController {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {
+        unsafe {
+            let irq = self.irq_ack();
+            if let Some(virq) = self.irq_to_virq(irq)
+                && virq < 1024
+            {
+                if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler {
+                    handler.irq_handler(virq as u32, token);
+                }
+            } else {
+                error!("unexpected irq num {}", irq);
+            }
+            self.irq_eoi(irq);
+        }
+    }
+}

src/arch/aarch64/device/irqchip/irq_bcm2836.rs

@@ -0,0 +1,231 @@
+use super::InterruptController;
+use crate::{
+    arch::device::{ROOT_IC_IDX, ROOT_IC_IDX_IS_SET},
+    dtb::{
+        get_mmio_address,
+        irqchip::{InterruptHandler, IrqCell, IrqDesc},
+    },
+    sync::CleanLockToken,
+};
+use core::{
+    arch::asm,
+    ptr::{read_volatile, write_volatile},
+    sync::atomic::Ordering,
+};
+use fdt::{node::FdtNode, Fdt};
+use syscall::{
+    error::{Error, EINVAL},
+    Result,
+};
+
+const LOCAL_CONTROL: u32 = 0x000;
+const LOCAL_PRESCALER: u32 = 0x008;
+const LOCAL_GPU_ROUTING: u32 = 0x00C;
+const LOCAL_TIMER_INT_CONTROL0: u32 = 0x040;
+const LOCAL_IRQ_PENDING: u32 = 0x060;
+
+const LOCAL_IRQ_CNTPNSIRQ: u32 = 0x1;
+const LOCAL_IRQ_GPU_FAST: u32 = 0x8;
+const LOCAL_IRQ_PMU_FAST: u32 = 0x9;
+const LOCAL_IRQ_LAST: u32 = LOCAL_IRQ_PMU_FAST;
+
+#[inline(always)]
+fn ffs(num: u32) -> u32 {
+    let mut x = num;
+    if x == 0 {
+        return 0;
+    }
+    let mut r = 1;
+    if (x & 0xffff) == 0 {
+        x >>= 16;
+        r += 16;
+    }
+    if (x & 0xff) == 0 {
+        x >>= 8;
+        r += 8;
+    }
+    if (x & 0xf) == 0 {
+        x >>= 4;
+        r += 4;
+    }
+    if (x & 0x3) == 0 {
+        x >>= 2;
+        r += 2;
+    }
+    if (x & 0x1) == 0 {
+        r += 1;
+    }
+
+    r
+}
+
+pub struct Bcm2836ArmInterruptController {
+    pub address: usize,
+    pub irq_range: (usize, usize),
+    pub active_cpu: u32,
+}
+
+impl Bcm2836ArmInterruptController {
+    pub fn new() -> Self {
+        Bcm2836ArmInterruptController {
+            address: 0,
+            irq_range: (0, 0),
+            active_cpu: 0,
+        }
+    }
+    pub fn parse(fdt: &Fdt) -> Result<(usize, usize)> {
+        if let Some(node) = fdt.find_compatible(&["brcm,bcm2836-l1-intc"]) {
+            return Bcm2836ArmInterruptController::parse_inner(fdt, &node);
+        } else {
+            return Err(Error::new(EINVAL));
+        }
+    }
+    fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize)> {
+        //assert address_cells == 0x1, size_cells == 0x1
+        let reg = node.reg().unwrap().nth(0).unwrap();
+        let addr = get_mmio_address(fdt, node, &reg).unwrap();
+
+        Ok((addr, reg.size.unwrap()))
+    }
+
+    unsafe fn init(&mut self) {
+        unsafe {
+            debug!("IRQ BCM2836 INIT");
+            //init local timer freq
+            self.write(LOCAL_CONTROL, 0x0);
+            self.write(LOCAL_PRESCALER, 0x8000_0000);
+
+            //routing all irq to core
+            self.write(LOCAL_GPU_ROUTING, self.active_cpu);
+            debug!("routing all irq to core {}", self.active_cpu);
+            debug!("IRQ BCM2836 END");
+        }
+    }
+
+    unsafe fn read(&self, reg: u32) -> u32 {
+        unsafe {
+            let val = read_volatile((self.address + reg as usize) as *const u32);
+            val
+        }
+    }
+
+    unsafe fn write(&mut self, reg: u32, value: u32) {
+        unsafe {
+            write_volatile((self.address + reg as usize) as *mut u32, value);
+        }
+    }
+}
+
+impl InterruptHandler for Bcm2836ArmInterruptController {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
+}
+
+impl InterruptController for Bcm2836ArmInterruptController {
+    fn irq_init(
+        &mut self,
+        fdt_opt: Option<&Fdt>,
+        irq_desc: &mut [IrqDesc; 1024],
+        ic_idx: usize,
+        irq_idx: &mut usize,
+    ) -> Result<()> {
+        let (base, _size) = match Bcm2836ArmInterruptController::parse(fdt_opt.unwrap()) {
+            Ok((a, b)) => (a, b),
+            Err(_) => return Err(Error::new(EINVAL)),
+        };
+        unsafe {
+            self.address = base + crate::PHYS_OFFSET;
+            let cpuid: usize;
+            asm!("mrs {}, mpidr_el1", out(reg) cpuid);
+            self.active_cpu = cpuid as u32 & 0x3;
+
+            self.init();
+            let idx = *irq_idx;
+            let cnt = LOCAL_IRQ_LAST as usize;
+            let mut i: usize = 0;
+            //only support linear irq map now.
+            while i < cnt && (idx + i < 1024) {
+                irq_desc[idx + i].basic.ic_idx = ic_idx;
+                irq_desc[idx + i].basic.ic_irq = i as u32;
+                irq_desc[idx + i].basic.used = true;
+
+                i += 1;
+            }
+
+            info!("bcm2836 irq_range = ({}, {})", idx, idx + cnt);
+            self.irq_range = (idx, idx + cnt);
+            *irq_idx = idx + cnt;
+        }
+
+        //raspi 3b+ dts doesn't follow the rule to set root parent interrupt controller
+        //so we should set it manually.
+        ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed);
+        ROOT_IC_IDX_IS_SET.store(1, Ordering::Relaxed);
+
+        Ok(())
+    }
+
+    fn irq_ack(&mut self) -> u32 {
+        let cpuid: usize;
+        unsafe {
+            asm!("mrs {}, mpidr_el1", out(reg) cpuid);
+        }
+        let cpu = cpuid as u32 & 0x3;
+        let sources: u32 = unsafe { self.read(LOCAL_IRQ_PENDING + 4 * cpu) };
+        ffs(sources) - 1
+    }
+
+    fn irq_eoi(&mut self, _irq_num: u32) {}
+
+    fn irq_enable(&mut self, irq_num: u32) {
+        debug!("bcm2836 enable {}", irq_num);
+        match irq_num {
+            LOCAL_IRQ_CNTPNSIRQ => unsafe {
+                let cpuid: usize;
+                asm!("mrs {}, mpidr_el1", out(reg) cpuid);
+                let cpu = cpuid as u32 & 0x3;
+                let mut reg_val = self.read(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu);
+                reg_val |= 0x2;
+                self.write(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu, reg_val);
+            },
+            LOCAL_IRQ_GPU_FAST => {
+                //GPU IRQ always enable
+            }
+            _ => {
+                //ignore
+            }
+        }
+    }
+
+    fn irq_disable(&mut self, irq_num: u32) {
+        match irq_num {
+            LOCAL_IRQ_CNTPNSIRQ => unsafe {
+                let cpuid: usize;
+                asm!("mrs {}, mpidr_el1", out(reg) cpuid);
+                let cpu = cpuid as u32 & 0x3;
+                let mut reg_val = self.read(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu);
+                reg_val &= !0x2;
+                self.write(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu, reg_val);
+            },
+            LOCAL_IRQ_GPU_FAST => {
+                //GPU IRQ always enable
+            }
+            _ => {
+                //ignore
+            }
+        }
+    }
+    fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
+        //assert interrupt-cells == 0x2
+        match irq_data {
+            IrqCell::L2(irq, _) => Ok(irq as usize + self.irq_range.0),
+            _ => Err(Error::new(EINVAL)),
+        }
+    }
+    fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
+        if hwirq > LOCAL_IRQ_LAST {
+            None
+        } else {
+            Some(self.irq_range.0 + hwirq as usize)
+        }
+    }
+}

src/arch/aarch64/device/irqchip/mod.rs

@@ -0,0 +1,41 @@
+use crate::dtb::irqchip::{InterruptController, IRQ_CHIP};
+use alloc::boxed::Box;
+use fdt::{node::FdtNode, Fdt};
+
+pub(crate) mod gic;
+pub(crate) mod gicv3;
+mod irq_bcm2835;
+mod irq_bcm2836;
+mod null;
+
+pub(crate) fn new_irqchip(ic_str: &str) -> Option<Box<dyn InterruptController>> {
+    if ic_str.contains("arm,gic-v3") {
+        Some(Box::new(gicv3::GicV3::new()))
+    } else if ic_str.contains("arm,cortex-a15-gic") || ic_str.contains("arm,gic-400") {
+        Some(Box::new(gic::GenericInterruptController::new()))
+    } else if ic_str.contains("brcm,bcm2836-l1-intc") {
+        Some(Box::new(irq_bcm2836::Bcm2836ArmInterruptController::new()))
+    } else if ic_str.contains("brcm,bcm2836-armctrl-ic") {
+        Some(Box::new(irq_bcm2835::Bcm2835ArmInterruptController::new()))
+    } else {
+        warn!("no driver for interrupt controller {:?}", ic_str);
+        //TODO: return None and handle it properly
+        Some(Box::new(null::Null))
+    }
+}
+
+pub(crate) fn ic_for_chip(fdt: &Fdt, node: &FdtNode) -> Option<usize> {
+    if let Some(_) = node.property("interrupts-extended") {
+        error!("multi-parented device not supported");
+        None
+    } else if let Some(irqc_phandle) = node
+        .property("interrupt-parent")
+        .or(fdt.root().property("interrupt-parent"))
+        .and_then(|f| f.as_usize())
+    {
+        unsafe { IRQ_CHIP.phandle_to_ic_idx(irqc_phandle as u32) }
+    } else {
+        error!("no irq parent found");
+        None
+    }
+}

src/arch/aarch64/device/irqchip/null.rs

@@ -0,0 +1,41 @@
+use fdt::Fdt;
+use syscall::{
+    error::{Error, EINVAL},
+    Result,
+};
+
+use super::InterruptController;
+use crate::{
+    dtb::irqchip::{InterruptHandler, IrqCell, IrqDesc},
+    sync::CleanLockToken,
+};
+
+pub struct Null;
+
+impl InterruptHandler for Null {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
+}
+
+impl InterruptController for Null {
+    fn irq_init(
+        &mut self,
+        _fdt_opt: Option<&Fdt>,
+        _irq_desc: &mut [IrqDesc; 1024],
+        _ic_idx: usize,
+        _irq_idx: &mut usize,
+    ) -> Result<()> {
+        Ok(())
+    }
+    fn irq_ack(&mut self) -> u32 {
+        unimplemented!()
+    }
+    fn irq_eoi(&mut self, _irq_num: u32) {}
+    fn irq_enable(&mut self, _irq_num: u32) {}
+    fn irq_disable(&mut self, _irq_num: u32) {}
+    fn irq_xlate(&self, _irq_data: IrqCell) -> Result<usize> {
+        Err(Error::new(EINVAL))
+    }
+    fn irq_to_virq(&self, _hwirq: u32) -> Option<usize> {
+        None
+    }
+}

src/arch/aarch64/device/mod.rs

@@ -0,0 +1,60 @@
+use crate::info;
+use core::sync::atomic::{AtomicUsize, Ordering};
+use fdt::Fdt;
+
+pub mod cpu;
+pub mod generic_timer;
+pub mod irqchip;
+pub mod rtc;
+pub mod serial;
+
+use crate::dtb::irqchip::IRQ_CHIP;
+use irqchip::ic_for_chip;
+
+pub static ROOT_IC_IDX: AtomicUsize = AtomicUsize::new(0);
+pub static ROOT_IC_IDX_IS_SET: AtomicUsize = AtomicUsize::new(0);
+
+unsafe fn init_root_ic(fdt: &Fdt) {
+    unsafe {
+        let is_set = ROOT_IC_IDX_IS_SET.load(Ordering::Relaxed);
+        if is_set != 0 {
+            let ic_idx = ROOT_IC_IDX.load(Ordering::Relaxed);
+            info!("Already selected {} as root ic", ic_idx);
+            return;
+        }
+
+        let root_irqc_phandle = fdt
+            .root()
+            .property("interrupt-parent")
+            .unwrap()
+            .as_usize()
+            .unwrap();
+        let ic_idx = IRQ_CHIP
+            .phandle_to_ic_idx(root_irqc_phandle as u32)
+            .unwrap();
+        info!("select {} as root ic", ic_idx);
+        ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed);
+    }
+}
+
+pub unsafe fn init_devicetree(fdt: &Fdt) {
+    unsafe {
+        info!("IRQCHIP INIT");
+        crate::dtb::irqchip::init(&fdt);
+        init_root_ic(&fdt);
+        info!("GIT INIT");
+        generic_timer::init(fdt);
+        info!("SERIAL INIT");
+        serial::init(fdt);
+        info!("RTC INIT");
+        rtc::init(fdt);
+    }
+}
+
+pub struct ArchPercpuMisc;
+
+impl ArchPercpuMisc {
+    pub const fn default() -> Self {
+        Self
+    }
+}

src/arch/aarch64/device/rtc.rs

@@ -0,0 +1,41 @@
+use crate::{dtb::get_mmio_address, sync::CleanLockToken, time};
+use core::ptr::read_volatile;
+
+static RTC_DR: usize = 0x000;
+
+pub unsafe fn init(fdt: &fdt::Fdt) {
+    if let Some(node) = fdt.find_compatible(&["arm,pl031"]) {
+        match node
+            .reg()
+            .and_then(|mut iter| iter.next())
+            .and_then(|region| get_mmio_address(fdt, &node, &region))
+        {
+            Some(phys) => {
+                let mut rtc = Pl031rtc { phys };
+                info!("PL031 RTC at {:#x}", rtc.phys);
+                let mut token = unsafe { CleanLockToken::new() };
+                *time::START.lock(token.token()) = (rtc.time() as u128) * time::NANOS_PER_SEC;
+            }
+            None => {
+                warn!("No PL031 RTC registers");
+            }
+        }
+    } else {
+        warn!("No PL031 RTC found");
+    }
+}
+
+struct Pl031rtc {
+    pub phys: usize,
+}
+
+impl Pl031rtc {
+    unsafe fn read(&self, reg: usize) -> u32 {
+        unsafe { read_volatile((crate::PHYS_OFFSET + self.phys + reg) as *const u32) }
+    }
+
+    pub fn time(&mut self) -> u64 {
+        let seconds = unsafe { self.read(RTC_DR) } as u64;
+        seconds
+    }
+}

src/arch/aarch64/device/serial.rs

@@ -0,0 +1,59 @@
+use alloc::boxed::Box;
+use fdt::Fdt;
+
+pub use crate::dtb::serial::COM1;
+use crate::{
+    arch::device::irqchip::ic_for_chip,
+    dtb::{
+        get_interrupt,
+        irqchip::{register_irq, InterruptHandler, IRQ_CHIP},
+    },
+    scheme::irq::irq_trigger,
+    sync::CleanLockToken,
+};
+
+pub struct Com1Irq {}
+
+impl InterruptHandler for Com1Irq {
+    fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) {
+        COM1.lock().receive(token);
+        unsafe {
+            // FIXME add_irq accepts a u8 as irq number
+            // PercpuBlock::current().stats.add_irq(irq);
+            irq_trigger(irq.try_into().unwrap(), token);
+            IRQ_CHIP.irq_eoi(irq);
+        }
+    }
+}
+
+pub unsafe fn init(fdt: &Fdt) {
+    unsafe {
+        //TODO: find actual serial device, not just any PL011
+        if let Some(node) = fdt.find_compatible(&["arm,pl011"]) {
+            let irq = get_interrupt(fdt, &node, 0).unwrap();
+            if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
+                let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
+                    .ic
+                    .irq_xlate(irq)
+                    .unwrap();
+                info!("serial_port virq = {}", virq);
+                register_irq(virq as u32, Box::new(Com1Irq {}));
+                IRQ_CHIP.irq_enable(virq as u32);
+            } else {
+                error!("serial port irq parent not found");
+            }
+        }
+        COM1.lock().enable_irq();
+    }
+}
+
+pub unsafe fn init_acpi(irq: u32) {
+    unsafe {
+        //TODO: what should chip index be?
+        let virq = IRQ_CHIP.irq_chip_list.chips[0].ic.irq_to_virq(irq).unwrap();
+        info!("serial_port virq = {}", virq);
+        register_irq(virq as u32, Box::new(Com1Irq {}));
+        IRQ_CHIP.irq_enable(virq as u32);
+        COM1.lock().enable_irq();
+    }
+}

src/arch/aarch64/interrupt/exception.rs

@@ -0,0 +1,236 @@
+use ::syscall::Exception;
+use rmm::VirtualAddress;
+
+use crate::{
+    context::signal::excp_handler,
+    exception_stack,
+    memory::{ArchIntCtx, GenericPfFlags},
+    sync::CleanLockToken,
+    syscall,
+};
+
+use super::InterruptStack;
+
+exception_stack!(synchronous_exception_at_el1_with_sp0, |stack| {
+    println!("Synchronous exception at EL1 with SP0");
+    stack.trace();
+    loop {}
+});
+
+fn exception_code(esr: usize) -> u8 {
+    ((esr >> 26) & 0x3f) as u8
+}
+fn iss(esr: usize) -> u32 {
+    (esr & 0x01ff_ffff) as u32
+}
+
+unsafe fn far_el1() -> usize {
+    unsafe {
+        let ret: usize;
+        core::arch::asm!("mrs {}, far_el1", out(reg) ret);
+        ret
+    }
+}
+
+unsafe fn instr_data_abort_inner(
+    stack: &mut InterruptStack,
+    from_user: bool,
+    instr_not_data: bool,
+    _from: &str,
+) -> bool {
+    unsafe {
+        let iss = iss(stack.iret.esr_el1);
+        let fsc = iss & 0x3F;
+        //dbg!(fsc);
+
+        let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111;
+        //let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111;
+        let write_not_read_if_data = iss & (1 << 6) != 0;
+
+        let mut flags = GenericPfFlags::empty();
+        flags.set(GenericPfFlags::PRESENT, !was_translation_fault);
+
+        // TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64
+        // doesn't appear to require that flag to be set if the read alone would trigger a fault.
+        flags.set(
+            GenericPfFlags::INVOLVED_WRITE,
+            write_not_read_if_data && !instr_not_data,
+        );
+        flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data);
+        flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user);
+
+        let faulting_addr = VirtualAddress::new(far_el1());
+        //dbg!(faulting_addr, flags, from);
+
+        crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok()
+    }
+}
+
+unsafe fn cntfrq_el0() -> usize {
+    unsafe {
+        let ret: usize;
+        core::arch::asm!("mrs {}, cntfrq_el0", out(reg) ret);
+        ret
+    }
+}
+
+unsafe fn cntpct_el0() -> usize {
+    unsafe {
+        let ret: usize;
+        core::arch::asm!("mrs {}, cntpct_el0", out(reg) ret);
+        ret
+    }
+}
+
+unsafe fn cntvct_el0() -> usize {
+    unsafe {
+        let ret: usize;
+        core::arch::asm!("mrs {}, cntvct_el0", out(reg) ret);
+        ret
+    }
+}
+
+unsafe fn instr_trapped_msr_mrs_inner(
+    stack: &mut InterruptStack,
+    _from_user: bool,
+    _instr_not_data: bool,
+    _from: &str,
+) -> bool {
+    unsafe {
+        let iss = iss(stack.iret.esr_el1);
+        // let res0 = (iss & 0x1C0_0000) >> 22;
+        let op0 = (iss & 0x030_0000) >> 20;
+        let op2 = (iss & 0x00e_0000) >> 17;
+        let op1 = (iss & 0x001_c000) >> 14;
+        let crn = (iss & 0x000_3c00) >> 10;
+        let rt = (iss & 0x000_03e0) >> 5;
+        let crm = (iss & 0x000_001e) >> 1;
+        let dir = iss & 0x000_0001;
+
+        /*
+        print!("iss=0x{:x}, res0=0b{:03b}, op0=0b{:02b}\n
+                op2=0b{:03b}, op1=0b{:03b}, crn=0b{:04b}\n
+                rt=0b{:05b}, crm=0b{:04b}, dir=0b{:b}\n",
+                iss, res0, op0, op2, op1, crn, rt, crm, dir);
+        */
+
+        match (op0, op1, crn, crm, op2, dir) {
+            //MRS <Xt>, CNTFRQ_EL0
+            (0b11, 0b011, 0b1110, 0b0000, 0b000, 0b1) => {
+                let reg_val = cntfrq_el0();
+                stack.store_reg(rt as usize, reg_val);
+                //skip faulting instruction, A64 instructions are always 32-bits
+                stack.iret.elr_el1 += 4;
+                return true;
+            }
+            //MRS <Xt>, CNTPCT_EL0
+            (0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => {
+                let reg_val = cntpct_el0();
+                stack.store_reg(rt as usize, reg_val);
+                //skip faulting instruction, A64 instructions are always 32-bits
+                stack.iret.elr_el1 += 4;
+                return true;
+            }
+            //MRS <Xt>, CNTVCT_EL0
+            (0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => {
+                let reg_val = cntvct_el0();
+                stack.store_reg(rt as usize, reg_val);
+                //skip faulting instruction, A64 instructions are always 32-bits
+                stack.iret.elr_el1 += 4;
+                return true;
+            }
+            _ => {}
+        }
+
+        false
+    }
+}
+
+exception_stack!(synchronous_exception_at_el1_with_spx, |stack| {
+    unsafe {
+        if !pf_inner(
+            stack,
+            exception_code(stack.iret.esr_el1),
+            "sync_exc_el1_spx",
+        ) {
+            println!("Synchronous exception at EL1 with SPx");
+            if exception_code(stack.iret.esr_el1) == 0b100101 {
+                let far_el1 = far_el1();
+                println!("FAR_EL1 = 0x{:08x}", far_el1);
+            } else if exception_code(stack.iret.esr_el1) == 0b100100 {
+                let far_el1 = far_el1();
+                println!("USER FAR_EL1 = 0x{:08x}", far_el1);
+            }
+            stack.trace();
+            loop {}
+        }
+    }
+});
+unsafe fn pf_inner(stack: &mut InterruptStack, ty: u8, from: &str) -> bool {
+    unsafe {
+        match ty {
+            // "Data Abort taken from a lower Exception level"
+            0b100100 => instr_data_abort_inner(stack, true, false, from),
+            // "Data Abort taken without a change in Exception level"
+            0b100101 => instr_data_abort_inner(stack, false, false, from),
+            // "Instruction Abort taken from a lower Exception level"
+            0b100000 => instr_data_abort_inner(stack, true, true, from),
+            // "Instruction Abort taken without a change in Exception level"
+            0b100001 => instr_data_abort_inner(stack, false, true, from),
+            // "Trapped MSR, MRS or System instruction execution in AArch64 state"
+            0b011000 => instr_trapped_msr_mrs_inner(stack, true, true, from),
+
+            _ => return false,
+        }
+    }
+}
+
+exception_stack!(synchronous_exception_at_el0, |stack| {
+    unsafe {
+        match exception_code(stack.iret.esr_el1) {
+            0b010101 => {
+                let scratch = &stack.scratch;
+                let mut token = CleanLockToken::new();
+                let ret = syscall::syscall(
+                    scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4,
+                    scratch.x5, &mut token,
+                );
+                stack.scratch.x0 = ret;
+            }
+
+            ty => {
+                if !pf_inner(stack, ty as u8, "sync_exc_el0") {
+                    error!(
+                        "FATAL: Not an SVC induced synchronous exception (ty={:b})",
+                        ty
+                    );
+                    println!("FAR_EL1: {:#0x}", far_el1());
+                    //crate::debugger::debugger(None);
+                    stack.trace();
+                    excp_handler(Exception {
+                        kind: 0, // TODO
+                    });
+                }
+            }
+        }
+    }
+});
+
+exception_stack!(unhandled_exception, |stack| {
+    println!("Unhandled exception");
+    stack.trace();
+    loop {}
+});
+
+impl ArchIntCtx for InterruptStack {
+    fn ip(&self) -> usize {
+        self.iret.elr_el1
+    }
+    fn recover_and_efault(&mut self) {
+        // Set the return value to nonzero to indicate usercopy failure (EFAULT), and emulate the
+        // return instruction by setting the return pointer to the saved LR value.
+
+        self.iret.elr_el1 = self.preserved.x30;
+        self.scratch.x0 = 1;
+    }
+}

src/arch/aarch64/interrupt/handler.rs

@@ -0,0 +1,420 @@
+use crate::{panic, syscall::IntRegisters};
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct ScratchRegisters {
+    pub x0: usize,
+    pub x1: usize,
+    pub x2: usize,
+    pub x3: usize,
+    pub x4: usize,
+    pub x5: usize,
+    pub x6: usize,
+    pub x7: usize,
+    pub x8: usize,
+    pub x9: usize,
+    pub x10: usize,
+    pub x11: usize,
+    pub x12: usize,
+    pub x13: usize,
+    pub x14: usize,
+    pub x15: usize,
+    pub x16: usize,
+    pub x17: usize,
+    pub x18: usize,
+    pub _padding: usize,
+}
+
+impl ScratchRegisters {
+    pub fn dump(&self) {
+        println!("X0:    {:>016X}", { self.x0 });
+        println!("X1:    {:>016X}", { self.x1 });
+        println!("X2:    {:>016X}", { self.x2 });
+        println!("X3:    {:>016X}", { self.x3 });
+        println!("X4:    {:>016X}", { self.x4 });
+        println!("X5:    {:>016X}", { self.x5 });
+        println!("X6:    {:>016X}", { self.x6 });
+        println!("X7:    {:>016X}", { self.x7 });
+        println!("X8:    {:>016X}", { self.x8 });
+        println!("X9:    {:>016X}", { self.x9 });
+        println!("X10:   {:>016X}", { self.x10 });
+        println!("X11:   {:>016X}", { self.x11 });
+        println!("X12:   {:>016X}", { self.x12 });
+        println!("X13:   {:>016X}", { self.x13 });
+        println!("X14:   {:>016X}", { self.x14 });
+        println!("X15:   {:>016X}", { self.x15 });
+        println!("X16:   {:>016X}", { self.x16 });
+        println!("X17:   {:>016X}", { self.x17 });
+        println!("X18:   {:>016X}", { self.x18 });
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct PreservedRegisters {
+    //TODO: is X30 a preserved register?
+    pub x19: usize,
+    pub x20: usize,
+    pub x21: usize,
+    pub x22: usize,
+    pub x23: usize,
+    pub x24: usize,
+    pub x25: usize,
+    pub x26: usize,
+    pub x27: usize,
+    pub x28: usize,
+    pub x29: usize,
+    pub x30: usize,
+}
+
+impl PreservedRegisters {
+    pub fn dump(&self) {
+        println!("X19:   {:>016X}", { self.x19 });
+        println!("X20:   {:>016X}", { self.x20 });
+        println!("X21:   {:>016X}", { self.x21 });
+        println!("X22:   {:>016X}", { self.x22 });
+        println!("X23:   {:>016X}", { self.x23 });
+        println!("X24:   {:>016X}", { self.x24 });
+        println!("X25:   {:>016X}", { self.x25 });
+        println!("X26:   {:>016X}", { self.x26 });
+        println!("X27:   {:>016X}", { self.x27 });
+        println!("X28:   {:>016X}", { self.x28 });
+        println!("X29:   {:>016X}", { self.x29 });
+        println!("X30:   {:>016X}", { self.x30 });
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct IretRegisters {
+    // occurred
+    // The exception vector disambiguates at which EL the interrupt
+    pub sp_el0: usize, // Shouldn't be used if interrupt occurred at EL1
+    pub esr_el1: usize,
+    pub spsr_el1: usize,
+    pub elr_el1: usize,
+}
+
+impl IretRegisters {
+    pub fn dump(&self) {
+        println!("ELR_EL1: {:>016X}", { self.elr_el1 });
+        println!("SPSR_EL1: {:>016X}", { self.spsr_el1 });
+        println!("ESR_EL1: {:>016X}", { self.esr_el1 });
+        println!("SP_EL0: {:>016X}", { self.sp_el0 });
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct InterruptStack {
+    pub iret: IretRegisters,
+    pub scratch: ScratchRegisters,
+    pub preserved: PreservedRegisters,
+}
+
+impl InterruptStack {
+    pub fn init(&mut self) {}
+    pub fn frame_pointer(&self) -> usize {
+        self.preserved.x29
+    }
+    pub fn stack_pointer(&self) -> usize {
+        self.iret.sp_el0
+    }
+    pub fn set_stack_pointer(&mut self, sp: usize) {
+        self.iret.sp_el0 = sp;
+    }
+    pub fn sig_archdep_reg(&self) -> usize {
+        self.scratch.x0
+    }
+    pub fn set_instr_pointer(&mut self, ip: usize) {
+        self.iret.elr_el1 = ip;
+    }
+    pub fn instr_pointer(&self) -> usize {
+        self.iret.elr_el1
+    }
+    pub fn set_arg1(&mut self, arg_opt: Option<usize>) {
+        if let Some(arg) = arg_opt {
+            self.scratch.x1 = arg;
+        }
+    }
+    pub fn dump(&self) {
+        self.iret.dump();
+        self.scratch.dump();
+        self.preserved.dump();
+    }
+    pub fn trace(&self) {
+        self.dump();
+        unsafe {
+            panic::user_stack_trace(&self);
+            panic::stack_trace();
+        }
+    }
+
+    /// Saves all registers to a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn save(&self, all: &mut IntRegisters) {
+        /*TODO: aarch64 registers
+        all.elr_el1 = self.iret.elr_el1;
+        all.spsr_el1 = self.iret.spsr_el1;
+        all.esr_el1 = self.iret.esr_el1;
+        all.sp_el0 = self.iret.sp_el0;
+        all.padding = 0;
+        */
+        all.x30 = self.preserved.x30;
+        all.x29 = self.preserved.x29;
+        all.x28 = self.preserved.x28;
+        all.x27 = self.preserved.x27;
+        all.x26 = self.preserved.x26;
+        all.x25 = self.preserved.x25;
+        all.x24 = self.preserved.x24;
+        all.x23 = self.preserved.x23;
+        all.x22 = self.preserved.x22;
+        all.x21 = self.preserved.x21;
+        all.x20 = self.preserved.x20;
+        all.x19 = self.preserved.x19;
+        all.x18 = self.scratch.x18;
+        all.x17 = self.scratch.x17;
+        all.x16 = self.scratch.x16;
+        all.x15 = self.scratch.x15;
+        all.x14 = self.scratch.x14;
+        all.x13 = self.scratch.x13;
+        all.x12 = self.scratch.x12;
+        all.x11 = self.scratch.x11;
+        all.x10 = self.scratch.x10;
+        all.x9 = self.scratch.x9;
+        all.x8 = self.scratch.x8;
+        all.x7 = self.scratch.x7;
+        all.x6 = self.scratch.x6;
+        all.x5 = self.scratch.x5;
+        all.x4 = self.scratch.x4;
+        all.x3 = self.scratch.x3;
+        all.x2 = self.scratch.x2;
+        all.x1 = self.scratch.x1;
+        all.x0 = self.scratch.x0;
+    }
+
+    /// Loads all registers from a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn load(&mut self, all: &IntRegisters) {
+        /*TODO: aarch64 registers
+        self.iret.elr_el1 = all.elr_el1;
+        self.iret.spsr_el1 = all.spsr_el1;
+        self.iret.esr_el1 = all.esr_el1;
+        self.iret.sp_el0 = all.sp_el0;
+        */
+        self.preserved.x30 = all.x30;
+        self.preserved.x29 = all.x29;
+        self.preserved.x28 = all.x28;
+        self.preserved.x27 = all.x27;
+        self.preserved.x26 = all.x26;
+        self.preserved.x25 = all.x25;
+        self.preserved.x24 = all.x24;
+        self.preserved.x23 = all.x23;
+        self.preserved.x22 = all.x22;
+        self.preserved.x21 = all.x21;
+        self.preserved.x20 = all.x20;
+        self.preserved.x19 = all.x19;
+        self.scratch.x18 = all.x18;
+        self.scratch.x17 = all.x17;
+        self.scratch.x16 = all.x16;
+        self.scratch.x15 = all.x15;
+        self.scratch.x14 = all.x14;
+        self.scratch.x13 = all.x13;
+        self.scratch.x12 = all.x12;
+        self.scratch.x11 = all.x11;
+        self.scratch.x10 = all.x10;
+        self.scratch.x9 = all.x9;
+        self.scratch.x8 = all.x8;
+        self.scratch.x7 = all.x7;
+        self.scratch.x6 = all.x6;
+        self.scratch.x5 = all.x5;
+        self.scratch.x4 = all.x4;
+        self.scratch.x3 = all.x3;
+        self.scratch.x2 = all.x2;
+        self.scratch.x1 = all.x1;
+        self.scratch.x0 = all.x0;
+    }
+
+    /// Store a specific generic registers
+    pub fn store_reg(&mut self, idx: usize, val: usize) {
+        match idx {
+            0 => self.scratch.x0 = val,
+            1 => self.scratch.x1 = val,
+            2 => self.scratch.x2 = val,
+            3 => self.scratch.x3 = val,
+            4 => self.scratch.x4 = val,
+            5 => self.scratch.x5 = val,
+            6 => self.scratch.x6 = val,
+            7 => self.scratch.x7 = val,
+            8 => self.scratch.x8 = val,
+            9 => self.scratch.x9 = val,
+            10 => self.scratch.x10 = val,
+            11 => self.scratch.x11 = val,
+            12 => self.scratch.x12 = val,
+            13 => self.scratch.x13 = val,
+            14 => self.scratch.x14 = val,
+            15 => self.scratch.x15 = val,
+            16 => self.scratch.x16 = val,
+            17 => self.scratch.x17 = val,
+            18 => self.scratch.x18 = val,
+            19 => self.preserved.x19 = val,
+            20 => self.preserved.x20 = val,
+            21 => self.preserved.x21 = val,
+            22 => self.preserved.x22 = val,
+            23 => self.preserved.x23 = val,
+            24 => self.preserved.x24 = val,
+            25 => self.preserved.x25 = val,
+            26 => self.preserved.x26 = val,
+            27 => self.preserved.x27 = val,
+            28 => self.preserved.x28 = val,
+            29 => self.preserved.x29 = val,
+            30 => self.preserved.x30 = val,
+            _ => {}
+        }
+    }
+
+    //TODO
+    pub fn set_singlestep(&mut self, _singlestep: bool) {}
+}
+
+#[macro_export]
+macro_rules! push_scratch {
+    () => {
+        "
+        // Push scratch registers
+        str     x18,      [sp, #-16]!
+        stp     x16, x17, [sp, #-16]!
+        stp     x14, x15, [sp, #-16]!
+        stp     x12, x13, [sp, #-16]!
+        stp     x10, x11, [sp, #-16]!
+        stp     x8, x9, [sp, #-16]!
+        stp     x6, x7, [sp, #-16]!
+        stp     x4, x5, [sp, #-16]!
+        stp     x2, x3, [sp, #-16]!
+        stp     x0, x1, [sp, #-16]!
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! pop_scratch {
+    () => {
+        "
+        // Pop scratch registers
+        ldp     x0, x1, [sp], #16
+        ldp     x2, x3, [sp], #16
+        ldp     x4, x5, [sp], #16
+        ldp     x6, x7, [sp], #16
+        ldp     x8, x9, [sp], #16
+        ldp     x10, x11, [sp], #16
+        ldp     x12, x13, [sp], #16
+        ldp     x14, x15, [sp], #16
+        ldp     x16, x17, [sp], #16
+        ldr     x18,      [sp], #16
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! push_preserved {
+    () => {
+        "
+        // Push preserved registers
+        stp     x29, x30, [sp, #-16]!
+        stp     x27, x28, [sp, #-16]!
+        stp     x25, x26, [sp, #-16]!
+        stp     x23, x24, [sp, #-16]!
+        stp     x21, x22, [sp, #-16]!
+        stp     x19, x20, [sp, #-16]!
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! pop_preserved {
+    () => {
+        "
+        // Pop preserved registers
+        ldp     x19, x20, [sp], #16
+        ldp     x21, x22, [sp], #16
+        ldp     x23, x24, [sp], #16
+        ldp     x25, x26, [sp], #16
+        ldp     x27, x28, [sp], #16
+        ldp     x29, x30, [sp], #16
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! push_special {
+    () => {
+        "
+        mrs     x14, spsr_el1
+        mrs     x15, elr_el1
+        stp     x14, x15, [sp, #-16]!
+
+        mrs     x14, sp_el0
+        mrs     x15, esr_el1
+        stp     x14, x15, [sp, #-16]!
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! pop_special {
+    () => {
+        "
+        ldp     x14, x15, [sp], 16
+        msr     esr_el1, x15
+        msr     sp_el0, x14
+
+        ldp     x14, x15, [sp], 16
+        msr     elr_el1, x15
+        msr     spsr_el1, x14
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! exception_stack {
+    ($name:ident, |$stack:ident| $code:block) => {
+        #[unsafe(naked)]
+        #[unsafe(no_mangle)]
+        pub unsafe extern "C" fn $name(stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
+            unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
+                $code
+            }
+            core::arch::naked_asm!(
+                // Backup all userspace registers to stack
+                push_preserved!(),
+                push_scratch!(),
+                push_special!(),
+
+                // Call inner function with pointer to stack
+                "mov x29, sp",
+                "mov x0, sp",
+                "bl {}",
+
+                // Restore all userspace registers
+                pop_special!(),
+                pop_scratch!(),
+                pop_preserved!(),
+
+                "eret",
+
+                sym inner,
+            );
+        }
+    };
+}
+#[unsafe(naked)]
+pub unsafe extern "C" fn enter_usermode() -> ! {
+    core::arch::naked_asm!(
+        "blr x28",
+        // Restore all userspace registers
+        pop_special!(),
+        pop_scratch!(),
+        pop_preserved!(),
+        "eret",
+    );
+}

src/arch/aarch64/interrupt/irq.rs

@@ -0,0 +1,56 @@
+use crate::{arch::device::ROOT_IC_IDX, dtb::irqchip::IRQ_CHIP, sync::CleanLockToken};
+use core::sync::atomic::Ordering;
+
+// use crate::percpu::PercpuBlock;
+
+unsafe fn irq_ack() -> (u32, Option<usize>) {
+    unsafe {
+        let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic;
+        let irq = ic.irq_ack();
+        (irq, ic.irq_to_virq(irq))
+    }
+}
+
+exception_stack!(irq_at_el0, |_stack| {
+    unsafe {
+        let mut token = CleanLockToken::new();
+        let (irq, virq) = irq_ack();
+        if let Some(virq) = virq
+            && virq < 1024
+        {
+            IRQ_CHIP.trigger_virq(virq as u32, &mut token);
+        } else {
+            println!("unexpected irq num {}", irq);
+        }
+    }
+});
+
+exception_stack!(irq_at_el1, |_stack| {
+    unsafe {
+        let mut token = CleanLockToken::new();
+        let (irq, virq) = irq_ack();
+        if let Some(virq) = virq
+            && virq < 1024
+        {
+            IRQ_CHIP.trigger_virq(virq as u32, &mut token);
+        } else {
+            println!("unexpected irq num {}", irq);
+        }
+    }
+});
+
+/*
+pub unsafe fn irq_handler_gentimer(irq: u32) {
+    GENTIMER.clear_irq();
+    {
+        *time::OFFSET.lock() += GENTIMER.clk_freq as u128;
+    }
+
+    timeout::trigger();
+
+    context::switch::tick();
+
+    trigger(irq);
+    GENTIMER.reload_count();
+}
+*/

src/arch/aarch64/interrupt/mod.rs

@@ -0,0 +1,49 @@
+//! Interrupt instructions
+
+use core::arch::asm;
+
+#[macro_use]
+pub mod handler;
+
+pub mod exception;
+pub mod irq;
+pub mod syscall;
+pub mod trace;
+
+pub use self::handler::InterruptStack;
+
+/// Clear interrupts
+#[inline(always)]
+pub unsafe fn disable() {
+    unsafe {
+        asm!("msr daifset, #2");
+    }
+}
+
+/// Set interrupts and halt
+/// This will atomically wait for the next interrupt
+/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_halt() {
+    unsafe {
+        asm!("wfi", "msr daifclr, #2", "nop");
+    }
+}
+
+/// Set interrupts and nop
+/// This will enable interrupts and allow the IF flag to be processed
+/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_nop() {
+    unsafe {
+        asm!("msr daifclr, #2", "nop");
+    }
+}
+
+/// Halt instruction
+#[inline(always)]
+pub unsafe fn halt() {
+    unsafe {
+        asm!("wfi");
+    }
+}

src/arch/aarch64/interrupt/syscall.rs

@@ -0,0 +1,49 @@
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn do_exception_unhandled() {}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn do_exception_synchronous() {}
+
+#[allow(dead_code)]
+#[repr(C, packed)]
+pub struct SyscallStack {
+    pub elr_el1: usize,
+    pub padding: usize,
+    pub tpidr: usize,
+    pub tpidrro: usize,
+    pub rflags: usize,
+    pub esr: usize,
+    pub sp: usize,
+    pub lr: usize,
+    pub fp: usize,
+    pub x28: usize,
+    pub x27: usize,
+    pub x26: usize,
+    pub x25: usize,
+    pub x24: usize,
+    pub x23: usize,
+    pub x22: usize,
+    pub x21: usize,
+    pub x20: usize,
+    pub x19: usize,
+    pub x18: usize,
+    pub x17: usize,
+    pub x16: usize,
+    pub x15: usize,
+    pub x14: usize,
+    pub x13: usize,
+    pub x12: usize,
+    pub x11: usize,
+    pub x10: usize,
+    pub x9: usize,
+    pub x8: usize,
+    pub x7: usize,
+    pub x6: usize,
+    pub x5: usize,
+    pub x4: usize,
+    pub x3: usize,
+    pub x2: usize,
+    pub x1: usize,
+    pub x0: usize,
+}
+pub use super::handler::enter_usermode;

src/arch/aarch64/interrupt/trace.rs

@@ -0,0 +1,32 @@
+use core::arch::asm;
+
+pub struct StackTrace {
+    pub fp: usize,
+    pub pc_ptr: *const usize,
+}
+
+impl StackTrace {
+    #[inline(always)]
+    pub unsafe fn start() -> Option<Self> {
+        unsafe {
+            let fp: usize;
+            asm!("mov {}, fp", out(reg) fp);
+            let pc_ptr = fp.checked_add(size_of::<usize>())?;
+            Some(StackTrace {
+                fp,
+                pc_ptr: pc_ptr as *const usize,
+            })
+        }
+    }
+
+    pub unsafe fn next(self) -> Option<Self> {
+        unsafe {
+            let fp = *(self.fp as *const usize);
+            let pc_ptr = fp.checked_add(size_of::<usize>())?;
+            Some(StackTrace {
+                fp: fp,
+                pc_ptr: pc_ptr as *const usize,
+            })
+        }
+    }
+}

src/arch/aarch64/ipi.rs

@@ -0,0 +1,30 @@
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum IpiKind {
+    Wakeup = 0x40,
+    Tlb = 0x41,
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum IpiTarget {
+    Other = 3,
+}
+
+#[inline(always)]
+pub fn ipi(_kind: IpiKind, _target: IpiTarget) {
+    if cfg!(not(feature = "multi_core")) {
+        return;
+    }
+
+    // FIXME implement
+}
+
+#[inline(always)]
+pub fn ipi_single(_kind: IpiKind, _target: &crate::percpu::PercpuBlock) {
+    if cfg!(not(feature = "multi_core")) {
+        return;
+    }
+
+    // FIXME implement
+}

src/arch/aarch64/misc.rs

@@ -0,0 +1,23 @@
+use crate::{
+    cpu_set::LogicalCpuId,
+    memory::{RmmA, RmmArch},
+    percpu::PercpuBlock,
+};
+
+impl PercpuBlock {
+    pub fn current() -> &'static Self {
+        unsafe { &*(crate::arch::device::cpu::registers::control_regs::tpidr_el1() as *const Self) }
+    }
+}
+
+#[cold]
+pub unsafe fn init(cpu_id: LogicalCpuId) {
+    unsafe {
+        let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
+        let virt = RmmA::phys_to_virt(frame.base()).data() as *mut PercpuBlock;
+
+        virt.write(PercpuBlock::init(cpu_id));
+
+        crate::arch::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64);
+    }
+}

src/arch/aarch64/mod.rs

@@ -0,0 +1,71 @@
+/// Constants like memory locations
+pub mod consts;
+
+/// Debugging support
+pub mod debug;
+
+/// Devices
+pub mod device;
+
+/// Interrupt instructions
+pub mod interrupt;
+
+/// Inter-processor interrupts
+pub mod ipi;
+
+/// Miscellaneous
+pub mod misc;
+
+/// Paging
+pub mod paging;
+
+/// Initialization and start function
+pub mod start;
+
+/// Stop function
+pub mod stop;
+
+// Interrupt vectors
+pub mod vectors;
+
+pub mod time;
+
+pub use ::rmm::aarch64::AArch64Arch as CurrentRmmArch;
+
+pub use arch_copy_to_user as arch_copy_from_user;
+
+#[unsafe(naked)]
+pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
+    // x0, x1, x2
+    core::arch::naked_asm!(
+        "
+    .global __usercopy_start
+    __usercopy_start:
+        mov x4, x0
+        mov x0, 0
+    2:
+        cmp x2, 0
+        b.eq 3f
+
+        ldrb w3, [x1]
+        strb w3, [x4]
+
+        add x4, x4, 1
+        add x1, x1, 1
+        sub x2, x2, 1
+
+        b 2b
+    3:
+        ret
+    .global __usercopy_end
+    __usercopy_end:
+    "
+    );
+}
+
+pub const KFX_SIZE: usize = 1024;
+
+// This function exists as the KFX size is dynamic on x86_64.
+pub fn kfx_size() -> usize {
+    KFX_SIZE
+}

src/arch/aarch64/paging.rs

@@ -0,0 +1,7 @@
+/// Initialize MAIR
+#[cold]
+pub unsafe fn init() {
+    unsafe {
+        rmm::aarch64::init_mair();
+    }
+}

src/arch/aarch64/start.rs

@@ -0,0 +1,148 @@
+//! This function is where the kernel sets up IRQ handlers
+//! It is incredibly unsafe, and should be minimal in nature
+//! It must create the IDT with the correct entries, those entries are
+//! defined in other files inside of the `arch` module
+use core::{arch::naked_asm, cell::SyncUnsafeCell, slice};
+
+use fdt::Fdt;
+
+use crate::{
+    allocator,
+    arch::{device, paging},
+    devices::graphical_debug,
+    dtb,
+    startup::KernelArgs,
+};
+
+/// Test of zero values in BSS.
+static mut BSS_TEST_ZERO: usize = 0;
+/// Test of non-zero values in data.
+static mut DATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
+
+#[repr(C, align(16))]
+struct StackAlign<T>(T);
+
+static STACK: SyncUnsafeCell<StackAlign<[u8; 128 * 1024]>> =
+    SyncUnsafeCell::new(StackAlign([0; 128 * 1024]));
+
+// FIXME use extern "custom"
+#[unsafe(naked)]
+#[unsafe(no_mangle)]
+extern "C" fn kstart() {
+    naked_asm!("
+        // BSS should already be zero
+        adrp x9, {bss_test_zero}
+        ldr x9, [x9, :lo12:{bss_test_zero}]
+        cbnz x9, .Lkstart_crash
+        adrp x9, {data_test_nonzero}
+        ldr x9, [x9, :lo12:{data_test_nonzero}]
+        cbz x9, .Lkstart_crash
+
+        adrp x1, {stack}
+        add x1, x1, :lo12:{stack}
+        mov x2, {stack_size}-16
+        add sp, x1, x2
+
+        // Setup interrupt handlers
+        ldr x9, =exception_vector_base
+        msr vbar_el1, x9
+
+        mov lr, 0
+        b {start}
+
+    .Lkstart_crash:
+        mov x9, 0
+        br x9
+    ",
+        bss_test_zero = sym BSS_TEST_ZERO,
+        data_test_nonzero = sym DATA_TEST_NONZERO,
+        stack = sym STACK,
+        stack_size = const size_of_val(&STACK),
+        start = sym start,
+    );
+}
+
+/// The entry to Rust, all things must be initialized
+unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
+    unsafe {
+        let bootstrap = {
+            let args = args_ptr.read();
+
+            // Set up graphical debug
+            graphical_debug::init(args.env());
+
+            // Get hardware descriptor data
+            //TODO: use env {DTB,RSDT}_{BASE,SIZE}?
+            let hwdesc_data = if args.hwdesc_base != 0 {
+                Some(slice::from_raw_parts(
+                    (crate::PHYS_OFFSET + args.hwdesc_base as usize) as *const u8,
+                    args.hwdesc_size as usize,
+                ))
+            } else {
+                None
+            };
+
+            let dtb_res = hwdesc_data
+                .ok_or(fdt::FdtError::BadPtr)
+                .and_then(|data| Fdt::new(data));
+
+            // Try to find serial port prior to logging
+            if let Ok(dtb) = &dtb_res {
+                dtb::serial::init_early(dtb);
+            }
+
+            info!("Redox OS starting...");
+            args.print();
+
+            // Initialize RMM
+            crate::startup::memory::init(&args, None, None);
+
+            // Initialize paging
+            paging::init();
+
+            crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0));
+
+            // Setup kernel heap
+            allocator::init();
+
+            // Activate memory logging
+            crate::log::init();
+
+            // Initialize devices
+            match dtb_res {
+                Ok(dtb) => {
+                    dtb::init(hwdesc_data.map(|slice| (slice.as_ptr() as usize, slice.len())));
+                    device::init_devicetree(&dtb);
+                }
+                Err(err) => {
+                    dtb::init(None);
+                    warn!("failed to parse DTB: {}", err);
+
+                    #[cfg(feature = "acpi")]
+                    {
+                        crate::acpi::init(args.acpi_rsdp());
+                    }
+                }
+            }
+
+            args.bootstrap()
+        };
+
+        crate::startup::kmain(bootstrap);
+    }
+}
+
+#[repr(C, packed)]
+#[allow(unused)]
+pub struct KernelArgsAp {
+    cpu_id: u64,
+    page_table: u64,
+    stack_start: u64,
+    stack_end: u64,
+}
+
+/// Entry to rust for an AP
+#[allow(unused)]
+pub unsafe extern "C" fn kstart_ap(_args_ptr: *const KernelArgsAp) -> ! {
+    loop {}
+}

src/arch/aarch64/stop.rs

@@ -0,0 +1,33 @@
+use crate::sync::CleanLockToken;
+use core::arch::asm;
+
+pub unsafe fn kreset() -> ! {
+    unsafe {
+        println!("kreset");
+
+        asm!("hvc   #0",
+             in("x0") 0x8400_0009_usize,
+             options(noreturn),
+        )
+    }
+}
+
+pub unsafe fn emergency_reset() -> ! {
+    unsafe {
+        asm!("hvc   #0",
+             in("x0")  0x8400_0009_usize,
+             options(noreturn),
+        )
+    }
+}
+
+pub unsafe fn kstop(_token: &mut CleanLockToken) -> ! {
+    unsafe {
+        println!("kstop");
+
+        asm!("hvc   #0",
+             in("x0")  0x8400_0008_usize,
+             options(noreturn),
+        )
+    }
+}

src/arch/aarch64/time.rs

@@ -0,0 +1,18 @@
+use crate::{sync::CleanLockToken, time::NANOS_PER_SEC};
+
+pub fn monotonic_absolute(_token: &mut CleanLockToken) -> u128 {
+    //TODO: aarch64 generic timer counter
+    let ticks: usize;
+    unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) ticks) };
+    let freq: usize;
+    unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) freq) };
+
+    ticks as u128 * NANOS_PER_SEC / freq as u128
+}
+
+pub fn monotonic_resolution() -> u128 {
+    let freq: usize;
+    unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) freq) };
+
+    NANOS_PER_SEC / freq as u128
+}

src/arch/aarch64/vectors.rs

@@ -0,0 +1,112 @@
+core::arch::global_asm!(
+    "
+    //  Exception vector stubs
+    //
+    //  Unhandled exceptions spin in a wfi loop for the moment
+    //  This can be macro-ified
+
+.globl exception_vector_base
+
+    .align 11
+exception_vector_base:
+
+    // Synchronous
+    .align 7
+__vec_00:
+    b       synchronous_exception_at_el1_with_sp0
+    b       __vec_00
+
+    // IRQ
+    .align 7
+__vec_01:
+    b       irq_at_el1
+    b       __vec_01
+
+    // FIQ
+    .align 7
+__vec_02:
+    b       unhandled_exception
+    b       __vec_02
+
+    // SError
+    .align 7
+__vec_03:
+    b       unhandled_exception
+    b       __vec_03
+
+    // Synchronous
+    .align 7
+__vec_04:
+    b       synchronous_exception_at_el1_with_spx
+    b       __vec_04
+
+    // IRQ
+    .align 7
+__vec_05:
+    b       irq_at_el1
+    b       __vec_05
+
+    // FIQ
+    .align 7
+__vec_06:
+    b       unhandled_exception
+    b       __vec_06
+
+    // SError
+    .align 7
+__vec_07:
+    b       unhandled_exception
+    b       __vec_07
+
+    // Synchronous
+    .align 7
+__vec_08:
+    b       synchronous_exception_at_el0
+    b       __vec_08
+
+    // IRQ
+    .align 7
+__vec_09:
+    b       irq_at_el0
+    b       __vec_09
+
+    // FIQ
+    .align 7
+__vec_10:
+    b       unhandled_exception
+    b       __vec_10
+
+    // SError
+    .align 7
+__vec_11:
+    b       unhandled_exception
+    b       __vec_11
+
+    // Synchronous
+    .align 7
+__vec_12:
+    b       unhandled_exception
+    b       __vec_12
+
+    // IRQ
+    .align 7
+__vec_13:
+    b       unhandled_exception
+    b       __vec_13
+
+    // FIQ
+    .align 7
+__vec_14:
+    b       unhandled_exception
+    b       __vec_14
+
+    // SError
+    .align 7
+__vec_15:
+    b       unhandled_exception
+    b       __vec_15
+
+    .align 7
+exception_vector_end:
+"
+);

src/arch/mod.rs

@@ -0,0 +1,27 @@
+#[cfg(target_arch = "aarch64")]
+#[macro_use]
+pub mod aarch64;
+#[cfg(target_arch = "aarch64")]
+pub use self::aarch64::*;
+
+#[cfg(target_arch = "x86")]
+#[macro_use]
+pub mod x86;
+#[cfg(target_arch = "x86")]
+pub use self::x86::*;
+
+#[cfg(target_arch = "x86_64")]
+#[macro_use]
+pub mod x86_64;
+#[cfg(target_arch = "x86_64")]
+pub use self::x86_64::*;
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[macro_use]
+mod x86_shared;
+
+#[cfg(target_arch = "riscv64")]
+#[macro_use]
+pub mod riscv64;
+#[cfg(target_arch = "riscv64")]
+pub use self::riscv64::*;

src/arch/riscv64/consts.rs

@@ -0,0 +1,16 @@
+use super::CurrentRmmArch;
+use rmm::Arch;
+
+const PML4_SHIFT: usize = (CurrentRmmArch::PAGE_LEVELS - 1) * CurrentRmmArch::PAGE_ENTRY_SHIFT
+    + CurrentRmmArch::PAGE_SHIFT;
+/// The size of a single PML4
+pub const PML4_SIZE: usize = 1_usize << PML4_SHIFT;
+
+/// Offset to kernel heap
+#[inline(always)]
+pub fn kernel_heap_offset() -> usize {
+    crate::kernel_executable_offsets::KERNEL_OFFSET() - PML4_SIZE
+}
+
+/// End offset of the user image, i.e. kernel start
+pub const USER_END_OFFSET: usize = 1_usize << (CurrentRmmArch::PAGE_ADDRESS_SHIFT - 1);

src/arch/riscv64/debug.rs

@@ -0,0 +1,19 @@
+use spin::MutexGuard;
+
+use crate::{arch::device::serial::COM1, devices::serial::SerialKind};
+
+pub struct Writer<'a> {
+    serial: MutexGuard<'a, SerialKind>,
+}
+
+impl<'a> Writer<'a> {
+    pub fn new() -> Writer<'a> {
+        Writer {
+            serial: COM1.lock(),
+        }
+    }
+
+    pub fn write(&mut self, buf: &[u8]) {
+        self.serial.write(buf);
+    }
+}

src/arch/riscv64/device/cpu/mod.rs

@@ -0,0 +1,5 @@
+use core::fmt::{Result, Write};
+
+pub fn cpu_info<W: Write>(_w: &mut W) -> Result {
+    unimplemented!()
+}

src/arch/riscv64/device/irqchip/clint.rs

@@ -0,0 +1,42 @@
+use spin::Mutex;
+use syscall::{Io, Mmio};
+use crate::context::switch::tick;
+
+#[repr(packed(4))]
+#[repr(C)]
+struct ClintRegs {
+    /// per-hart MSIP registers
+    /// bit 0: trigger IPI for the hart
+    msip: [Mmio<u32>; 4095], // +0000 -- 3fff
+    _rsrv1: u32,
+    /// per-hart MTIMECMP registers
+    /// timer interrupt trigger threshold
+    mtimecmp: [Mmio<u64>; 4095], // +4000 - bff7
+    mtime: Mmio<u64>  // current time
+}
+
+pub struct Clint {
+    regs: &'static mut ClintRegs,
+    freq: u64
+}
+
+pub static CLINT: Mutex<Option<Clint>> = Mutex::new(None);
+
+impl Clint {
+    pub fn new(addr: *mut u8, size: usize, freq: usize) -> Self {
+        assert!(size >= size_of::<ClintRegs>());
+        Self {
+            regs: unsafe { (addr as *mut ClintRegs).as_mut().unwrap() },
+            freq: freq as u64
+        }
+    }
+
+    pub fn init(self: &mut Self) {
+        (*self.regs).mtimecmp[0].write((*self.regs).mtime.read() + self.freq / 100);
+    }
+
+    pub fn timer_irq(self: &mut Self, hart: usize) {
+        (*self.regs).mtimecmp[hart].write((*self.regs).mtimecmp[hart].read() + self.freq / 100);
+        tick();
+    }
+}

src/arch/riscv64/device/irqchip/clint_sbi.rs

@@ -0,0 +1,150 @@
+use crate::{
+    context,
+    context::timeout,
+    dtb::irqchip::{register_irq, InterruptHandler, IrqCell, IRQ_CHIP},
+    sync::CleanLockToken,
+};
+use alloc::{boxed::Box, vec::Vec};
+use core::{arch::asm, cmp::max};
+use fdt::node::FdtNode;
+use spin::Mutex;
+// This is a Core-Local Interruptor (CLINT). A single device directly routed into each HLIC
+// It is responsible for local timer and IPI interrupts
+// An example DTS:
+// /soc/
+//   clint@2000000/
+//     interrupts-extended = <&hlic0 3>, <&hlic0 7>, <&hlic1 3>, <&hlic1 7>,
+//                           <&hlic2 3>, <&hlic2 7>, <&hlic3 3>, <&hlic3 7>;
+//     reg = <0x200000000 0x10000>;
+//     compatible = "sifive,clint0", "riscv,clint0";
+
+pub struct Clint {
+    freq: u64,
+    next_event: Vec<u64>,
+}
+
+pub static CLINT: Mutex<Option<Clint>> = Mutex::new(None);
+const TICKS_PER_SECOND: u64 = 100;
+const IRQ_IPI: usize = 0;
+const IRQ_TIMER: usize = 1;
+
+struct ClintConnector {
+    hart_id: usize,
+    irq: usize,
+}
+
+impl InterruptHandler for ClintConnector {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {
+        CLINT
+            .lock()
+            .as_mut()
+            .unwrap()
+            .irq_handler(self.hart_id, self.irq);
+        if self.irq == IRQ_TIMER {
+            // a bit of hack, but it is a really bad idea to call scheduler
+            // from inside clint irq handler
+            timeout::trigger(token);
+            context::switch::tick(token);
+        }
+    }
+}
+
+fn map_interrupt(irq: u32) -> u32 {
+    match irq {
+        3 => 1, // map M-mode IPI to S-mode IPI
+        7 => 5, // map M-mode timer to S-mode timer
+        x => x,
+    }
+}
+
+impl Clint {
+    pub fn new(freq: usize, node: &FdtNode) -> Self {
+        // TODO IPI
+        // let reg = clint_node.reg().unwrap().next().unwrap();
+        // reg.starting_address.add(crate::PHYS_OFFSET) as *mut u8;
+        // reg.size.unwrap();
+
+        let mut me = Self {
+            freq: freq as u64,
+            next_event: Vec::new(),
+        };
+        let mut interrupts = node
+            .property("interrupts-extended")
+            .unwrap()
+            .value
+            .as_chunks::<4>()
+            .0
+            .iter()
+            .map(|&x| u32::from_be_bytes(x));
+        let mut hart_id = 0;
+        while let Ok([phandle1, irq0, phandle2, irq1]) = interrupts.next_chunk::<4>() {
+            assert_eq!(
+                phandle1, phandle2,
+                "Invalid interrupts-extended property for CLINT"
+            );
+            let hlic = unsafe {
+                IRQ_CHIP
+                    .irq_chip_list
+                    .chips
+                    .iter()
+                    .find(|x| x.phandle == phandle1)
+                    .expect("Couldn't find HLIC in irqchip list for CLINT")
+            };
+
+            // FIXME dirty hack map M-mode interrupts (handled by SBI) to S-mode interrupts we get from SBI
+            // Why aren't S-mode interrupts in the DTB already?
+            let irq0 = IrqCell::L1(map_interrupt(irq0));
+            let irq1 = IrqCell::L1(map_interrupt(irq1));
+
+            let virq0 = hlic
+                .ic
+                .irq_xlate(irq0)
+                .expect("Couldn't get virq 0 from HLIC");
+            let virq1 = hlic
+                .ic
+                .irq_xlate(irq1)
+                .expect("Couldn't get virq 1 from HLIC");
+            register_irq(virq0 as u32, Box::new(ClintConnector { hart_id, irq: 0 }));
+            register_irq(virq1 as u32, Box::new(ClintConnector { hart_id, irq: 1 }));
+            hart_id += 1;
+        }
+        me.next_event.resize_with(hart_id, || 0);
+        me
+    }
+
+    pub(crate) fn irq_handler(self: &mut Self, hart_id: usize, irq: usize) {
+        match irq {
+            IRQ_IPI => {
+                println!("IPI interrupt at {}", hart_id);
+            }
+            IRQ_TIMER => {
+                let mtime: usize;
+                unsafe {
+                    asm!(
+                    "rdtime t0",
+                    lateout("t0") mtime
+                    )
+                };
+
+                self.next_event[hart_id] =
+                    max(self.next_event[hart_id], mtime as u64) + self.freq / TICKS_PER_SECOND;
+                sbi_rt::set_timer(self.next_event[hart_id]).expect("SBI timer cannot be set!");
+            }
+            _ => {
+                panic!("Unexpected CLINT irq")
+            }
+        }
+    }
+
+    pub fn init(self: &mut Self, hart: usize) {
+        let mtime: usize;
+        unsafe {
+            asm!(
+            "rdtime t0",
+            lateout("t0") mtime
+            )
+        };
+        self.next_event[hart] = mtime as u64 + (self.freq / TICKS_PER_SECOND);
+        sbi_rt::set_timer(self.next_event[hart]).expect("SBI timer cannot be set!");
+    }
+}

src/arch/riscv64/device/irqchip/hlic.rs

@@ -0,0 +1,170 @@
+use crate::{
+    dtb::irqchip::{InterruptController, InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP},
+    sync::CleanLockToken,
+};
+use alloc::vec::Vec;
+use core::arch::asm;
+use fdt::{node::NodeProperty, Fdt};
+use syscall::{Error, EINVAL};
+
+// This is a hart-local interrupt controller, a root of irqchip tree
+// An example DTS:
+// /cpus/
+//    cpu@1/
+//      interrupt-controller/
+//        #interrupt-cells = 0x00000001
+//        interrupt-controller =
+//        compatible = "riscv,cpu-intc"
+//        phandle = 0x00000006
+
+fn acknowledge(interrupt: usize) {
+    unsafe {
+        asm!(
+        "csrc sip, t0",
+        in("t0") 1usize << interrupt,
+        options(nostack)
+        )
+    }
+}
+
+pub unsafe fn interrupt(hart: usize, interrupt: usize, token: &mut CleanLockToken) {
+    unsafe {
+        assert!(
+            hart < CPU_INTERRUPT_HANDLERS.len(),
+            "Unexpected hart in interrupt routine"
+        );
+        acknowledge(interrupt);
+        let ic_idx = CPU_INTERRUPT_HANDLERS[hart].unwrap_or_else(|| {
+            panic!(
+                "No hlic connected to hart {} yet interrupt {} occurred",
+                hart, interrupt
+            )
+        });
+        let virq = IRQ_CHIP
+            .irq_to_virq(ic_idx, interrupt as u32)
+            .unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt));
+        match &mut IRQ_CHIP.irq_desc[virq].handler {
+            Some(handler) => {
+                handler.irq_handler(virq as u32, token);
+            }
+            _ => match IRQ_CHIP.irq_desc[virq].basic.child_ic_idx {
+                Some(ic_idx) => {
+                    IRQ_CHIP.irq_chip_list.chips[ic_idx]
+                        .ic
+                        .irq_handler(virq as u32, token);
+                }
+                _ => {
+                    panic!(
+                        "Unconnected interrupt {} occurred on hlic connected to hart {}",
+                        interrupt, hart
+                    );
+                }
+            },
+        }
+    }
+}
+
+pub fn init() {
+    unsafe {
+        asm!(
+            "csrs sie, t0",
+            in("t0") (0xFFFF),
+            options(nostack)
+        )
+    }
+}
+
+static mut CPU_INTERRUPT_HANDLERS: Vec<Option<usize>> = Vec::new();
+
+pub struct Hlic {
+    virq_base: usize,
+}
+
+impl Hlic {
+    pub(crate) fn new() -> Self {
+        return Self { virq_base: 0 };
+    }
+}
+impl InterruptHandler for Hlic {
+    fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) {
+        assert!(irq < 16, "Unsupported HLIC interrupt raised!");
+        unsafe {
+            IRQ_CHIP.trigger_virq(self.virq_base as u32 + irq, token);
+        }
+    }
+}
+
+impl InterruptController for Hlic {
+    fn irq_init(
+        &mut self,
+        fdt_opt: Option<&Fdt>,
+        irq_desc: &mut [IrqDesc; 1024],
+        ic_idx: usize,
+        irq_idx: &mut usize,
+    ) -> syscall::Result<()> {
+        let desc = unsafe { &IRQ_CHIP.irq_chip_list.chips[ic_idx] };
+        let fdt = fdt_opt.unwrap();
+        let cpu_node = fdt
+            .find_all_nodes("/cpus/cpu")
+            .find(|x| {
+                x.children().any(|x| {
+                    x.property("phandle").and_then(NodeProperty::as_usize)
+                        == Some(desc.phandle as usize)
+                })
+            })
+            .expect("Could not find CPU node for HLIC controller");
+        let hart = cpu_node.property("reg").unwrap().as_usize().unwrap();
+        unsafe {
+            if CPU_INTERRUPT_HANDLERS.len() <= hart {
+                CPU_INTERRUPT_HANDLERS.resize(hart + 1, None);
+            }
+            assert!(
+                CPU_INTERRUPT_HANDLERS[hart].replace(ic_idx).is_none(),
+                "Conflicting HLIC interrupt handler found"
+            );
+        }
+        self.virq_base = *irq_idx;
+        for i in 0..16 {
+            irq_desc[self.virq_base + i].basic.ic_idx = ic_idx;
+            irq_desc[self.virq_base + i].basic.ic_irq = i as u32;
+        }
+        *irq_idx += 16;
+        Ok(())
+    }
+
+    fn irq_ack(&mut self) -> u32 {
+        panic!("Cannot ack HLIC interrupt");
+    }
+
+    fn irq_eoi(&mut self, _irq_num: u32) {}
+
+    fn irq_enable(&mut self, _irq_num: u32) {
+        // This would require IPI to a correct core
+        // Not bothering with this, all interrupts are enabled at all times
+    }
+
+    fn irq_disable(&mut self, _irq_num: u32) {
+        // This would require IPI to a correct core
+        // Not bothering with this, all interrupts are enabled at all times
+    }
+
+    fn irq_xlate(&self, irq_data: IrqCell) -> syscall::Result<usize> {
+        match irq_data {
+            IrqCell::L1(irq) if irq <= 0xF => Ok(self.virq_base + irq as usize),
+            _ => Err(Error::new(EINVAL)),
+        }
+    }
+
+    fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
+        if hwirq > 0 && hwirq <= 0xF {
+            Some(self.virq_base + hwirq as usize)
+        } else {
+            None
+        }
+    }
+}
+
+pub fn irqchip_for_hart(hart: usize) -> Option<usize> {
+    let value = unsafe { CPU_INTERRUPT_HANDLERS.get(hart) }?;
+    *value
+}

src/arch/riscv64/device/irqchip/mod.rs

@@ -0,0 +1,44 @@
+use self::clint::Clint;
+use crate::dtb::irqchip::InterruptController;
+use alloc::boxed::Box;
+use fdt::Fdt;
+
+pub(crate) mod hlic;
+mod plic;
+
+#[path = "clint_sbi.rs"]
+mod clint;
+
+// pub mod clint; // actual clint.rs off limits if SBI is present
+
+pub fn new_irqchip(ic_str: &str) -> Option<Box<dyn InterruptController>> {
+    if ic_str.contains("riscv,cpu-intc") {
+        Some(Box::new(hlic::Hlic::new()))
+    } else if ic_str.contains("riscv,plic0") || ic_str.contains("sifive,plic-1.0.0") {
+        Some(Box::new(plic::Plic::new()))
+    } else {
+        warn!("no driver for interrupt controller {:?}", ic_str);
+        None
+    }
+}
+
+pub unsafe fn init_clint(fdt: &Fdt) {
+    let cpus = fdt.find_node("/cpus").unwrap();
+    let clock_freq = cpus
+        .property("timebase-frequency")
+        .unwrap()
+        .as_usize()
+        .unwrap();
+
+    let clint_node = fdt.find_node("/soc/clint").unwrap();
+    assert!(clint_node
+        .compatible()
+        .unwrap()
+        .all()
+        .find(|x| (*x).eq("riscv,clint0"))
+        .is_some());
+
+    let clint = Clint::new(clock_freq, &clint_node);
+    *clint::CLINT.lock() = Some(clint);
+    clint::CLINT.lock().as_mut().unwrap().init(0);
+}

src/arch/riscv64/device/irqchip/plic.rs

@@ -0,0 +1,198 @@
+use crate::{
+    arch::{device::irqchip::hlic, start::BOOT_HART_ID},
+    dtb::{
+        get_mmio_address,
+        irqchip::{InterruptController, InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP},
+    },
+    sync::CleanLockToken,
+};
+use core::{mem, num::NonZero, sync::atomic::Ordering};
+use fdt::Fdt;
+use syscall::{Error, Io, Mmio, EINVAL};
+
+#[repr(packed(4))]
+#[repr(C)]
+struct InterruptThresholdRegs {
+    threshold: Mmio<u32>,
+    claim_complete: Mmio<u32>,
+    _rsrv: [u32; 1022],
+}
+
+static MAX_CONTEXTS: usize = 64;
+
+#[repr(packed(4))]
+#[repr(C)]
+struct PlicRegs {
+    /// source priorities
+    source_priority: [Mmio<u32>; 1024], // +0000 -- 0fff
+    // pending interrupts
+    pending: [Mmio<u32>; 1024], // +1000 -- 1fff
+    // per-context interrupt enable
+    enable: [[Mmio<u32>; 32]; 16320], // +2000 - 1f'ffff
+    // per-context priority threshold and acknowledge
+    thresholds: [InterruptThresholdRegs; 64], // specced at +20'0000 - 0fff'ffff for 15872 contexts
+                                              // but actual memory allotted in firmware is much lower
+}
+
+const _: () = assert!(0x1000 == mem::offset_of!(PlicRegs, pending));
+const _: () = assert!(0x2000 == mem::offset_of!(PlicRegs, enable));
+const _: () = assert!(0x20_0000 == mem::offset_of!(PlicRegs, thresholds));
+const _: () = assert!(0x1000 == size_of::<InterruptThresholdRegs>());
+
+impl PlicRegs {
+    pub fn set_priority(self: &mut Self, irq: usize, priority: usize) {
+        assert!(irq > 0 && irq <= 1023 && priority < 8);
+        self.source_priority[irq].write(priority as u32);
+    }
+
+    pub fn pending(self: &Self, irq_lane: usize) -> u32 {
+        assert!(irq_lane < 32);
+        self.pending[irq_lane].read()
+    }
+
+    pub fn enable(self: &mut Self, context: usize, irq: NonZero<usize>, enable: bool) {
+        assert!(irq.get() <= 1023 && context < MAX_CONTEXTS);
+        let irq_lane = irq.get() / 32;
+        let irq = irq.get() % 32;
+        self.enable[context][irq_lane].writef(1u32 << irq, enable);
+    }
+
+    pub fn set_priority_threshold(self: &mut Self, context: usize, priority: usize) {
+        assert!(context < MAX_CONTEXTS && priority <= 7);
+        self.thresholds[context].threshold.write(priority as u32);
+    }
+
+    pub fn claim(self: &mut Self, context: usize) -> Option<NonZero<usize>> {
+        assert!(context < MAX_CONTEXTS);
+        let claim = self.thresholds[context].claim_complete.read();
+        NonZero::new(claim as usize)
+    }
+
+    pub fn complete(self: &mut Self, context: usize, claim: NonZero<usize>) {
+        assert!(context < MAX_CONTEXTS);
+        self.thresholds[context]
+            .claim_complete
+            .write(claim.get() as u32);
+    }
+}
+
+pub struct Plic {
+    regs: *mut PlicRegs,
+    ndev: usize,
+    virq_base: usize,
+    context: usize,
+}
+
+impl Plic {
+    pub fn new() -> Self {
+        Self {
+            regs: 0 as *mut PlicRegs,
+            ndev: 0,
+            virq_base: 0,
+            context: 0,
+        }
+    }
+}
+impl InterruptHandler for Plic {
+    fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {
+        unsafe {
+            let irq = self.irq_ack();
+            //println!("PLIC interrupt {}", irq);
+            if let Some(virq) = self.irq_to_virq(irq) {
+                IRQ_CHIP.trigger_virq(virq as u32, token);
+            } else {
+                error!("unexpected irq num {}", irq);
+                self.irq_eoi(irq);
+            }
+        }
+        //println!("PLIC interrupt done");
+    }
+}
+
+impl InterruptController for Plic {
+    fn irq_init(
+        &mut self,
+        fdt_opt: Option<&Fdt>,
+        irq_desc: &mut [IrqDesc; 1024],
+        ic_idx: usize,
+        irq_idx: &mut usize,
+    ) -> syscall::Result<()> {
+        let desc = unsafe { &IRQ_CHIP.irq_chip_list.chips[ic_idx] };
+        let fdt = fdt_opt.unwrap();
+        let my_node = fdt.find_phandle(desc.phandle).unwrap();
+
+        // MMIO region
+        let reg = my_node.reg().unwrap().next().unwrap();
+        let addr = get_mmio_address(&fdt, &my_node, &reg).unwrap();
+        // Specifies how many external interrupts are supported by this controller.
+        let ndev = my_node
+            .property("riscv,ndev")
+            .and_then(|x| x.as_usize())
+            .unwrap();
+
+        self.regs = (addr + crate::PHYS_OFFSET) as *mut PlicRegs;
+        self.ndev = ndev;
+
+        self.virq_base = *irq_idx;
+        for i in 0..ndev {
+            irq_desc[self.virq_base + i].basic.ic_idx = ic_idx;
+            irq_desc[self.virq_base + i].basic.ic_irq = i as u32;
+        }
+        *irq_idx += ndev;
+
+        // route all interrupts to boot HART
+        // TODO spread irqs over all the cores when we have them?
+        let hlic_ic_idx = hlic::irqchip_for_hart(BOOT_HART_ID.load(Ordering::Relaxed))
+            .expect("Could not find HLIC irqchip for the boot hart while initing PLIC");
+        self.context = desc
+            .parents
+            .iter()
+            .position(|x| x.parent_interrupt.is_some() && x.parent == hlic_ic_idx)
+            .unwrap();
+        info!("PLIC: using context {}", self.context);
+
+        let regs = unsafe { self.regs.as_mut().unwrap() };
+        regs.set_priority_threshold(self.context, 0);
+
+        Ok(())
+    }
+
+    fn irq_ack(&mut self) -> u32 {
+        let regs = unsafe { self.regs.as_mut().unwrap() };
+        regs.claim(self.context).unwrap().get() as u32
+    }
+
+    fn irq_eoi(&mut self, irq_num: u32) {
+        let regs = unsafe { self.regs.as_mut().unwrap() };
+        regs.complete(self.context, NonZero::new(irq_num as usize).unwrap());
+    }
+
+    fn irq_enable(&mut self, irq_num: u32) {
+        assert!(irq_num > 0 && irq_num as usize <= self.ndev);
+        let regs = unsafe { self.regs.as_mut().unwrap() };
+        regs.set_priority(irq_num as usize, 1);
+        regs.enable(self.context, NonZero::new(irq_num as usize).unwrap(), true);
+    }
+
+    fn irq_disable(&mut self, irq_num: u32) {
+        assert!(irq_num > 0 && irq_num as usize <= self.ndev);
+        let regs = unsafe { self.regs.as_mut().unwrap() };
+        regs.set_priority(irq_num as usize, 1);
+        regs.enable(self.context, NonZero::new(irq_num as usize).unwrap(), false);
+    }
+
+    fn irq_xlate(&self, irq_data: IrqCell) -> syscall::Result<usize> {
+        match irq_data {
+            IrqCell::L1(irq) => Ok(self.virq_base + irq as usize),
+            _ => Err(Error::new(EINVAL)),
+        }
+    }
+
+    fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
+        if (hwirq as usize) < self.ndev {
+            Some(self.virq_base + hwirq as usize)
+        } else {
+            None
+        }
+    }
+}

src/arch/riscv64/device/mod.rs

@@ -0,0 +1,110 @@
+use crate::{
+    arch::{device::irqchip::hlic, time},
+    dtb::DTB_BINARY,
+};
+use fdt::{
+    node::{FdtNode, NodeProperty},
+    Fdt,
+};
+
+pub mod cpu;
+pub(crate) mod irqchip;
+pub mod serial;
+
+use crate::arch::device::irqchip::init_clint;
+
+fn string_property(name: &str) -> bool {
+    name == "compatible"
+        || name == "model"
+        || name == "device_type"
+        || name == "status"
+        || name == "riscv,isa-base"
+        || name == "riscv,isa"
+        || name == "mmu-type"
+        || name == "stdout-path"
+}
+
+fn print_property(prop: &NodeProperty, n_spaces: usize) {
+    (0..n_spaces).for_each(|_| print!(" "));
+    print!("{} =", prop.name);
+    if string_property(prop.name)
+        && let Some(str) = prop.as_str()
+    {
+        println!(" \"{}\"", str);
+    } else if let Some(value) = prop.as_usize() {
+        println!(" 0x{:08x}", value);
+    } else {
+        for v in prop.value {
+            print!(" {:02x}", v);
+        }
+        println!();
+    }
+}
+fn print_node(node: &FdtNode<'_, '_>, n_spaces: usize) {
+    (0..n_spaces).for_each(|_| print!(" "));
+    println!("{}/", node.name);
+    for prop in node.properties() {
+        print_property(&prop, n_spaces + 4);
+    }
+
+    for child in node.children() {
+        print_node(&child, n_spaces + 4);
+    }
+}
+
+pub(crate) fn dump_fdt(fdt: &Fdt) {
+    if let Some(root) = fdt.find_node("/") {
+        print_node(&root, 0);
+    }
+}
+
+unsafe fn init_intc(cpu: &FdtNode) {
+    let intc_node = cpu
+        .children()
+        .find(|x| x.name == "interrupt-controller")
+        .unwrap();
+    assert_eq!(intc_node.compatible().unwrap().first(), "riscv,cpu-intc");
+    // This controller is hardwired into interrupt handler code and has no Mmios
+    hlic::init(); // enable interrupts at HLIC level
+}
+
+pub unsafe fn init() {
+    unsafe {
+        let data = DTB_BINARY.get().unwrap();
+        let fdt = Fdt::new(data).unwrap();
+
+        crate::dtb::irqchip::init(&fdt);
+
+        let cpu = fdt.find_node(format!("/cpus/cpu@{}", 0).as_str()).unwrap();
+        init_intc(&cpu);
+        init_time(&fdt);
+    }
+}
+
+fn init_time(fdt: &Fdt) {
+    let cpus = fdt.find_node("/cpus").unwrap();
+    let clock_freq = cpus
+        .property("timebase-frequency")
+        .unwrap()
+        .as_usize()
+        .unwrap();
+    time::init(clock_freq);
+}
+
+pub unsafe fn init_noncore() {
+    unsafe {
+        let data = DTB_BINARY.get().unwrap();
+        let fdt = Fdt::new(data).unwrap();
+
+        init_clint(&fdt);
+        serial::init(&fdt);
+    }
+}
+
+pub struct ArchPercpuMisc;
+
+impl ArchPercpuMisc {
+    pub const fn default() -> Self {
+        Self
+    }
+}

src/arch/riscv64/device/serial.rs

@@ -0,0 +1,47 @@
+use alloc::boxed::Box;
+use fdt::Fdt;
+
+pub use crate::dtb::serial::COM1;
+use crate::{
+    dtb::{
+        get_interrupt, interrupt_parent,
+        irqchip::{register_irq, InterruptHandler, IRQ_CHIP},
+    },
+    scheme::irq::irq_trigger,
+    sync::CleanLockToken,
+};
+
+pub struct Com1Irq {}
+
+impl InterruptHandler for Com1Irq {
+    fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) {
+        COM1.lock().receive(token);
+        unsafe {
+            // FIXME add_irq accepts a u8 as irq number
+            // PercpuBlock::current().stats.add_irq(irq);
+            irq_trigger(irq.try_into().unwrap(), token);
+            IRQ_CHIP.irq_eoi(irq);
+        }
+    }
+}
+
+pub unsafe fn init(fdt: &Fdt) -> Option<()> {
+    unsafe {
+        if let Some(node) = fdt.find_compatible(&["ns16550a", "snps,dw-apb-uart"]) {
+            let intr = get_interrupt(fdt, &node, 0).unwrap();
+            let interrupt_parent = interrupt_parent(fdt, &node)?;
+            let phandle = interrupt_parent.property("phandle")?.as_usize()? as u32;
+            let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle)?;
+
+            let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
+                .ic
+                .irq_xlate(intr)
+                .unwrap();
+            info!("serial_port virq = {}", virq);
+            register_irq(virq as u32, Box::new(Com1Irq {}));
+            IRQ_CHIP.irq_enable(virq as u32);
+        }
+        // COM1.lock().enable_irq(); // FIXME receive int is enabled by default in 16550. Disable by default?
+        Some(())
+    }
+}

src/arch/riscv64/interrupt/exception.rs

@@ -0,0 +1,229 @@
+use ::syscall::Exception;
+use core::{arch::naked_asm, sync::atomic::Ordering};
+use rmm::VirtualAddress;
+
+use crate::{
+    arch::{device::irqchip, start::BOOT_HART_ID},
+    context::signal::excp_handler,
+    memory::GenericPfFlags,
+    ptrace,
+    sync::CleanLockToken,
+    syscall::{self, flag::*},
+};
+
+const BREAKPOINT: usize = 3;
+const USERMODE_ECALL: usize = 8;
+const INSTRUCTION_PAGE_FAULT: usize = 12;
+const LOAD_PAGE_FAULT: usize = 13;
+const STORE_PAGE_FAULT: usize = 15;
+
+use super::InterruptStack;
+
+#[unsafe(naked)]
+// FIXME use extern "custom"
+// FIXME use align(4)
+pub unsafe extern "C" fn exception_handler() {
+    naked_asm!(
+        "csrrw tp, sscratch, tp",
+        "beq   tp, x0, 3f", // exception before percpu data is available; got to be S mode
+
+        "sd    t0, 0(tp)",
+        "csrr  t0, sstatus",
+        "andi  t0, t0, 1<<8",// SPP bit
+        "bne   t0, x0, 2f",
+
+        // trap/interrupt from U mode, switch stacks
+        "ld      t0, 0(tp)",
+        "sd      sp, 0(tp)",
+        "ld      sp, 8(tp)",
+
+        push_registers!(),
+        "ld      t0, 0(tp)",
+        "sd      t0, (1 * 8)(sp)", // save original SP
+        "csrrw   t0, sscratch, tp",
+        "sd      t0, (3 * 8)(sp)", // save original TP, and restore sscratch to handle double faults
+
+        "mv      a0, sp",
+        "jal     {0}",
+
+        // save S mode stack to percpu
+        "addi    t0, sp, 32 * 8",
+        "sd      t0, 8(tp)",
+        "li      t0, 1 << 8", // return to U mode (sstatus might've been modified by nested trap or context switch)
+        "csrc    sstatus, t0",
+        "j       4f",
+
+    "2:  ld      t0, 0(tp)", // S-mode
+    "3:",                    // S mode early
+
+        "addi    sp, sp, -2 * 8", // fake stack frame for the stack tracer
+
+        push_registers!(),
+
+        "addi    t1, sp, 34 * 8",
+        "sd      t1, (1 * 8)(sp)", // save original SP
+        "csrrw   t1, sscratch, tp",
+        "sd      t1, (3 * 8)(sp)", // save original TP, and restore sscratch to handle double faults
+
+        "sd      t0, (33 * 8)(sp)",  // fill the stack frame. t0 holds original pc after push_registers
+        "sd      fp, (32 * 8)(sp)",
+        "addi    fp, sp, 34 * 8",
+
+        "mv      a0, sp",
+        "jal     {0}",
+        // return to S mode with interrupts disabled
+        // (sstatus might've been modified by nested trap or context switch)
+        "li      t0, 1 << 8",
+        "csrs   sstatus, t0",
+        "li      t0, 1 << 5",
+        "csrc   sstatus, t0",
+
+    "4:",
+        pop_registers!(),
+        "sret",
+        sym exception_handler_inner
+    );
+}
+
+unsafe fn exception_handler_inner(regs: &mut InterruptStack) {
+    unsafe {
+        let scause: usize;
+        let sstatus: usize;
+        core::arch::asm!(
+            "csrr t0, scause",
+            "csrr t1, sstatus",
+            lateout("t0") scause,
+            lateout("t1") sstatus,
+            options(nostack)
+        );
+
+        //info!("Exception handler incoming: sepc={:x} scause={:x} sstatus={:x}", regs.iret.sepc, scause, sstatus);
+
+        let user_mode = sstatus & (1 << 8) == 0;
+
+        if (scause as isize) < 0 {
+            handle_interrupt(scause & 0xF);
+        } else if page_fault(scause, regs, user_mode) {
+        } else if user_mode {
+            handle_user_exception(scause, regs);
+        } else {
+            handle_system_exception(scause, regs);
+        }
+        //info!("Exception handler outgoing");
+    }
+}
+
+unsafe fn handle_system_exception(scause: usize, regs: &InterruptStack) {
+    unsafe {
+        let stval: usize;
+        let tp: usize;
+        core::arch::asm!(
+            "csrr t0, stval",
+            "mv t1, tp",
+            lateout("t0") stval,
+            lateout("t1") tp,
+            options(nostack)
+        );
+
+        error!(
+            "S-mode exception! scause={:#016x}, stval={:#016x}",
+            scause, stval
+        );
+
+        if tp == 0 {
+            // Early failure - before misc::init and potentially before RMM init
+            // Do not attempt to trace stack because it would probably trap again
+            regs.dump();
+        } else {
+            regs.trace();
+        }
+        loop {}
+    }
+}
+
+unsafe fn handle_interrupt(interrupt: usize) {
+    unsafe {
+        let mut token = CleanLockToken::new();
+        // FIXME retrieve from percpu area
+        // For now all the interrupts go to boot hart so this suffices...
+        let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed);
+        irqchip::hlic::interrupt(hart, interrupt, &mut token);
+    }
+}
+
+unsafe fn handle_user_exception(scause: usize, regs: &mut InterruptStack) {
+    unsafe {
+        let mut token = CleanLockToken::new();
+
+        if scause == USERMODE_ECALL {
+            let r = &mut regs.registers;
+            regs.iret.sepc += 4; // skip ecall
+            let ret = syscall::syscall(r.x17, r.x10, r.x11, r.x12, r.x13, r.x14, r.x15, &mut token);
+            r.x10 = ret;
+            return;
+        }
+
+        if scause == BREAKPOINT {
+            if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None, &mut token).is_some() {
+                return;
+            }
+        }
+
+        let stval: usize;
+        core::arch::asm!(
+        "csrr t0, stval",
+        lateout("t0") stval,
+        options(nostack)
+        );
+
+        info!(
+            "U-mode exception! scause={:#016x}, stval={:#016x}",
+            scause, stval
+        );
+        regs.dump();
+
+        // TODO
+        /*
+        let signal = match scause {
+            0 | 4 | 6 | 18 | 19 => SIGBUS, // misaligned / machine check
+            2 | 8 | 9 => SIGILL,           // Illegal instruction / breakpoint / ecall
+            BREAKPOINT => SIGTRAP,
+            _ => SIGSEGV,
+        };
+        */
+        excp_handler(Exception { kind: scause });
+    }
+}
+
+unsafe fn page_fault(scause: usize, regs: &mut InterruptStack, user_mode: bool) -> bool {
+    unsafe {
+        if scause != INSTRUCTION_PAGE_FAULT
+            && scause != LOAD_PAGE_FAULT
+            && scause != STORE_PAGE_FAULT
+        {
+            return false;
+        }
+
+        let stval: usize;
+        core::arch::asm!(
+            "csrr t0, stval",
+            lateout("t0") stval,
+            options(nostack)
+        );
+
+        let address = VirtualAddress::new(stval);
+        let mut generic_flags = GenericPfFlags::empty();
+
+        generic_flags.set(GenericPfFlags::INVOLVED_WRITE, scause == STORE_PAGE_FAULT);
+        generic_flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, user_mode);
+        generic_flags.set(
+            GenericPfFlags::INSTR_NOT_DATA,
+            scause == INSTRUCTION_PAGE_FAULT,
+        );
+        // FIXME can these conditions be distinguished? Should they be?
+        generic_flags.set(GenericPfFlags::INVL, false);
+        generic_flags.set(GenericPfFlags::PRESENT, false);
+
+        crate::memory::page_fault_handler(regs, generic_flags, address).is_ok()
+    }
+}

src/arch/riscv64/interrupt/handler.rs

@@ -0,0 +1,332 @@
+use crate::{memory::ArchIntCtx, panic, syscall::IntRegisters};
+
+#[derive(Default)]
+#[repr(C)]
+pub struct Registers {
+    pub x1: usize,  // ra
+    pub x2: usize,  // sp
+    pub x3: usize,  // gp
+    pub x4: usize,  // tp
+    pub x5: usize,  // t0
+    pub x6: usize,  // t1
+    pub x7: usize,  // t2
+    pub x8: usize,  // s0/fp
+    pub x9: usize,  // s1
+    pub x10: usize, // a0...
+    pub x11: usize,
+    pub x12: usize,
+    pub x13: usize,
+    pub x14: usize,
+    pub x15: usize,
+    pub x16: usize,
+    pub x17: usize, // a7
+    pub x18: usize, // s2...
+    pub x19: usize,
+    pub x20: usize,
+    pub x21: usize,
+    pub x22: usize,
+    pub x23: usize,
+    pub x24: usize,
+    pub x25: usize,
+    pub x26: usize,
+    pub x27: usize, // s11
+    pub x28: usize, // t3...
+    pub x29: usize,
+    pub x30: usize,
+    pub x31: usize, // t6
+}
+
+impl Registers {
+    pub fn dump(&self) {
+        println!("X1:    {:>016X}", { self.x1 });
+        println!("X2:    {:>016X}", { self.x2 });
+        println!("X3:    {:>016X}", { self.x3 });
+        println!("X4:    {:>016X}", { self.x4 });
+        println!("X5:    {:>016X}", { self.x5 });
+        println!("X6:    {:>016X}", { self.x6 });
+        println!("X7:    {:>016X}", { self.x7 });
+        println!("X8:    {:>016X}", { self.x8 });
+        println!("X9:    {:>016X}", { self.x9 });
+        println!("X10:   {:>016X}", { self.x10 });
+        println!("X11:   {:>016X}", { self.x11 });
+        println!("X12:   {:>016X}", { self.x12 });
+        println!("X13:   {:>016X}", { self.x13 });
+        println!("X14:   {:>016X}", { self.x14 });
+        println!("X15:   {:>016X}", { self.x15 });
+        println!("X16:   {:>016X}", { self.x16 });
+        println!("X17:   {:>016X}", { self.x17 });
+        println!("X18:   {:>016X}", { self.x18 });
+        println!("X19:   {:>016X}", { self.x19 });
+        println!("X20:   {:>016X}", { self.x20 });
+        println!("X21:   {:>016X}", { self.x21 });
+        println!("X22:   {:>016X}", { self.x22 });
+        println!("X23:   {:>016X}", { self.x23 });
+        println!("X24:   {:>016X}", { self.x24 });
+        println!("X25:   {:>016X}", { self.x25 });
+        println!("X26:   {:>016X}", { self.x26 });
+        println!("X27:   {:>016X}", { self.x27 });
+        println!("X28:   {:>016X}", { self.x28 });
+        println!("X29:   {:>016X}", { self.x29 });
+        println!("X30:   {:>016X}", { self.x30 });
+        println!("X31:   {:>016X}", { self.x31 });
+    }
+}
+
+#[derive(Default)]
+#[repr(C)]
+pub struct IretRegisters {
+    pub sepc: usize,
+}
+
+impl IretRegisters {
+    pub fn dump(&self) {
+        println!("SEPC: {:>016X}", { self.sepc });
+    }
+}
+
+// NOTE: Layout of this structure must be synced with assembly code in exception.rs
+#[derive(Default)]
+#[repr(C)]
+pub struct InterruptStack {
+    pub registers: Registers,
+    pub iret: IretRegisters,
+}
+
+impl InterruptStack {
+    pub fn init(&mut self) {
+        const {
+            assert!(32 * 8 == size_of::<InterruptStack>());
+        }
+    }
+    pub fn frame_pointer(&self) -> usize {
+        self.registers.x8
+    }
+    pub fn stack_pointer(&self) -> usize {
+        self.registers.x2
+    }
+    pub fn set_stack_pointer(&mut self, sp: usize) {
+        self.registers.x2 = sp;
+    }
+    pub fn set_instr_pointer(&mut self, ip: usize) {
+        self.iret.sepc = ip;
+    }
+    pub fn instr_pointer(&self) -> usize {
+        self.iret.sepc
+    }
+    pub fn sig_archdep_reg(&self) -> usize {
+        self.registers.x5
+    }
+
+    pub fn set_syscall_ret_reg(&mut self, ret: usize) {
+        self.registers.x10 = ret;
+    }
+
+    pub fn set_arg1(&mut self, arg_opt: Option<usize>) {
+        if let Some(arg) = arg_opt {
+            self.registers.x11 = arg;
+        }
+    }
+
+    pub fn dump(&self) {
+        self.iret.dump();
+        self.registers.dump();
+    }
+
+    pub fn trace(&self) {
+        self.dump();
+        unsafe {
+            panic::user_stack_trace(&self);
+            panic::stack_trace();
+        }
+    }
+
+    /// Saves all registers to a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn save(&self, all: &mut IntRegisters) {
+        all.pc = self.iret.sepc;
+        all.x31 = self.registers.x31;
+        all.x30 = self.registers.x30;
+        all.x29 = self.registers.x29;
+        all.x28 = self.registers.x28;
+        all.x27 = self.registers.x27;
+        all.x26 = self.registers.x26;
+        all.x25 = self.registers.x25;
+        all.x24 = self.registers.x24;
+        all.x23 = self.registers.x23;
+        all.x22 = self.registers.x22;
+        all.x21 = self.registers.x21;
+        all.x20 = self.registers.x20;
+        all.x19 = self.registers.x19;
+        all.x18 = self.registers.x18;
+        all.x17 = self.registers.x17;
+        all.x16 = self.registers.x16;
+        all.x15 = self.registers.x15;
+        all.x14 = self.registers.x14;
+        all.x13 = self.registers.x13;
+        all.x12 = self.registers.x12;
+        all.x11 = self.registers.x11;
+        all.x10 = self.registers.x10;
+        all.x9 = self.registers.x9;
+        all.x8 = self.registers.x8;
+        all.x7 = self.registers.x7;
+        all.x6 = self.registers.x6;
+        all.x5 = self.registers.x5;
+        all.x2 = self.registers.x2;
+        all.x1 = self.registers.x1;
+    }
+
+    /// Loads all registers from a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn load(&mut self, all: &IntRegisters) {
+        self.iret.sepc = all.pc;
+        self.registers.x31 = all.x31;
+        self.registers.x30 = all.x30;
+        self.registers.x29 = all.x29;
+        self.registers.x28 = all.x28;
+        self.registers.x27 = all.x27;
+        self.registers.x26 = all.x26;
+        self.registers.x25 = all.x25;
+        self.registers.x24 = all.x24;
+        self.registers.x23 = all.x23;
+        self.registers.x22 = all.x22;
+        self.registers.x21 = all.x21;
+        self.registers.x20 = all.x20;
+        self.registers.x19 = all.x19;
+        self.registers.x18 = all.x18;
+        self.registers.x17 = all.x17;
+        self.registers.x16 = all.x16;
+        self.registers.x15 = all.x15;
+        self.registers.x14 = all.x14;
+        self.registers.x13 = all.x13;
+        self.registers.x12 = all.x12;
+        self.registers.x11 = all.x11;
+        self.registers.x10 = all.x10;
+        self.registers.x9 = all.x9;
+        self.registers.x8 = all.x8;
+        self.registers.x7 = all.x7;
+        self.registers.x6 = all.x6;
+        self.registers.x5 = all.x5;
+        self.registers.x2 = all.x2;
+        self.registers.x1 = all.x1;
+    }
+
+    //TODO
+    pub fn is_singlestep(&self) -> bool {
+        false
+    }
+    pub fn set_singlestep(&mut self, _singlestep: bool) {}
+}
+
+impl ArchIntCtx for InterruptStack {
+    fn ip(&self) -> usize {
+        self.iret.sepc
+    }
+    fn recover_and_efault(&mut self) {
+        // Set the return value to nonzero to indicate usercopy failure (EFAULT), and emulate the
+        // return instruction by setting the return pointer to the saved LR value.
+        self.iret.sepc = self.registers.x1; // ra
+        self.registers.x10 = 1; // a0
+    }
+}
+
+/// Except for sp and tp
+#[macro_export]
+macro_rules! push_registers {
+    () => {
+        "
+    addi    sp, sp, -32 * 8
+    sd      x1, (0 * 8)(sp)
+    // skip sp
+    sd      x3, (2 * 8)(sp)
+    // skip tp
+    sd      x5, (4 * 8)(sp)
+    sd      x6, (5 * 8)(sp)
+    sd      x7, (6 * 8)(sp)
+    sd      x8, (7 * 8)(sp)
+    sd      x9, (8 * 8)(sp)
+    sd      x10, (9 * 8)(sp)
+    sd      x11, (10 * 8)(sp)
+    sd      x12, (11 * 8)(sp)
+    sd      x13, (12 * 8)(sp)
+    sd      x14, (13 * 8)(sp)
+    sd      x15, (14 * 8)(sp)
+    sd      x16, (15 * 8)(sp)
+    sd      x17, (16 * 8)(sp)
+    sd      x18, (17 * 8)(sp)
+    sd      x19, (18 * 8)(sp)
+    sd      x20, (19 * 8)(sp)
+    sd      x21, (20 * 8)(sp)
+    sd      x22, (21 * 8)(sp)
+    sd      x23, (22 * 8)(sp)
+    sd      x24, (23 * 8)(sp)
+    sd      x25, (24 * 8)(sp)
+    sd      x26, (25 * 8)(sp)
+    sd      x27, (26 * 8)(sp)
+    sd      x28, (27 * 8)(sp)
+    sd      x29, (28 * 8)(sp)
+    sd      x30, (29 * 8)(sp)
+    sd      x31, (30 * 8)(sp)
+
+    csrr    t0, sepc
+    sd      t0, (31 * 8)(sp)
+    "
+    }; // keep sepc value in t0 on exit
+}
+
+#[macro_export]
+macro_rules! pop_registers {
+    () => {
+        "
+    ld      t0, (31 * 8)(sp)
+    csrw    sepc, t0
+
+    ld      x1, (0 * 8)(sp)
+    // skip sp, it'll be restored later
+    ld      x3, (2 * 8)(sp)
+    ld      x4, (3 * 8)(sp)
+    ld      x5, (4 * 8)(sp)
+    ld      x6, (5 * 8)(sp)
+    ld      x7, (6 * 8)(sp)
+    ld      x8, (7 * 8)(sp)
+    ld      x9, (8 * 8)(sp)
+    ld      x10, (9 * 8)(sp)
+    ld      x11, (10 * 8)(sp)
+    ld      x12, (11 * 8)(sp)
+    ld      x13, (12 * 8)(sp)
+    ld      x14, (13 * 8)(sp)
+    ld      x15, (14 * 8)(sp)
+    ld      x16, (15 * 8)(sp)
+    ld      x17, (16 * 8)(sp)
+    ld      x18, (17 * 8)(sp)
+    ld      x19, (18 * 8)(sp)
+    ld      x20, (19 * 8)(sp)
+    ld      x21, (20 * 8)(sp)
+    ld      x22, (21 * 8)(sp)
+    ld      x23, (22 * 8)(sp)
+    ld      x24, (23 * 8)(sp)
+    ld      x25, (24 * 8)(sp)
+    ld      x26, (25 * 8)(sp)
+    ld      x27, (26 * 8)(sp)
+    ld      x28, (27 * 8)(sp)
+    ld      x29, (28 * 8)(sp)
+    ld      x30, (29 * 8)(sp)
+    ld      x31, (30 * 8)(sp)
+    ld      sp, (1 * 8)(sp)
+    "
+    };
+}
+
+#[unsafe(naked)]
+pub unsafe extern "C" fn enter_usermode() -> ! {
+    core::arch::naked_asm!(
+        "jalr    s11",
+        "li      t0, 1 << 8", // force U mode on sret
+        "csrc    sstatus, t0",
+        "li      t0, 0x6000", // set FS to dirty (enable FPU in U mode)
+        "csrs    sstatus, t0",
+        "addi    t0, sp, 32 * 8", // save S mode stack to percpu
+        "sd      t0, 8(tp)",
+        pop_registers!(),
+        "sret",
+    )
+}

src/arch/riscv64/interrupt/mod.rs

@@ -0,0 +1,39 @@
+use core::arch::asm;
+
+#[macro_use]
+mod handler;
+
+mod exception;
+pub mod syscall;
+pub mod trace;
+
+pub use exception::exception_handler;
+pub use handler::InterruptStack;
+
+/// Clear interrupts
+#[inline(always)]
+pub unsafe fn disable() {
+    unsafe { asm!("csrci sstatus, 1 << 1") }
+}
+
+/// Set interrupts and halt
+/// This will atomically wait for the next interrupt
+/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_halt() {
+    unsafe { asm!("wfi", "csrsi sstatus, 1 << 1", "nop") }
+}
+
+/// Set interrupts and nop
+/// This will enable interrupts and allow the IF flag to be processed
+/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_nop() {
+    unsafe { asm!("csrsi sstatus, 1 << 1", "nop") }
+}
+
+/// Halt instruction
+#[inline(always)]
+pub unsafe fn halt() {
+    unsafe { asm!("wfi", options(nomem, nostack)) }
+}

src/arch/riscv64/interrupt/syscall.rs

@@ -0,0 +1 @@
+pub use super::handler::enter_usermode;

src/arch/riscv64/interrupt/trace.rs

@@ -0,0 +1,35 @@
+use core::arch::asm;
+
+pub struct StackTrace {
+    pub fp: usize,
+    pub pc_ptr: *const usize,
+}
+
+impl StackTrace {
+    #[inline(always)]
+    pub unsafe fn start() -> Option<Self> {
+        unsafe {
+            let fp: usize;
+            asm!("mv {}, fp", out(reg) fp);
+
+            let pc_ptr = fp.checked_sub(size_of::<usize>())?;
+            let fp = pc_ptr.checked_sub(size_of::<usize>())?;
+            Some(StackTrace {
+                fp,
+                pc_ptr: pc_ptr as *const usize,
+            })
+        }
+    }
+
+    pub unsafe fn next(self) -> Option<Self> {
+        unsafe {
+            let fp = *(self.fp as *const usize);
+            let pc_ptr = fp.checked_sub(size_of::<usize>())?;
+            let fp = pc_ptr.checked_sub(size_of::<usize>())?;
+            Some(StackTrace {
+                fp: fp,
+                pc_ptr: pc_ptr as *const usize,
+            })
+        }
+    }
+}

src/arch/riscv64/ipi.rs

@@ -0,0 +1,34 @@
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum IpiKind {
+    Wakeup = 0x40,
+    Tlb = 0x41,
+    Switch = 0x42,
+    Pit = 0x43,
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum IpiTarget {
+    Current = 1,
+    All = 2,
+    Other = 3,
+}
+
+#[inline(always)]
+pub fn ipi(_kind: IpiKind, _target: IpiTarget) {
+    if cfg!(not(feature = "multi_core")) {
+        return;
+    }
+
+    // FIXME implement
+}
+
+#[inline(always)]
+pub fn ipi_single(_kind: IpiKind, _target: &crate::percpu::PercpuBlock) {
+    if cfg!(not(feature = "multi_core")) {
+        return;
+    }
+
+    // FIXME implement
+}

src/arch/riscv64/misc.rs

@@ -0,0 +1,47 @@
+use core::arch::asm;
+
+use crate::{
+    cpu_set::LogicalCpuId,
+    memory::{RmmA, RmmArch},
+    percpu::PercpuBlock,
+};
+
+#[repr(C)]
+pub struct ArchPercpu {
+    // These fields must be kept first and in this order. Assembly in exception.rs depends on it
+    pub tmp: usize,
+    pub s_sp: usize,
+
+    pub percpu: PercpuBlock,
+}
+
+impl PercpuBlock {
+    pub fn current() -> &'static Self {
+        unsafe {
+            let tp: *const ArchPercpu;
+            asm!( "mv t0, tp", out("t0") tp );
+            let arch_percpu = &*tp;
+            &arch_percpu.percpu
+        }
+    }
+}
+
+#[cold]
+pub unsafe fn init(cpu_id: LogicalCpuId) {
+    unsafe {
+        let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
+        let virt = RmmA::phys_to_virt(frame.base()).data() as *mut ArchPercpu;
+
+        virt.write(ArchPercpu {
+            tmp: 0,
+            s_sp: 0,
+            percpu: PercpuBlock::init(cpu_id),
+        });
+
+        asm!(
+            "mv tp, {}",
+            "csrw sscratch, tp",
+            in(reg) virt as usize
+        );
+    }
+}

src/arch/riscv64/mod.rs

@@ -0,0 +1,69 @@
+pub mod consts;
+pub mod debug;
+pub mod device;
+pub mod interrupt;
+pub mod ipi;
+pub mod misc;
+pub mod paging;
+pub mod start;
+pub mod stop;
+pub mod time;
+
+pub use ::rmm::riscv64::RiscV64Sv39Arch as CurrentRmmArch;
+use core::arch::naked_asm;
+
+pub use arch_copy_to_user as arch_copy_from_user;
+
+#[unsafe(naked)]
+pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
+    naked_asm!(
+        "
+    .global __usercopy_start
+    __usercopy_start:
+        addi   sp, sp, -16
+        sd     fp, 0(sp)
+        sd     ra, 8(sp)
+        addi   fp, sp, 16
+        li     t1, 1 << 18 // SUM
+        csrs   sstatus, t1
+        jal    2f
+        csrc   sstatus, t1
+        ld     ra, -8(fp)
+        ld     fp, -16(fp)
+        addi   sp, sp, 16
+        ret
+
+    2:  or     t0, a0, a1
+        andi   t0, t0, 7
+        bne    t0, x0, 4f
+        srli   t2, a2, 3
+        andi   a2, a2, 7
+        beq    t2, x0, 4f
+    3:  ld     t0, 0(a1)
+        sd     t0, 0(a0)
+        addi   a0, a0, 8
+        addi   a1, a1, 8
+        addi   t2, t2, -1
+        bne    t2, x0, 3b
+
+    4:  beq    a2, x0, 5f
+        lb     t0, 0(a1)
+        sb     t0, 0(a0)
+        addi   a0, a0, 1
+        addi   a1, a1, 1
+        addi   a2, a2, -1
+        bne    a2, x0, 4b
+    5:  mv     a0, x0
+        ret
+    .global __usercopy_end
+    __usercopy_end:
+    "
+    )
+}
+
+pub const KFX_SIZE: usize = 1024;
+
+// This function exists as the KFX size is dynamic on x86_64.
+pub fn kfx_size() -> usize {
+    KFX_SIZE
+}

src/arch/riscv64/paging.rs

@@ -0,0 +1,5 @@
+#[cold]
+pub unsafe fn init() {
+    // Assuming SBI already set up PMAs correctly for us
+    // TODO: detect Svpbmt present/enabled and override device memory with PBMT=IO
+}

src/arch/riscv64/start.rs

@@ -0,0 +1,140 @@
+use core::{
+    arch::naked_asm,
+    cell::SyncUnsafeCell,
+    sync::atomic::{AtomicUsize, Ordering},
+};
+
+use crate::{
+    allocator,
+    arch::{device, interrupt::exception_handler, paging},
+    devices::graphical_debug,
+    dtb::serial::init_early,
+    startup::KernelArgs,
+};
+
+/// Test of zero values in BSS.
+static mut BSS_TEST_ZERO: usize = 0;
+/// Test of non-zero values in data.
+static mut DATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
+
+pub static BOOT_HART_ID: AtomicUsize = AtomicUsize::new(0);
+
+fn get_boot_hart_id(env: &[u8]) -> Option<usize> {
+    for line in core::str::from_utf8(env).unwrap_or("").lines() {
+        let mut parts = line.splitn(2, '=');
+        let name = parts.next().unwrap_or("");
+        let value = parts.next().unwrap_or("");
+
+        if name == "BOOT_HART_ID" {
+            return usize::from_str_radix(value, 16).ok();
+        }
+    }
+    None
+}
+
+#[repr(C, align(16))]
+struct StackAlign<T>(T);
+
+static STACK: SyncUnsafeCell<StackAlign<[u8; 128 * 1024]>> =
+    SyncUnsafeCell::new(StackAlign([0; 128 * 1024]));
+
+// FIXME use extern "custom"
+#[unsafe(naked)]
+#[unsafe(no_mangle)]
+extern "C" fn kstart() {
+    naked_asm!("
+        mv gp, x0 // ensure gp relative accesses crash
+        mv tp, x0 // reset percpu until it is initialized
+        csrw sscratch, tp
+
+        // BSS should already be zero
+        ld t0, {bss_test_zero}
+        bnez t0, .Lkstart_crash
+        ld t0, {data_test_nonzero}
+        beqz t0, .Lkstart_crash
+
+    .Lpcrel_hi0:
+        auipc   sp, %pcrel_hi({stack}+{stack_size}-16)
+        addi    sp, sp, %pcrel_lo(.Lpcrel_hi0)
+
+        la t0, {exception_handler} // WARL=0 - direct mode combined handler
+        csrw stvec, t0
+
+        li ra, 0
+        j {start}
+
+    .Lkstart_crash:
+        jr x0
+    ",
+        bss_test_zero = sym BSS_TEST_ZERO,
+        data_test_nonzero = sym DATA_TEST_NONZERO,
+        exception_handler = sym exception_handler,
+        stack = sym STACK,
+        stack_size = const size_of_val(&STACK),
+        start = sym start,
+    );
+}
+
+/// The entry to Rust, all things must be initialized
+unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
+    unsafe {
+        let bootstrap = {
+            let args = args_ptr.read();
+
+            let dtb_data = if args.hwdesc_base != 0 {
+                Some((
+                    crate::PHYS_OFFSET + args.hwdesc_base as usize,
+                    args.hwdesc_size as usize,
+                ))
+            } else {
+                None
+            };
+            let dtb = args.dtb();
+
+            graphical_debug::init(args.env());
+
+            if let Some(dtb) = &dtb {
+                init_early(dtb);
+            }
+
+            info!("Redox OS starting...");
+            args.print();
+
+            if let Some(dtb) = &dtb {
+                device::dump_fdt(&dtb);
+            }
+
+            // Initialize RMM
+            crate::startup::memory::init(&args, None, None);
+
+            let boot_hart_id =
+                get_boot_hart_id(args.env()).expect("Didn't get boot HART id from bootloader");
+            info!("Booting on HART {}", boot_hart_id);
+            BOOT_HART_ID.store(boot_hart_id, Ordering::Relaxed);
+
+            paging::init();
+
+            crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0));
+
+            // Setup kernel heap
+            allocator::init();
+
+            // Activate memory logging
+            crate::log::init();
+
+            crate::dtb::init(dtb_data);
+
+            // Initialize devices
+            device::init();
+
+            // Initialize all of the non-core devices not otherwise needed to complete initialization
+            device::init_noncore();
+
+            // FIXME bringup AP HARTs
+
+            args.bootstrap()
+        };
+
+        crate::startup::kmain(bootstrap);
+    }
+}

src/arch/riscv64/stop.rs

@@ -0,0 +1,15 @@
+use crate::sync::CleanLockToken;
+
+pub unsafe fn kreset() -> ! {
+    println!("kreset");
+    unimplemented!()
+}
+
+pub unsafe fn emergency_reset() -> ! {
+    unimplemented!()
+}
+
+pub unsafe fn kstop(_token: &mut CleanLockToken) -> ! {
+    println!("kstop");
+    unimplemented!()
+}

src/arch/riscv64/time.rs

@@ -0,0 +1,34 @@
+use core::{
+    arch::asm,
+    sync::atomic::{AtomicUsize, Ordering},
+};
+
+use crate::sync::CleanLockToken;
+
+static MTIME_FREQ_HZ: AtomicUsize = AtomicUsize::new(0);
+
+pub fn init(freq_hz: usize) {
+    MTIME_FREQ_HZ.store(freq_hz, Ordering::Relaxed);
+}
+
+pub fn monotonic_absolute(_token: &mut CleanLockToken) -> u128 {
+    let freq_hz = MTIME_FREQ_HZ.load(Ordering::Relaxed);
+    if freq_hz > 0 {
+        let counter: usize;
+        unsafe {
+            asm!(
+            "rdtime t0",
+            lateout("t0") counter
+            );
+        };
+        counter as u128 * 1_000_000_000u128 / freq_hz as u128
+    } else {
+        0
+    }
+}
+
+pub fn monotonic_resolution() -> u128 {
+    let freq_hz = MTIME_FREQ_HZ.load(Ordering::Relaxed);
+
+    1_000_000_000u128 / freq_hz as u128
+}

src/arch/x86/consts.rs

@@ -0,0 +1,20 @@
+// Because the memory map is so important to not be aliased, it is defined here, in one place
+// The lower 256 PML4 entries are reserved for userspace
+// Each PML4 entry references up to 512 GB of memory
+// The second from the top (510) PML4 is reserved for the kernel
+
+// Framebuffer mapped by bootloader to 0xD000_0000 (128 MiB max)
+
+// Offset to APIC mappings (optional)
+pub const LAPIC_OFFSET: usize = 0xD800_0000;
+pub const IOAPIC_OFFSET: usize = LAPIC_OFFSET + 4096;
+pub const HPET_OFFSET: usize = IOAPIC_OFFSET + 4096;
+
+/// Offset to kernel heap (256 MiB max)
+#[inline(always)]
+pub fn kernel_heap_offset() -> usize {
+    0xE000_0000
+}
+
+/// End offset of the user image, i.e. kernel start
+pub const USER_END_OFFSET: usize = 0x8000_0000;

src/arch/x86/interrupt/handler.rs

@@ -0,0 +1,471 @@
+use crate::{arch::flags::FLAG_SINGLESTEP, memory::ArchIntCtx, panic, syscall::IntRegisters};
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct ScratchRegisters {
+    pub edx: usize,
+    pub ecx: usize,
+    pub eax: usize,
+}
+
+impl ScratchRegisters {
+    pub fn dump(&self) {
+        println!("EAX:   {:08x}", { self.eax });
+        println!("ECX:   {:08x}", { self.ecx });
+        println!("EDX:   {:08x}", { self.edx });
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct PreservedRegisters {
+    pub ebp: usize,
+    pub esi: usize,
+    pub edi: usize,
+    pub ebx: usize,
+}
+
+impl PreservedRegisters {
+    pub fn dump(&self) {
+        println!("EBX:   {:08x}", { self.ebx });
+        println!("EDI:   {:08x}", { self.edi });
+        println!("ESI:   {:08x}", { self.esi });
+        println!("EBP:   {:08x}", { self.ebp });
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct IretRegisters {
+    pub eip: usize,
+    pub cs: usize,
+    pub eflags: usize,
+
+    // ----
+    // The following will only be present if interrupt is raised from another
+    // privilege ring. Otherwise, they are undefined values.
+    // ----
+    pub esp: usize,
+    pub ss: usize,
+}
+
+impl IretRegisters {
+    pub fn dump(&self) {
+        println!("EFLAG: {:08x}", { self.eflags });
+        println!("CS:    {:08x}", { self.cs });
+        println!("EIP:   {:08x}", { self.eip });
+
+        if self.cs & 0b11 != 0b00 {
+            println!("ESP:   {:08x}", { self.esp });
+            println!("SS:    {:08x}", { self.ss });
+        }
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct InterruptStack {
+    pub gs: usize,
+    pub preserved: PreservedRegisters,
+    pub scratch: ScratchRegisters,
+    pub iret: IretRegisters,
+}
+
+impl InterruptStack {
+    pub fn init(&mut self) {
+        // Always enable interrupts!
+        self.iret.eflags = x86::bits32::eflags::EFlags::FLAGS_IF.bits() as usize;
+        self.iret.ss = (crate::arch::gdt::GDT_USER_DATA << 3) | 3;
+        self.iret.cs = (crate::arch::gdt::GDT_USER_CODE << 3) | 3;
+        self.gs = (crate::arch::gdt::GDT_USER_GS << 3) | 3;
+    }
+    pub fn dump(&self) {
+        self.iret.dump();
+        self.scratch.dump();
+        self.preserved.dump();
+    }
+    pub fn trace(&self) {
+        self.dump();
+        unsafe {
+            panic::user_stack_trace(&self);
+            panic::stack_trace();
+        }
+    }
+    /// Saves all registers to a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn save(&self, all: &mut IntRegisters) {
+        all.ebp = self.preserved.ebp;
+        all.esi = self.preserved.esi;
+        all.edi = self.preserved.edi;
+        all.ebx = self.preserved.ebx;
+        all.edx = self.scratch.edx;
+        all.ecx = self.scratch.ecx;
+        all.eax = self.scratch.eax;
+        all.eip = self.iret.eip;
+        all.cs = self.iret.cs;
+        all.eflags = self.iret.eflags;
+
+        // Set esp and ss:
+
+        const CPL_MASK: usize = 0b11;
+
+        let cs: usize;
+        unsafe {
+            core::arch::asm!("mov {}, cs", out(reg) cs);
+        }
+
+        if self.iret.cs & CPL_MASK == cs & CPL_MASK {
+            // Privilege ring didn't change, so neither did the stack
+            all.esp = self as *const Self as usize // esp after Self was pushed to the stack
+                + size_of::<Self>() // disregard Self
+                - size_of::<usize>() * 2; // well, almost: esp and ss need to be excluded as they aren't present
+            unsafe {
+                core::arch::asm!("mov {}, ss", out(reg) all.ss);
+            }
+        } else {
+            all.esp = self.iret.esp;
+            all.ss = self.iret.ss;
+        }
+    }
+    pub fn frame_pointer(&self) -> usize {
+        self.preserved.ebp
+    }
+    pub fn stack_pointer(&self) -> usize {
+        self.iret.esp
+    }
+    pub fn set_stack_pointer(&mut self, esp: usize) {
+        self.iret.esp = esp;
+    }
+    pub fn instr_pointer(&self) -> usize {
+        self.iret.eip
+    }
+    pub fn sig_archdep_reg(&self) -> usize {
+        self.iret.eflags
+    }
+    pub fn set_instr_pointer(&mut self, eip: usize) {
+        self.iret.eip = eip;
+    }
+    /// Loads all registers from a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn load(&mut self, all: &IntRegisters) {
+        // TODO: Which of these should be allowed to change?
+
+        self.preserved.ebp = all.ebp;
+        self.preserved.esi = all.esi;
+        self.preserved.edi = all.edi;
+        self.preserved.ebx = all.ebx;
+        self.scratch.edx = all.edx;
+        self.scratch.ecx = all.ecx;
+        self.scratch.eax = all.eax;
+        self.iret.eip = all.eip;
+
+        // FIXME: The interrupt stack on which this is called, is always from userspace, but make
+        // the API safer.
+        self.iret.esp = all.esp;
+
+        // OF, DF, 0, TF => D
+        // SF, ZF, 0, AF => D
+        // 0, PF, 1, CF => 5
+        const ALLOWED_EFLAGS: usize = 0xDD5;
+
+        // These should probably be restricted
+        // self.iret.cs = all.cs;
+        self.iret.eflags &= !ALLOWED_EFLAGS;
+        self.iret.eflags |= all.eflags & ALLOWED_EFLAGS;
+    }
+    /// Enables the "Trap Flag" in the FLAGS register, causing the CPU
+    /// to send a Debug exception after the next instruction. This is
+    /// used for singlestep in the proc: scheme.
+    pub fn set_singlestep(&mut self, enabled: bool) {
+        if enabled {
+            self.iret.eflags |= FLAG_SINGLESTEP;
+        } else {
+            self.iret.eflags &= !FLAG_SINGLESTEP;
+        }
+    }
+}
+
+#[derive(Default)]
+#[repr(C, packed)]
+pub struct InterruptErrorStack {
+    pub code: usize,
+    pub inner: InterruptStack,
+}
+
+impl InterruptErrorStack {
+    pub fn dump(&self) {
+        println!("CODE:  {:08x}", { self.code });
+        self.inner.dump();
+    }
+    pub fn trace(&self) {
+        self.dump();
+        unsafe {
+            panic::user_stack_trace(&self.inner);
+            panic::stack_trace();
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! push_scratch {
+    () => {
+        "
+        // Push scratch registers (minus eax)
+        push ecx
+        push edx
+    "
+    };
+}
+#[macro_export]
+macro_rules! pop_scratch {
+    () => {
+        "
+        // Pop scratch registers
+        pop edx
+        pop ecx
+        pop eax
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! push_preserved {
+    () => {
+        "
+        // Push preserved registers
+        push ebx
+        push edi
+        push esi
+        push ebp
+    "
+    };
+}
+#[macro_export]
+macro_rules! pop_preserved {
+    () => {
+        "
+        // Pop preserved registers
+        pop ebp
+        pop esi
+        pop edi
+        pop ebx
+    "
+    };
+}
+
+// Must always happen after push_scratch
+macro_rules! enter_gs {
+    () => {
+        "
+        // Enter kernel GS segment
+        mov ecx, gs
+        push ecx
+        mov ecx, 0x18
+        mov gs, ecx
+    "
+    };
+}
+
+// Must always happen before pop_scratch
+macro_rules! exit_gs {
+    () => {
+        "
+        // Exit kernel GS segment
+        pop ecx
+        mov gs, ecx
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! interrupt_stack {
+    // XXX: Apparently we cannot use $expr and check for bool exhaustiveness, so we will have to
+    // use idents directly instead.
+    ($name:ident, |$stack:ident| $code:block) => {
+        #[unsafe(naked)]
+        pub unsafe extern "C" fn $name() {
+            unsafe extern "fastcall" fn inner($stack: &mut $crate::arch::x86::interrupt::InterruptStack) {
+                // TODO: Force the declarations to specify unsafe?
+
+                #[allow(unused_unsafe)]
+                unsafe {
+                    $code
+                }
+            }
+            core::arch::naked_asm!(
+                // Backup all userspace registers to stack
+                "push eax",
+                push_scratch!(),
+                push_preserved!(),
+
+                // Enter kernel TLS segment
+                enter_gs!(),
+
+                // TODO: Map PTI
+                // $crate::arch::x86::pti::map();
+
+                // Call inner function with pointer to stack
+                "
+                mov ecx, esp
+                call {inner}
+                ",
+
+                // TODO: Unmap PTI
+                // $crate::arch::x86::pti::unmap();
+
+                // Exit kernel TLS segment
+                exit_gs!(),
+
+                // Restore all userspace registers
+                pop_preserved!(),
+                pop_scratch!(),
+
+                "iretd",
+
+                inner = sym inner,
+            );
+        }
+    };
+    ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, |$stack| $code); };
+    ($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, |$stack| $code); }
+}
+
+#[macro_export]
+macro_rules! interrupt {
+    ($name:ident, || $code:block) => {
+        #[unsafe(naked)]
+        pub unsafe extern "C" fn $name() {
+            unsafe extern "C" fn inner() {
+                $code
+            }
+
+            core::arch::naked_asm!(
+                // Backup all userspace registers to stack
+                "push eax",
+                push_scratch!(),
+
+                // Enter kernel TLS segment
+                enter_gs!(),
+
+                // TODO: Map PTI
+                // $crate::arch::x86::pti::map();
+
+                // Call inner function with pointer to stack
+                "call {inner}",
+
+                // TODO: Unmap PTI
+                // $crate::arch::x86::pti::unmap();
+
+                // Exit kernel TLS segment
+                exit_gs!(),
+
+                // Restore all userspace registers
+                pop_scratch!(),
+
+                "iretd",
+
+                inner = sym inner,
+            );
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! interrupt_error {
+    ($name:ident, |$stack:ident, $error_code:ident| $code:block) => {
+        #[unsafe(naked)]
+        pub unsafe extern "C" fn $name() {
+            unsafe extern "C" fn inner($stack: &mut $crate::arch::x86::interrupt::handler::InterruptErrorStack) {
+                let $error_code: usize = $stack.code;
+                $code
+            }
+
+            core::arch::naked_asm!(
+                // Move eax into code's place, put code in last instead (to be
+                // compatible with InterruptStack)
+                "xchg [esp], eax",
+
+                // Push all userspace registers
+                push_scratch!(),
+                push_preserved!(),
+
+                // Enter kernel TLS segment
+                enter_gs!(),
+
+                // Put code in, it's now in eax
+                "push eax",
+
+                // TODO: Map PTI
+                // $crate::arch::x86::pti::map();
+
+                // Call inner function with pointer to stack
+                "
+                push esp
+                call {inner}
+                ",
+                // add esp, 4
+
+                // TODO: Unmap PTI (split "add esp, 8" into two "add esp, 4"s maybe?)
+                // $crate::arch::x86::pti::unmap();
+
+                // Pop previous esp and code
+                "add esp, 8",
+
+                // Exit kernel TLS segment
+                exit_gs!(),
+
+                // Restore all userspace registers
+                pop_preserved!(),
+                pop_scratch!(),
+
+                // The error code has already been popped, so use the regular macro.
+                "iretd",
+                inner = sym inner,
+            );
+        }
+    };
+}
+#[unsafe(naked)]
+unsafe extern "C" fn usercopy_trampoline() {
+    core::arch::naked_asm!(
+        "
+        pop esi
+        pop edi
+
+        mov eax, 1
+        ret
+    "
+    );
+}
+
+impl ArchIntCtx for InterruptStack {
+    fn ip(&self) -> usize {
+        self.iret.eip
+    }
+    fn recover_and_efault(&mut self) {
+        // Unlike on x86_64, Protected Mode interrupts will not save/restore esp and ss unless
+        // privilege rings changed, which they won't here as we are catching a kernel-induced page
+        // fault.
+        //
+        // Thus, it is only possible to change scratch/preserved registers, and EIP. While it may
+        // be feasible to set ECX to zero to stop the REP MOVSB, or increase EIP by 2 (REP MOVSB is
+        // f3 a4, i.e. 2 bytes), this trampoline allows any memcpy implementation, that reasonably
+        // pushes preserved registers to the stack.
+        self.iret.eip = usercopy_trampoline as usize;
+    }
+}
+
+#[unsafe(naked)]
+pub unsafe extern "C" fn enter_usermode() {
+    core::arch::naked_asm!(
+        // TODO: Unmap PTI
+        // $crate::arch::x86::pti::unmap();
+
+        // Exit kernel TLS segment
+        exit_gs!(),
+        // Restore all userspace registers
+        pop_preserved!(),
+        pop_scratch!(),
+        "iretd",
+    )
+}

src/arch/x86/interrupt/mod.rs

@@ -0,0 +1,10 @@
+//! Interrupt instructions
+
+pub use crate::arch::x86_shared::interrupt::*;
+
+#[macro_use]
+pub mod handler;
+
+pub mod syscall;
+
+pub use self::handler::InterruptStack;

src/arch/x86/interrupt/syscall.rs

@@ -0,0 +1,48 @@
+use crate::{
+    ptrace,
+    sync::CleanLockToken,
+    syscall,
+    syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL},
+};
+
+pub unsafe fn init() {}
+
+macro_rules! with_interrupt_stack {
+    (|$stack:ident, $token:ident| $code:block) => {{
+        let mut $token = CleanLockToken::new();
+
+        let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None, &mut $token)
+            .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE)));
+
+        if allowed.unwrap_or(true) {
+            // If the syscall is `clone`, the clone won't return here. Instead,
+            // it'll return early and leave any undropped values. This is
+            // actually GOOD, because any references are at that point UB
+            // anyway, because they are based on the wrong stack.
+            let $stack = &mut *$stack;
+            $code
+        }
+
+        ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None, &mut $token);
+    }};
+}
+
+interrupt_stack!(syscall, |stack| {
+    with_interrupt_stack!(|stack, token| {
+        let scratch = &stack.scratch;
+        let preserved = &stack.preserved;
+        let ret = syscall::syscall(
+            scratch.eax,
+            preserved.ebx,
+            scratch.ecx,
+            scratch.edx,
+            preserved.esi,
+            preserved.edi,
+            preserved.ebp,
+            &mut token,
+        );
+        stack.scratch.eax = ret;
+    })
+});
+
+pub use super::handler::enter_usermode;

src/arch/x86/mod.rs

@@ -0,0 +1,41 @@
+pub use crate::arch::x86_shared::*;
+
+/// Constants like memory locations
+pub mod consts;
+
+/// Interrupt instructions
+#[macro_use]
+pub mod interrupt;
+
+#[unsafe(naked)]
+pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
+    core::arch::naked_asm!(
+        "
+    .global __usercopy_start
+    __usercopy_start:
+        push edi
+        push esi
+
+        mov edi, [esp + 12] # dst
+        mov esi, [esp + 16] # src
+        mov ecx, [esp + 20] # len
+        rep movsb
+
+        pop esi
+        pop edi
+
+        xor eax, eax
+        ret
+    .global __usercopy_end
+    __usercopy_end:
+    "
+    );
+}
+pub use arch_copy_to_user as arch_copy_from_user;
+
+pub const KFX_SIZE: usize = 512;
+
+// This function exists as the KFX size is dynamic on x86_64.
+pub fn kfx_size() -> usize {
+    KFX_SIZE
+}

src/arch/x86_64/alternative.rs

@@ -0,0 +1,317 @@
+#![allow(unused_imports)]
+
+use spin::Once;
+use x86::controlregs::{Cr4, Xcr0};
+
+use crate::{
+    arch::cpuid::{cpuid, feature_info, has_ext_feat},
+    context::memory::PageSpan,
+    memory::{KernelMapper, Page, PageFlags, VirtualAddress, PAGE_SIZE},
+};
+
+#[cfg(all(cpu_feature_never = "xsave", not(cpu_feature_never = "xsaveopt")))]
+compile_error!("cannot force-disable xsave without force-disabling xsaveopt");
+
+#[repr(C)]
+#[derive(Clone, Copy, Debug)]
+pub struct AltReloc {
+    // These two fields point to a utf-8 name of the feature, see the match statement below.
+    pub name_start: *const u8,
+    pub name_len: usize,
+
+    // Base address of the code that may later be overwritten.
+    pub code_start: *mut u8,
+    // Length of the default code, excluding NOPs if the altcode sequence is longer.
+    pub origcode_len: usize,
+    // Actual length of the overwritable code, i.e. max(origcode_len, altcode_len).
+    pub padded_len: usize,
+    pub _rsvd: usize,
+
+    // These two fields point to the alternative code (in .rodata), and possible new nop bytes,
+    // that will replace the code_start..+padded_len
+    pub altcode_start: *const u8,
+    pub altcode_len: usize,
+}
+
+#[cold]
+pub unsafe fn early_init(bsp: bool) {
+    unsafe {
+        let relocs_offset = crate::kernel_executable_offsets::__altrelocs_start();
+        // __altrelocs_end > __altrelocs_start so this cannot overflow
+        #[expect(clippy::arithmetic_side_effects)]
+        let relocs_size = crate::kernel_executable_offsets::__altrelocs_end() - relocs_offset;
+
+        // AltReloc is not a ZST so the modulo and division will never panic
+        #[expect(clippy::arithmetic_side_effects)]
+        {
+            assert_eq!(relocs_size % size_of::<AltReloc>(), 0)
+        }
+        #[expect(clippy::arithmetic_side_effects)]
+        let relocs = core::slice::from_raw_parts(
+            relocs_offset as *const AltReloc,
+            relocs_size / size_of::<AltReloc>(),
+        );
+
+        let mut enable = KcpuFeatures::empty();
+
+        if cfg!(not(cpu_feature_never = "smap")) && has_ext_feat(|feat| feat.has_smap()) {
+            // SMAP (Supervisor-Mode Access Prevention) forbids the kernel from accessing any
+            // userspace-accessible pages, with the necessary exception of when RFLAGS.AC = 1. This
+            // limits user-memory accesses to the UserSlice wrapper, so that no data outside of
+            // usercopy functions can be accidentally accessed by the kernel.
+            x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMAP);
+            // Clear CLAC in (the probably unlikely) case the bootloader set it earlier.
+            x86::bits64::rflags::clac();
+
+            enable |= KcpuFeatures::SMAP;
+        } else {
+            assert!(cfg!(not(cpu_feature_always = "smap")));
+        }
+
+        if cfg!(not(cpu_feature_never = "fsgsbase"))
+            && let Some(f) = cpuid().get_extended_feature_info()
+            && f.has_fsgsbase()
+        {
+            x86::controlregs::cr4_write(
+                x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_FSGSBASE,
+            );
+
+            enable |= KcpuFeatures::FSGSBASE;
+        } else {
+            assert!(cfg!(not(cpu_feature_always = "fsgsbase")));
+        }
+
+        #[cfg(not(cpu_feature_never = "xsave"))]
+        if feature_info().has_xsave() {
+            use raw_cpuid::{ExtendedRegisterStateLocation, ExtendedRegisterType};
+
+            x86::controlregs::cr4_write(
+                x86::controlregs::cr4() | x86::controlregs::Cr4::CR4_ENABLE_OS_XSAVE,
+            );
+
+            let mut xcr0 = Xcr0::XCR0_FPU_MMX_STATE | Xcr0::XCR0_SSE_STATE;
+            x86::controlregs::xcr0_write(xcr0);
+            let ext_state_info = cpuid()
+                .get_extended_state_info()
+                .expect("must be present if XSAVE is supported");
+
+            enable |= KcpuFeatures::XSAVE;
+            enable.set(KcpuFeatures::XSAVEOPT, ext_state_info.has_xsaveopt());
+
+            let info = xsave::XsaveInfo {
+                ymm_upper_offset: feature_info().has_avx().then(|| {
+                    xcr0 |= Xcr0::XCR0_AVX_STATE;
+                    x86::controlregs::xcr0_write(xcr0);
+
+                    let state = ext_state_info
+                        .iter()
+                        .find(|state| {
+                            state.register() == ExtendedRegisterType::Avx
+                                && state.location() == ExtendedRegisterStateLocation::Xcr0
+                        })
+                        .expect("CPUID said AVX was supported but there's no state info");
+
+                    // 16 * size_of::<u128>() is well below usize::MAX
+                    #[expect(clippy::arithmetic_side_effects)]
+                    if state.size() as usize != 16 * size_of::<u128>() {
+                        warn!("Unusual AVX state size {}", state.size());
+                    }
+
+                    state.offset()
+                }),
+                xsave_size: ext_state_info.xsave_area_size_enabled_features(),
+            };
+            debug!("XSAVE: {:?}", info);
+
+            xsave::XSAVE_INFO.call_once(|| info);
+        } else {
+            assert!(cfg!(not(cpu_feature_always = "xsave")));
+        }
+
+        if !bsp {
+            return;
+        }
+
+        overwrite(relocs, enable);
+
+        if cfg!(not(feature = "self_modifying")) {
+            assert!(
+                cfg!(not(cpu_feature_auto = "smap"))
+                    && cfg!(not(cpu_feature_auto = "fsgsbase"))
+                    && cfg!(not(cpu_feature_auto = "xsave"))
+                    && cfg!(not(cpu_feature_auto = "xsaveopt"))
+            );
+        }
+
+        FEATURES.call_once(|| enable);
+    }
+}
+
+unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) {
+    unsafe {
+        if cfg!(not(feature = "self_modifying")) {
+            return;
+        }
+
+        debug!("self-modifying features: {:?}", enable);
+
+        let mut mapper = KernelMapper::lock_rw();
+        for reloc in relocs.iter().copied() {
+            let name = core::str::from_utf8(core::slice::from_raw_parts(
+                reloc.name_start,
+                reloc.name_len,
+            ))
+            .expect("invalid feature name");
+            let altcode = core::slice::from_raw_parts(reloc.altcode_start, reloc.altcode_len);
+
+            let dst_pages = PageSpan::between(
+                Page::containing_address(VirtualAddress::new(reloc.code_start as usize)),
+                Page::containing_address(VirtualAddress::new(
+                    (reloc.code_start as usize + reloc.padded_len).next_multiple_of(PAGE_SIZE),
+                )),
+            );
+            for page in dst_pages.pages() {
+                mapper
+                    .remap(
+                        page.start_address(),
+                        PageFlags::new().write(true).execute(true).global(true),
+                    )
+                    .unwrap()
+                    .flush();
+            }
+
+            let code = core::slice::from_raw_parts_mut(reloc.code_start, reloc.padded_len);
+
+            trace!(
+                "feature {} current {:x?} altcode {:x?}",
+                name,
+                code,
+                altcode
+            );
+
+            let feature_is_enabled = match name {
+                "smap" => enable.contains(KcpuFeatures::SMAP),
+                "fsgsbase" => enable.contains(KcpuFeatures::FSGSBASE),
+                "xsave" => enable.contains(KcpuFeatures::XSAVE),
+                "xsaveopt" => enable.contains(KcpuFeatures::XSAVEOPT),
+                //_ => panic!("unknown altcode relocation: {}", name),
+                _ => true,
+            };
+
+            // XXX: The `.nops` directive only works for constant lengths, and the variable `.skip -X`
+            // only outputs the (slower) single-byte 0x90 NOP.
+
+            // This table is from the "Software Optimization Guide for AMD Family 19h Processors" (November
+            // 2020).
+            const NOPS_TABLE: [&[u8]; 11] = [
+                &[0x90],
+                &[0x66, 0x90],
+                &[0x0f, 0x1f, 0x00],
+                &[0x0f, 0x1f, 0x40, 0x00],
+                &[0x0f, 0x1f, 0x44, 0x00, 0x00],
+                &[0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00],
+                &[0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00],
+                &[0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+                &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+                &[0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00],
+                &[
+                    0x66, 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
+                ],
+            ];
+
+            if feature_is_enabled {
+                trace!("feature {} origcode {:x?}", name, code);
+                let (dst, dst_nops) = code.split_at_mut(altcode.len());
+                dst.copy_from_slice(altcode);
+
+                for chunk in dst_nops.chunks_mut(NOPS_TABLE.len()) {
+                    // `chunk.len() - 1` is always in bounds because we are chunking by
+                    // `NOPS_TABLE.len()`
+                    #[expect(clippy::indexing_slicing)]
+                    // `chunk.len()` will never be 0
+                    #[expect(clippy::arithmetic_side_effects)]
+                    chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]);
+                }
+                trace!("feature {} new {:x?} altcode {:x?}", name, code, altcode);
+            } else {
+                trace!("feature !{} origcode {:x?}", name, code);
+                let (_, padded) = code.split_at_mut(reloc.origcode_len);
+
+                // Not strictly necessary, but reduces the number of instructions using longer nop
+                // instructions.
+                for chunk in padded.chunks_mut(NOPS_TABLE.len()) {
+                    // `chunk.len() - 1` is always in bounds because we are chunking by
+                    // `NOPS_TABLE.len()`
+                    #[expect(clippy::indexing_slicing)]
+                    // `chunk.len()` will never be 0
+                    #[expect(clippy::arithmetic_side_effects)]
+                    chunk.copy_from_slice(NOPS_TABLE[chunk.len() - 1]);
+                }
+
+                trace!("feature !{} new {:x?}", name, code);
+            }
+
+            for page in dst_pages.pages() {
+                mapper
+                    .remap(
+                        page.start_address(),
+                        PageFlags::new().write(false).execute(true).global(true),
+                    )
+                    .unwrap()
+                    .flush();
+            }
+        }
+    }
+}
+
+bitflags! {
+    #[derive(Clone, Copy, Debug)]
+    pub struct KcpuFeatures: usize {
+        const SMAP = 1;
+        const FSGSBASE = 2;
+        const XSAVE = 4;
+        const XSAVEOPT = 8;
+    }
+}
+
+static FEATURES: Once<KcpuFeatures> = Once::new();
+
+pub fn features() -> KcpuFeatures {
+    *FEATURES.get().expect("early_cpu_init was not called")
+}
+
+#[cfg(not(cpu_feature_never = "xsave"))]
+mod xsave {
+    use spin::Once;
+
+    #[derive(Debug)]
+    pub struct XsaveInfo {
+        pub ymm_upper_offset: Option<u32>,
+        pub xsave_size: u32,
+    }
+    pub(super) static XSAVE_INFO: Once<XsaveInfo> = Once::new();
+
+    pub fn info() -> Option<&'static XsaveInfo> {
+        XSAVE_INFO.get()
+    }
+}
+
+pub fn kfx_size() -> usize {
+    #[cfg(not(cpu_feature_never = "xsave"))]
+    {
+        // This wont overflow
+        #[expect(clippy::arithmetic_side_effects)]
+        match xsave::info() {
+            Some(info) => FXSAVE_SIZE + XSAVE_HEADER_SIZE + info.xsave_size as usize,
+            None => FXSAVE_SIZE,
+        }
+    }
+    #[cfg(cpu_feature_never = "xsave")]
+    {
+        // FXSAVE size
+        FXSAVE_SIZE
+    }
+}
+
+pub const FXSAVE_SIZE: usize = 512;
+pub const XSAVE_HEADER_SIZE: usize = 64;

src/arch/x86_64/consts.rs

@@ -0,0 +1,26 @@
+// Because the memory map is so important to not be aliased, it is defined here, in one place.
+//
+// - The lower half (256 PML4 entries; 128 TiB) is reserved for userspace. These mappings are
+//   associated with _address spaces_, and change when context switching, unless the address spaces
+//   match.
+// - The upper half is reserved for the kernel. Kernel mappings are preserved across context
+//   switches.
+//
+// Each PML4 entry references 512 GiB of virtual memory.
+
+/// The size of a single PML4
+pub const PML4_SIZE: usize = 0x0000_0080_0000_0000;
+
+/// Offset of kernel
+const KERNEL_OFFSET: usize = (1_usize << 31).wrapping_neg();
+
+/// Offset to kernel heap
+#[inline(always)]
+pub fn kernel_heap_offset() -> usize {
+    crate::kernel_executable_offsets::KERNEL_OFFSET() - PML4_SIZE
+}
+
+/// End offset of the user image, i.e. kernel start
+// TODO: Make this offset at least PAGE_SIZE less? There are known hardware bugs on some arches,
+// for example on x86 if instructions execute near the 48-bit canonical address boundary.
+pub const USER_END_OFFSET: usize = 256 * PML4_SIZE;

src/arch/x86_64/interrupt/handler.rs

@@ -0,0 +1,532 @@
+use crate::{arch::flags::FLAG_SINGLESTEP, memory::ArchIntCtx, panic, syscall::IntRegisters};
+
+#[derive(Default)]
+#[repr(C)]
+pub struct ScratchRegisters {
+    pub r11: usize,
+    pub r10: usize,
+    pub r9: usize,
+    pub r8: usize,
+    pub rsi: usize,
+    pub rdi: usize,
+    pub rdx: usize,
+    pub rcx: usize,
+    pub rax: usize,
+}
+
+impl ScratchRegisters {
+    pub fn dump(&self) {
+        println!("RAX:   {:016x}", { self.rax });
+        println!("RCX:   {:016x}", { self.rcx });
+        println!("RDX:   {:016x}", { self.rdx });
+        println!("RDI:   {:016x}", { self.rdi });
+        println!("RSI:   {:016x}", { self.rsi });
+        println!("R8:    {:016x}", { self.r8 });
+        println!("R9:    {:016x}", { self.r9 });
+        println!("R10:   {:016x}", { self.r10 });
+        println!("R11:   {:016x}", { self.r11 });
+    }
+}
+
+#[derive(Default)]
+#[repr(C)]
+pub struct PreservedRegisters {
+    pub r15: usize,
+    pub r14: usize,
+    pub r13: usize,
+    pub r12: usize,
+    pub rbp: usize,
+    pub rbx: usize,
+}
+
+impl PreservedRegisters {
+    pub fn dump(&self) {
+        println!("RBX:   {:016x}", { self.rbx });
+        println!("RBP:   {:016x}", { self.rbp });
+        println!("R12:   {:016x}", { self.r12 });
+        println!("R13:   {:016x}", { self.r13 });
+        println!("R14:   {:016x}", { self.r14 });
+        println!("R15:   {:016x}", { self.r15 });
+    }
+}
+
+#[derive(Default)]
+#[repr(C)]
+pub struct IretRegisters {
+    pub rip: usize,
+    pub cs: usize,
+    pub rflags: usize,
+
+    // In x86 Protected Mode, i.e. 32-bit kernels, the following two registers are conditionally
+    // pushed if the privilege ring changes. In x86 Long Mode however, i.e. 64-bit kernels, they
+    // are unconditionally pushed, mostly due to stack alignment requirements.
+    pub rsp: usize,
+    pub ss: usize,
+}
+
+impl IretRegisters {
+    pub fn dump(&self) {
+        println!("RFLAG: {:016x}", { self.rflags });
+        println!("CS:    {:016x}", { self.cs });
+        println!("RIP:   {:016x}", { self.rip });
+
+        println!("RSP:   {:016x}", { self.rsp });
+        println!("SS:    {:016x}", { self.ss });
+
+        unsafe {
+            let fsbase = x86::msr::rdmsr(x86::msr::IA32_FS_BASE);
+            let gsbase = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE);
+            let kgsbase = x86::msr::rdmsr(x86::msr::IA32_GS_BASE);
+            println!(
+                "FSBASE  {:016x}\nGSBASE  {:016x}\nKGSBASE {:016x}",
+                fsbase, gsbase, kgsbase
+            );
+        }
+    }
+}
+
+#[derive(Default)]
+#[repr(C)]
+pub struct InterruptStack {
+    pub preserved: PreservedRegisters,
+    pub scratch: ScratchRegisters,
+    pub iret: IretRegisters,
+}
+
+impl InterruptStack {
+    pub fn init(&mut self) {
+        // Always enable interrupts!
+        self.iret.rflags = x86::bits64::rflags::RFlags::FLAGS_IF.bits() as usize;
+        self.iret.cs = (crate::arch::gdt::GDT_USER_CODE << 3) | 3;
+        self.iret.ss = (crate::arch::gdt::GDT_USER_DATA << 3) | 3;
+    }
+    pub fn frame_pointer(&self) -> usize {
+        self.preserved.rbp
+    }
+    pub fn stack_pointer(&self) -> usize {
+        self.iret.rsp
+    }
+    pub fn set_stack_pointer(&mut self, rsp: usize) {
+        self.iret.rsp = rsp;
+    }
+    pub fn instr_pointer(&self) -> usize {
+        self.iret.rip
+    }
+    pub fn sig_archdep_reg(&self) -> usize {
+        self.iret.rflags
+    }
+    pub fn set_instr_pointer(&mut self, rip: usize) {
+        self.iret.rip = rip;
+    }
+    pub fn set_arg1(&mut self, arg_opt: Option<usize>) {
+        if let Some(arg) = arg_opt {
+            self.scratch.rsi = arg;
+        }
+    }
+    pub fn dump(&self) {
+        self.iret.dump();
+        self.scratch.dump();
+        self.preserved.dump();
+    }
+    pub fn trace(&self) {
+        self.dump();
+        unsafe {
+            panic::user_stack_trace(self);
+            panic::stack_trace();
+        }
+    }
+    /// Saves all registers to a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn save(&self, all: &mut IntRegisters) {
+        all.r15 = self.preserved.r15;
+        all.r14 = self.preserved.r14;
+        all.r13 = self.preserved.r13;
+        all.r12 = self.preserved.r12;
+        all.rbp = self.preserved.rbp;
+        all.rbx = self.preserved.rbx;
+        all.r11 = self.scratch.r11;
+        all.r10 = self.scratch.r10;
+        all.r9 = self.scratch.r9;
+        all.r8 = self.scratch.r8;
+        all.rsi = self.scratch.rsi;
+        all.rdi = self.scratch.rdi;
+        all.rdx = self.scratch.rdx;
+        all.rcx = self.scratch.rcx;
+        all.rax = self.scratch.rax;
+        all.rip = self.iret.rip;
+        all.cs = self.iret.cs;
+        all.rflags = self.iret.rflags;
+        all.rsp = self.iret.rsp;
+        all.ss = self.iret.ss;
+    }
+    /// Loads all registers from a struct used by the proc:
+    /// scheme to read/write registers.
+    pub fn load(&mut self, all: &IntRegisters) {
+        self.preserved.r15 = all.r15;
+        self.preserved.r14 = all.r14;
+        self.preserved.r13 = all.r13;
+        self.preserved.r12 = all.r12;
+        self.preserved.rbp = all.rbp;
+        self.preserved.rbx = all.rbx;
+        self.scratch.r11 = all.r11;
+        self.scratch.r10 = all.r10;
+        self.scratch.r9 = all.r9;
+        self.scratch.r8 = all.r8;
+        self.scratch.rsi = all.rsi;
+        self.scratch.rdi = all.rdi;
+        self.scratch.rdx = all.rdx;
+        self.scratch.rcx = all.rcx;
+        self.scratch.rax = all.rax;
+        self.iret.rip = all.rip;
+        self.iret.rsp = all.rsp;
+
+        // CS and SS are immutable, at least their privilege levels.
+
+        // OF, DF, 0, TF => D
+        // SF, ZF, 0, AF => D
+        // 0, PF, 1, CF => 5
+        const ALLOWED_RFLAGS: usize = 0xDD5;
+
+        self.iret.rflags &= !ALLOWED_RFLAGS;
+        self.iret.rflags |= all.rflags & ALLOWED_RFLAGS;
+    }
+    /// Enables the "Trap Flag" in the FLAGS register, causing the CPU
+    /// to send a Debug exception after the next instruction. This is
+    /// used for singlestep in the proc: scheme.
+    pub fn set_singlestep(&mut self, enabled: bool) {
+        if enabled {
+            self.iret.rflags |= FLAG_SINGLESTEP;
+        } else {
+            self.iret.rflags &= !FLAG_SINGLESTEP;
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! push_scratch {
+    () => {
+        "
+        // Push scratch registers
+        push rcx
+        push rdx
+        push rdi
+        push rsi
+        push r8
+        push r9
+        push r10
+        push r11
+    "
+    };
+}
+#[macro_export]
+macro_rules! pop_scratch {
+    () => {
+        "
+        // Pop scratch registers
+        pop r11
+        pop r10
+        pop r9
+        pop r8
+        pop rsi
+        pop rdi
+        pop rdx
+        pop rcx
+        pop rax
+    "
+    };
+}
+
+#[macro_export]
+macro_rules! push_preserved {
+    () => {
+        "
+        // Push preserved registers
+        push rbx
+        push rbp
+        push r12
+        push r13
+        push r14
+        push r15
+    "
+    };
+}
+#[macro_export]
+macro_rules! pop_preserved {
+    () => {
+        "
+        // Pop preserved registers
+        pop r15
+        pop r14
+        pop r13
+        pop r12
+        pop rbp
+        pop rbx
+    "
+    };
+}
+macro_rules! swapgs_iff_ring3_fast {
+    // TODO: Spectre V1: LFENCE?
+    () => {
+        "
+        // Check whether the last two bits RSP+8 (code segment) are equal to zero.
+        test QWORD PTR [rsp + 8], 0x3
+        // Skip the SWAPGS instruction if CS & 0b11 == 0b00.
+        jz 2f
+        swapgs
+        2:
+    "
+    };
+}
+macro_rules! swapgs_iff_ring3_fast_errorcode {
+    // TODO: Spectre V1: LFENCE?
+    () => {
+        "
+        test QWORD PTR [rsp + 16], 0x3
+        jz 2f
+        swapgs
+        2:
+    "
+    };
+}
+
+macro_rules! conditional_swapgs_paranoid {
+    // For regular interrupt handlers and the syscall handler, managing IA32_GS_BASE and
+    // IA32_KERNEL_GS_BASE (the "GSBASE registers") is more or less trivial when using the SWAPGS
+    // instruction.
+    //
+    // The syscall handler simply runs SWAPGS, as syscalls can only originate from usermode,
+    // whereas interrupt handlers conditionally SWAPGS unless the interrupt was triggered from
+    // kernel mode, in which case the "swap state" is already valid, and there is no need to
+    // SWAPGS.
+    //
+    // Handling GSBASE correctly for paranoid interrupts however, is not as simple. NMIs can occur
+    // between the check of whether an interrupt came from usermode, and the actual SWAPGS
+    // instruction. #DB can also be triggered inside of a kernel interrupt handler, due to
+    // breakpoints, even though setting up such breakpoints in the first place, is not yet
+    // supported by the kernel.
+    //
+    // Luckily, the GDT always resides in the PCR (at least after init_paging, but there are no
+    // interrupt handlers set up before that), allowing GSBASE to be calculated relatively cheaply.
+    // Out of the two GSBASE registers, at least one must be *the* kernel GSBASE, allowing for a
+    // simple conditional SWAPGS.
+    //
+    // (An alternative to conditionally executing SWAPGS, would be to save and restore GSBASE via
+    // e.g. the stack. That would nonetheless require saving and restoring both GSBASE registers,
+    // if the interrupt handler should be allowed to context switch, which the current #DB handler
+    // may do.)
+    //
+    // TODO: Handle nested NMIs like Linux does (https://lwn.net/Articles/484932/)?.
+
+    () => { concat!(
+        // Put the GDT base pointer in RDI.
+        "
+        sub rsp, 16
+        sgdt [rsp + 6]
+        mov rdi, [rsp + 8]
+        add rsp, 16
+        ",
+        // Calculate the PCR address by subtracting the offset of the GDT in the PCR struct.
+        "sub rdi, {PCR_GDT_OFFSET};",
+
+        // Read the current IA32_GS_BASE value into RDX.
+        alternative!(
+            feature: "fsgsbase",
+            then: ["rdgsbase rdx"],
+            default: ["
+                mov ecx, {IA32_GS_BASE}
+                rdmsr
+                shl rdx, 32
+                or rdx, rax
+            "]
+        ),
+
+        // If they were not equal, the PCR address must instead be in IA32_KERNEL_GS_BASE,
+        // requiring a SWAPGS. GSBASE needs to be swapped back, so store the same flag in RBX.
+
+        // TODO: Spectre V1: LFENCE?
+        "
+        cmp rdx, rdi
+        sete bl
+        je 2f
+        swapgs
+        2:
+        ",
+    ) }
+}
+macro_rules! conditional_swapgs_back_paranoid {
+    () => {
+        "
+        test bl, bl
+        jnz 2f
+        swapgs
+        2:
+    "
+    };
+}
+macro_rules! nop {
+    () => {
+        "
+        // Unused: {IA32_GS_BASE} {PCR_GDT_OFFSET}
+        "
+    };
+}
+
+#[macro_export]
+macro_rules! interrupt_stack {
+    // XXX: Apparently we cannot use $expr and check for bool exhaustiveness, so we will have to
+    // use idents directly instead.
+    ($name:ident, $save1:ident!, $save2:ident!, $rstor2:ident!, $rstor1:ident!, is_paranoid: $is_paranoid:expr_2021, |$stack:ident| $code:block) => {
+        #[unsafe(naked)]
+        pub unsafe extern "C" fn $name() {
+            unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::InterruptStack) {
+                $code
+            }
+            core::arch::naked_asm!(
+                // Clear direction flag, required by ABI when running any Rust code in the kernel.
+                "cld;",
+
+                // Backup all userspace registers to stack
+                $save1!(),
+                "push rax",
+                push_scratch!(),
+                push_preserved!(),
+
+                $save2!(),
+
+                // TODO: Map PTI
+                // $crate::arch::x86_64::pti::map();
+
+                // Call inner function with pointer to stack
+                "
+                mov rdi, rsp
+                call {inner}
+                ",
+
+                // TODO: Unmap PTI
+                // $crate::arch::x86_64::pti::unmap();
+
+                $rstor2!(),
+
+                // Restore all userspace registers
+                pop_preserved!(),
+                pop_scratch!(),
+
+                $rstor1!(),
+                "iretq",
+
+                inner = sym inner,
+                IA32_GS_BASE = const(x86::msr::IA32_GS_BASE),
+
+                PCR_GDT_OFFSET = const(core::mem::offset_of!($crate::arch::gdt::ProcessorControlRegion, gdt)),
+            );
+        }
+    };
+    ($name:ident, |$stack:ident| $code:block) => { interrupt_stack!($name, swapgs_iff_ring3_fast!, nop!, nop!, swapgs_iff_ring3_fast!, is_paranoid: false, |$stack| $code); };
+    ($name:ident, @paranoid, |$stack:ident| $code:block) => { interrupt_stack!($name, nop!, conditional_swapgs_paranoid!, conditional_swapgs_back_paranoid!, nop!, is_paranoid: true, |$stack| $code); }
+}
+
+#[macro_export]
+macro_rules! interrupt {
+    ($name:ident, || $code:block) => {
+        #[unsafe(naked)]
+        pub unsafe extern "C" fn $name() {
+            unsafe extern "C" fn inner() {
+                $code
+            }
+
+            core::arch::naked_asm!(
+                // Clear direction flag, required by ABI when running any Rust code in the kernel.
+                "cld;",
+
+                // Backup all userspace registers to stack
+                swapgs_iff_ring3_fast!(),
+                "push rax",
+                push_scratch!(),
+
+                // TODO: Map PTI
+                // $crate::arch::x86_64::pti::map();
+
+                // Call inner function with pointer to stack
+                "call {inner}",
+
+                // TODO: Unmap PTI
+                // $crate::arch::x86_64::pti::unmap();
+
+                // Restore all userspace registers
+                pop_scratch!(),
+
+                swapgs_iff_ring3_fast!(),
+                "iretq",
+
+                inner = sym inner,
+            );
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! interrupt_error {
+    ($name:ident, |$stack:ident, $error_code:ident| $code:block) => {
+        #[unsafe(naked)]
+        pub unsafe extern "C" fn $name() {
+            unsafe extern "C" fn inner($stack: &mut $crate::arch::x86_64::interrupt::handler::InterruptStack, $error_code: usize) {
+                $code
+            }
+
+            core::arch::naked_asm!(
+                // Clear direction flag, required by ABI when running any Rust code in the kernel.
+                "cld;",
+
+                swapgs_iff_ring3_fast_errorcode!(),
+
+                // Don't push RAX yet, as the error code is already stored in RAX's position.
+
+                // Push all userspace registers
+                push_scratch!(),
+                push_preserved!(),
+
+                // Now that we have a couple of usable registers, put the error code in the second
+                // argument register for the inner function, and save RAX where it would normally
+                // be.
+                "mov rsi, [rsp + {rax_offset}];",
+                "mov [rsp + {rax_offset}], rax;",
+
+                // TODO: Map PTI
+                // $crate::arch::x86_64::pti::map();
+
+                // Call inner function with pointer to stack, and error code.
+                "mov rdi, rsp;",
+                "call {inner};",
+
+                // TODO: Unmap PTI
+                // $crate::arch::x86_64::pti::unmap();
+
+                // Restore all userspace registers
+                pop_preserved!(),
+                pop_scratch!(),
+
+                // The error code has already been popped, so use the regular macro.
+                swapgs_iff_ring3_fast!(),
+                "iretq;",
+
+                inner = sym inner,
+                rax_offset = const(size_of::<$crate::arch::interrupt::handler::PreservedRegisters>() + size_of::<$crate::arch::interrupt::handler::ScratchRegisters>() - 8),
+            );
+        }
+    };
+}
+
+impl ArchIntCtx for InterruptStack {
+    fn ip(&self) -> usize {
+        self.iret.rip
+    }
+    fn recover_and_efault(&mut self) {
+        // We were inside a usercopy function that failed. This is handled by setting rax to a
+        // nonzero value, and emulating the ret instruction.
+        self.scratch.rax = 1;
+        let ret_addr = unsafe { (self.iret.rsp as *const usize).read() };
+        self.iret.rsp += 8;
+        self.iret.rip = ret_addr;
+        self.iret.rflags &= !(1 << 18);
+    }
+}

src/arch/x86_64/interrupt/mod.rs

@@ -0,0 +1,10 @@
+//! Interrupt instructions
+
+pub use crate::arch::x86_shared::interrupt::*;
+
+#[macro_use]
+pub mod handler;
+
+pub mod syscall;
+
+pub use self::handler::InterruptStack;

src/arch/x86_64/interrupt/syscall.rs

@@ -0,0 +1,195 @@
+use crate::{
+    arch::{gdt, interrupt::InterruptStack},
+    ptrace,
+    sync::CleanLockToken,
+    syscall,
+    syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_POST_SYSCALL, PTRACE_STOP_PRE_SYSCALL},
+};
+use core::mem::offset_of;
+use x86::{
+    bits64::{rflags::RFlags, task::TaskStateSegment},
+    msr,
+    segmentation::SegmentSelector,
+};
+
+pub unsafe fn init() {
+    unsafe {
+        // IA32_STAR[31:0] are reserved.
+
+        // The base selector of the two consecutive segments for kernel code and the immediately
+        // succeeding stack (data).
+        let syscall_cs_ss_base = (gdt::GDT_KERNEL_CODE as u16) << 3;
+        // The base selector of the three consecutive segments (of which two are used) for user code
+        // and user data. It points to a 32-bit code segment, which must be followed by a data segment
+        // (stack), and a 64-bit code segment.
+        let sysret_cs_ss_base = ((gdt::GDT_USER_CODE32_UNUSED as u16) << 3) | 3;
+        let star_high = u32::from(syscall_cs_ss_base) | (u32::from(sysret_cs_ss_base) << 16);
+
+        msr::wrmsr(msr::IA32_STAR, u64::from(star_high) << 32);
+        #[expect(clippy::fn_to_numeric_cast)]
+        msr::wrmsr(msr::IA32_LSTAR, syscall_instruction as u64);
+
+        // DF needs to be cleared, required by the compiler ABI. If DF were not part of FMASK,
+        // userspace would be able to reverse the direction of in-kernel REP MOVS/STOS/(CMPS/SCAS), and
+        // cause all sorts of memory corruption.
+        //
+        // IF needs to be cleared, as the kernel currently assumes interrupts are disabled except in
+        // usermode and in kmain.
+        //
+        // TF needs to be cleared, as enabling userspace-rflags-controlled singlestep in the kernel
+        // would be a bad idea.
+        //
+        // AC it should always be cleared when entering the kernel (and never be set except in usercopy
+        // functions), if for some reason AC was set before entering userspace (AC can only be modified
+        // by kernel code).
+        //
+        // The other flags could indeed be preserved and excluded from FMASK, but since they are not
+        // used to pass data to the kernel, they might as well be masked with *marginal* security
+        // benefits.
+        //
+        // Flags not included here are IOPL (not relevant to the kernel at all), "CPUID flag" (not used
+        // at all in 64-bit mode), RF (not used yet, but DR breakpoints would remain enabled both in
+        // user and kernel mode), VM8086 (not used at all), and VIF/VIP (system-level status flags?).
+
+        let mask_critical =
+            RFlags::FLAGS_DF | RFlags::FLAGS_IF | RFlags::FLAGS_TF | RFlags::FLAGS_AC;
+        let mask_other = RFlags::FLAGS_CF
+            | RFlags::FLAGS_PF
+            | RFlags::FLAGS_AF
+            | RFlags::FLAGS_ZF
+            | RFlags::FLAGS_SF
+            | RFlags::FLAGS_OF;
+        msr::wrmsr(msr::IA32_FMASK, (mask_critical | mask_other).bits());
+
+        let efer = msr::rdmsr(msr::IA32_EFER);
+        msr::wrmsr(msr::IA32_EFER, efer | 1);
+    }
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn __inner_syscall_instruction(stack: *mut InterruptStack) {
+    unsafe {
+        let mut token = CleanLockToken::new();
+        let allowed = ptrace::breakpoint_callback(PTRACE_STOP_PRE_SYSCALL, None, &mut token)
+            .and_then(|_| ptrace::next_breakpoint().map(|f| !f.contains(PTRACE_FLAG_IGNORE)));
+
+        if allowed.unwrap_or(true) {
+            let scratch = &(*stack).scratch;
+
+            let ret = syscall::syscall(
+                scratch.rax,
+                scratch.rdi,
+                scratch.rsi,
+                scratch.rdx,
+                scratch.r10,
+                scratch.r8,
+                scratch.r9,
+                &mut token,
+            );
+            (*stack).scratch.rax = ret;
+        }
+
+        ptrace::breakpoint_callback(PTRACE_STOP_POST_SYSCALL, None, &mut token);
+    }
+}
+
+#[unsafe(naked)]
+pub unsafe extern "C" fn syscall_instruction() {
+    core::arch::naked_asm!(
+    // Yes, this is magic. No, you don't need to understand
+    "swapgs;",                    // Swap KGSBASE with GSBASE, allowing fast TSS access.
+    "mov gs:[{sp}], rsp;",        // Save userspace stack pointer
+    "mov rsp, gs:[{ksp}];",       // Load kernel stack pointer
+    "push QWORD PTR {ss_sel};",   // Push fake userspace SS (resembling iret frame)
+    "push QWORD PTR gs:[{sp}];",  // Push userspace rsp
+    "push r11;",                  // Push rflags
+    "push QWORD PTR {cs_sel};",   // Push fake CS (resembling iret stack frame)
+    "push rcx;",                  // Push userspace return pointer
+
+    // Push context registers
+    "push rax;",
+    push_scratch!(),
+    push_preserved!(),
+
+    // TODO: Map PTI
+    // $crate::arch::x86_64::pti::map();
+
+    // Call inner funtion
+    "mov rdi, rsp;",
+    "call __inner_syscall_instruction;",
+
+    // TODO: Unmap PTI
+    // $crate::arch::x86_64::pti::unmap();
+
+    "
+    .globl enter_usermode
+    enter_usermode:
+    ",
+
+    // Pop context registers
+    pop_preserved!(),
+    pop_scratch!(),
+
+    // Restore user GSBASE by swapping GSBASE and KGSBASE.
+    "swapgs;",
+
+    // TODO: Should we unconditionally jump or avoid jumping, to hint to the branch predictor that
+    // singlestep is NOT set?
+    //
+    // It appears Intel CPUs assume (previously unknown) forward conditional branches to not be
+    // taken, and AMD appears to assume all previously unknown conditional branches will not be
+    // taken.
+
+    // Check if the Trap Flag (singlestep flag) is set. If so, sysretq will return to before the
+    // instruction, whereas debuggers expect the iretq behavior of returning to after the
+    // instruction.
+
+    "test BYTE PTR [rsp + 17], 1;",
+    // If set, return using IRETQ instead.
+    "jnz 2f;",
+
+    // Otherwise, continue with the fast sysretq.
+
+    // Pop userspace return pointer
+    "pop rcx;",
+
+    // We must ensure RCX is canonical; if it is not when running sysretq, the consequences can be
+    // fatal from a security perspective.
+    //
+    // See https://xenproject.org/2012/06/13/the-intel-sysret-privilege-escalation/.
+    //
+    // This is not just theoretical; ptrace allows userspace to change RCX (via RIP) of target
+    // processes.
+    //
+    // While we could also conditionally IRETQ here, an easier method is to simply sign-extend RCX:
+
+    // Shift away the upper 16 bits (0xBAAD_8000_DEAD_BEEF => 0x8000_DEAD_BEEF_XXXX).
+    "shl rcx, 16;",
+    // Shift arithmetically right by 16 bits, effectively extending the 47th sign bit to bits
+    // 63:48 (0x8000_DEAD_BEEF_XXXX => 0xFFFF_8000_DEAD_BEEF).
+    "sar rcx, 16;",
+
+    "add rsp, 8;",              // Pop fake userspace CS
+    "pop r11;",                 // Pop rflags
+    "pop rsp;",                 // Restore userspace stack pointer
+    "sysretq;",                 // Return into userspace; RCX=>RIP,R11=>RFLAGS
+
+    // IRETQ fallback:
+    "
+    .p2align 4
+2:
+    xor rcx, rcx
+    xor r11, r11
+    iretq
+    ",
+
+    sp = const(offset_of!(gdt::ProcessorControlRegion, user_rsp_tmp)),
+    ksp = const(offset_of!(gdt::ProcessorControlRegion, tss) + offset_of!(TaskStateSegment, rsp)),
+    ss_sel = const(SegmentSelector::new(gdt::GDT_USER_DATA as u16, x86::Ring::Ring3).bits()),
+    cs_sel = const(SegmentSelector::new(gdt::GDT_USER_CODE as u16, x86::Ring::Ring3).bits()),
+    );
+}
+unsafe extern "C" {
+    // TODO: macro?
+    pub fn enter_usermode();
+}

src/arch/x86_64/macros.rs

@@ -0,0 +1,80 @@
+macro_rules! expand_bool(
+    ($value:expr_2021) => {
+        concat!($value)
+    }
+);
+
+macro_rules! alternative(
+    (feature: $feature:literal, then: [$($then:expr_2021),*], default: [$($default:expr_2021),*]) => {
+        alternative2!(feature1: $feature, then1: [$($then),*], feature2: "", then2: [""], default: [$($default),*])
+    }
+);
+macro_rules! saturating_sub(
+    ($lhs:literal, $rhs:literal) => { concat!(
+        "((", $lhs, ")>(", $rhs, "))*((", $lhs, ")-(", $rhs, "))",
+    ) }
+);
+// Use feature1 if present, otherwise try using feature2, otherwise use default.
+//
+// cpu_feature_always simply means it is always enabled. Thus, if feature2, which has lower
+// priority, is "always" but feature1 is "auto", feature2 will still be checked for, and feature2
+// will become the fallback code.
+//
+// An empty string as feature is equivalent with "never".
+macro_rules! alternative2(
+    (feature1: $feature1:literal, then1: [$($then1:expr_2021),*], feature2: $feature2:literal, then2: [$($then2:expr_2021),*], default: [$($default:expr_2021),*]) => {
+        concat!("
+            .set true, 1
+            .set false, 0
+            40:
+            .if ", expand_bool!(cfg!(cpu_feature_always = $feature1)), "
+            ", $($then1,)* "
+            .elseif ", expand_bool!(cfg!(cpu_feature_always = $feature2)), "
+            ", $($then2,)* "
+            .else
+            ", $($default,)* "
+            .endif
+            42:
+            .if ", expand_bool!(cfg!(cpu_feature_auto = $feature1)), "
+            .skip -", saturating_sub!("51f - 50f", "42b - 40b"), ", 0x90
+            .endif
+            .if ", expand_bool!(cfg!(cpu_feature_auto = $feature2)), "
+            .skip -", saturating_sub!("61f - 60f", "42b - 40b"), ", 0x90
+            .endif
+            41:
+            ",
+            // FIXME: The assembler apparently complains "invalid number of bytes" despite it being
+            // quite obvious what saturating_sub does.
+
+            // Declare them in reverse order. Last relocation wins!
+            alternative_auto!("6", $feature2, [$($then2),*]),
+            alternative_auto!("5", $feature1, [$($then1),*]),
+        )
+    };
+);
+macro_rules! alternative_auto(
+    ($first_digit:literal, $feature:literal, [$($then:expr_2021),*]) => { concat!(
+        ".if ", expand_bool!(cfg!(cpu_feature_auto = $feature)), "
+        .pushsection .altcode.", $feature, ",\"a\"
+        ", $first_digit, "0:
+        ", $($then,)* "
+        ", $first_digit, "1:
+        .popsection
+        .pushsection .altfeatures.", $feature, ",\"a\"
+        70: .ascii \"", $feature, "\"
+        71:
+        .popsection
+        .pushsection .altrelocs.", $feature, ",\"a\"
+        .quad 70b
+        .quad 71b - 70b
+        .quad 40b
+        .quad 42b - 40b
+        .quad 41b - 40b
+        .quad 0
+        .quad ", $first_digit, "0b
+        .quad ", $first_digit, "1b - ", $first_digit, "0b
+        .popsection
+        .endif
+        ",
+    ) }
+);

src/arch/x86_64/misc.rs

@@ -0,0 +1,29 @@
+use x86::controlregs::Cr4;
+
+use crate::{
+    arch::cpuid::{cpuid, has_ext_feat},
+    cpu_set::LogicalCpuId,
+};
+
+pub unsafe fn init(cpu_id: LogicalCpuId) {
+    unsafe {
+        if has_ext_feat(|feat| feat.has_umip()) {
+            // UMIP (UserMode Instruction Prevention) forbids userspace from calling SGDT, SIDT, SLDT,
+            // SMSW and STR. KASLR is currently not implemented, but this protects against leaking
+            // addresses.
+            x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_UMIP);
+        }
+        if has_ext_feat(|feat| feat.has_smep()) {
+            // SMEP (Supervisor-Mode Execution Prevention) forbids the kernel from executing
+            // instruction on any page marked "userspace-accessible". This improves security for
+            // obvious reasons.
+            x86::controlregs::cr4_write(x86::controlregs::cr4() | Cr4::CR4_ENABLE_SMEP);
+        }
+
+        if let Some(feats) = cpuid().get_extended_processor_and_feature_identifiers()
+            && feats.has_rdtscp()
+        {
+            x86::msr::wrmsr(x86::msr::IA32_TSC_AUX, cpu_id.get().into());
+        }
+    }
+}

src/arch/x86_64/mod.rs

@@ -0,0 +1,50 @@
+pub use crate::arch::x86_shared::*;
+
+pub mod alternative;
+
+#[macro_use]
+pub mod macros;
+
+/// Constants like memory locations
+pub mod consts;
+
+/// Interrupt instructions
+#[macro_use]
+pub mod interrupt;
+
+/// Miscellaneous processor features
+pub mod misc;
+
+// TODO: Maybe support rewriting relocations (using LD's --emit-relocs) when working with entire
+// functions?
+#[unsafe(naked)]
+pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
+    // TODO: spectre_v1
+
+    core::arch::naked_asm!(
+        ".global __usercopy_start
+        __usercopy_start:",
+        alternative!(
+            feature: "smap",
+            then: ["
+            xor eax, eax
+            mov rcx, rdx
+            stac
+            rep movsb
+            clac
+            ret
+        "],
+            default: ["
+            xor eax, eax
+            mov rcx, rdx
+            rep movsb
+            ret
+        "]
+        ),
+        ".global __usercopy_end
+        __usercopy_end:"
+    );
+}
+pub use arch_copy_to_user as arch_copy_from_user;
+
+pub use alternative::kfx_size;

src/arch/x86_shared/cpuid.rs

@@ -0,0 +1,29 @@
+use raw_cpuid::{CpuId, CpuIdResult, ExtendedFeatures, FeatureInfo};
+
+pub fn cpuid() -> CpuId {
+    // FIXME check for cpuid availability during early boot and error out if it doesn't exist.
+    CpuId::with_cpuid_fn(|a, c| {
+        #[cfg(target_arch = "x86")]
+        let result = unsafe { core::arch::x86::__cpuid_count(a, c) };
+        #[cfg(target_arch = "x86_64")]
+        let result = unsafe { core::arch::x86_64::__cpuid_count(a, c) };
+        CpuIdResult {
+            eax: result.eax,
+            ebx: result.ebx,
+            ecx: result.ecx,
+            edx: result.edx,
+        }
+    })
+}
+
+#[cfg_attr(not(target_arch = "x86_64"), expect(dead_code))]
+pub fn feature_info() -> FeatureInfo {
+    cpuid()
+        .get_feature_info()
+        .expect("x86_64 requires CPUID leaf=0x01 to be present")
+}
+
+#[cfg_attr(not(target_arch = "x86_64"), expect(dead_code))]
+pub fn has_ext_feat(feat: impl FnOnce(ExtendedFeatures) -> bool) -> bool {
+    cpuid().get_extended_feature_info().is_some_and(feat)
+}

src/arch/x86_shared/debug.rs

@@ -0,0 +1,62 @@
+#[cfg(feature = "qemu_debug")]
+use spin::Mutex;
+use spin::MutexGuard;
+
+use crate::devices::serial::SerialKind;
+#[cfg(feature = "lpss_debug")]
+use crate::devices::uart_16550::SerialPort;
+#[cfg(feature = "lpss_debug")]
+use crate::syscall::io::Mmio;
+#[cfg(feature = "qemu_debug")]
+use crate::syscall::io::Pio;
+#[cfg(feature = "qemu_debug")]
+use syscall::io::Io;
+
+use super::device::serial::{COM1, LPSS};
+#[cfg(feature = "system76_ec_debug")]
+use super::device::system76_ec::{System76Ec, SYSTEM76_EC};
+
+#[cfg(feature = "qemu_debug")]
+pub static QEMU: Mutex<Pio<u8>> = Mutex::new(Pio::<u8>::new(0x402));
+
+pub struct Writer<'a> {
+    lpss: MutexGuard<'a, SerialKind>,
+    #[cfg(feature = "qemu_debug")]
+    qemu: MutexGuard<'a, Pio<u8>>,
+    serial: MutexGuard<'a, SerialKind>,
+    #[cfg(feature = "system76_ec_debug")]
+    system76_ec: MutexGuard<'a, Option<System76Ec>>,
+}
+
+impl<'a> Writer<'a> {
+    pub fn new() -> Writer<'a> {
+        Writer {
+            lpss: LPSS.lock(),
+            #[cfg(feature = "qemu_debug")]
+            qemu: QEMU.lock(),
+            serial: COM1.lock(),
+            #[cfg(feature = "system76_ec_debug")]
+            system76_ec: SYSTEM76_EC.lock(),
+        }
+    }
+
+    pub fn write(&mut self, buf: &[u8]) {
+        self.lpss.write(buf);
+
+        #[cfg(feature = "qemu_debug")]
+        {
+            for &b in buf {
+                self.qemu.write(b);
+            }
+        }
+
+        self.serial.write(buf);
+
+        #[cfg(feature = "system76_ec_debug")]
+        {
+            if let Some(ref mut system76_ec) = *self.system76_ec {
+                system76_ec.print_slice(buf);
+            }
+        }
+    }
+}

src/arch/x86_shared/device/cpu.rs

@@ -0,0 +1,281 @@
+use core::fmt::{Result, Write};
+
+use crate::arch::cpuid::cpuid;
+
+pub fn cpu_info<W: Write>(w: &mut W) -> Result {
+    let cpuid = cpuid();
+
+    if let Some(info) = cpuid.get_vendor_info() {
+        writeln!(w, "Vendor: {}", info.as_str())?;
+    }
+
+    if let Some(brand) = cpuid.get_processor_brand_string() {
+        writeln!(w, "Model: {}", brand.as_str())?;
+    }
+
+    if let Some(info) = cpuid.get_processor_frequency_info() {
+        writeln!(w, "CPU Base MHz: {}", info.processor_base_frequency())?;
+        writeln!(w, "CPU Max MHz: {}", info.processor_max_frequency())?;
+        writeln!(w, "Bus MHz: {}", info.bus_frequency())?;
+    }
+
+    write!(w, "Features:")?;
+
+    if let Some(info) = cpuid.get_feature_info() {
+        if info.has_fpu() {
+            write!(w, " fpu")?
+        };
+        if info.has_vme() {
+            write!(w, " vme")?
+        };
+        if info.has_de() {
+            write!(w, " de")?
+        };
+        if info.has_pse() {
+            write!(w, " pse")?
+        };
+        if info.has_tsc() {
+            write!(w, " tsc")?
+        };
+        if info.has_msr() {
+            write!(w, " msr")?
+        };
+        if info.has_pae() {
+            write!(w, " pae")?
+        };
+        if info.has_mce() {
+            write!(w, " mce")?
+        };
+
+        if info.has_cmpxchg8b() {
+            write!(w, " cx8")?
+        };
+        if info.has_apic() {
+            write!(w, " apic")?
+        };
+        if info.has_sysenter_sysexit() {
+            write!(w, " sep")?
+        };
+        if info.has_mtrr() {
+            write!(w, " mtrr")?
+        };
+        if info.has_pge() {
+            write!(w, " pge")?
+        };
+        if info.has_mca() {
+            write!(w, " mca")?
+        };
+        if info.has_cmov() {
+            write!(w, " cmov")?
+        };
+        if info.has_pat() {
+            write!(w, " pat")?
+        };
+
+        if info.has_pse36() {
+            write!(w, " pse36")?
+        };
+        if info.has_psn() {
+            write!(w, " psn")?
+        };
+        if info.has_clflush() {
+            write!(w, " clflush")?
+        };
+        if info.has_ds() {
+            write!(w, " ds")?
+        };
+        if info.has_acpi() {
+            write!(w, " acpi")?
+        };
+        if info.has_mmx() {
+            write!(w, " mmx")?
+        };
+        if info.has_fxsave_fxstor() {
+            write!(w, " fxsr")?
+        };
+        if info.has_sse() {
+            write!(w, " sse")?
+        };
+
+        if info.has_sse2() {
+            write!(w, " sse2")?
+        };
+        if info.has_ss() {
+            write!(w, " ss")?
+        };
+        if info.has_htt() {
+            write!(w, " ht")?
+        };
+        if info.has_tm() {
+            write!(w, " tm")?
+        };
+        if info.has_pbe() {
+            write!(w, " pbe")?
+        };
+
+        if info.has_sse3() {
+            write!(w, " sse3")?
+        };
+        if info.has_pclmulqdq() {
+            write!(w, " pclmulqdq")?
+        };
+        if info.has_ds_area() {
+            write!(w, " dtes64")?
+        };
+        if info.has_monitor_mwait() {
+            write!(w, " monitor")?
+        };
+        if info.has_cpl() {
+            write!(w, " ds_cpl")?
+        };
+        if info.has_vmx() {
+            write!(w, " vmx")?
+        };
+        if info.has_smx() {
+            write!(w, " smx")?
+        };
+        if info.has_eist() {
+            write!(w, " est")?
+        };
+
+        if info.has_tm2() {
+            write!(w, " tm2")?
+        };
+        if info.has_ssse3() {
+            write!(w, " ssse3")?
+        };
+        if info.has_cnxtid() {
+            write!(w, " cnxtid")?
+        };
+        if info.has_fma() {
+            write!(w, " fma")?
+        };
+        if info.has_cmpxchg16b() {
+            write!(w, " cx16")?
+        };
+        if info.has_pdcm() {
+            write!(w, " pdcm")?
+        };
+        if info.has_pcid() {
+            write!(w, " pcid")?
+        };
+        if info.has_dca() {
+            write!(w, " dca")?
+        };
+
+        if info.has_sse41() {
+            write!(w, " sse4_1")?
+        };
+        if info.has_sse42() {
+            write!(w, " sse4_2")?
+        };
+        if info.has_x2apic() {
+            write!(w, " x2apic")?
+        };
+        if info.has_movbe() {
+            write!(w, " movbe")?
+        };
+        if info.has_popcnt() {
+            write!(w, " popcnt")?
+        };
+        if info.has_tsc_deadline() {
+            write!(w, " tsc_deadline_timer")?
+        };
+        if info.has_aesni() {
+            write!(w, " aes")?
+        };
+        if info.has_xsave() {
+            write!(w, " xsave")?
+        };
+
+        if info.has_oxsave() {
+            write!(w, " xsaveopt")?
+        };
+        if info.has_avx() {
+            write!(w, " avx")?
+        };
+        if info.has_f16c() {
+            write!(w, " f16c")?
+        };
+        if info.has_rdrand() {
+            write!(w, " rdrand")?
+        };
+    }
+
+    if let Some(info) = cpuid.get_extended_processor_and_feature_identifiers() {
+        if info.has_64bit_mode() {
+            write!(w, " lm")?
+        };
+        if info.has_rdtscp() {
+            write!(w, " rdtscp")?
+        };
+        if info.has_1gib_pages() {
+            write!(w, " pdpe1gb")?
+        };
+        if info.has_execute_disable() {
+            write!(w, " nx")?
+        };
+        if info.has_syscall_sysret() {
+            write!(w, " syscall")?
+        };
+        if info.has_prefetchw() {
+            write!(w, " prefetchw")?
+        };
+        if info.has_lzcnt() {
+            write!(w, " lzcnt")?
+        };
+        if info.has_lahf_sahf() {
+            write!(w, " lahf_lm")?
+        };
+    }
+
+    if let Some(info) = cpuid.get_advanced_power_mgmt_info()
+        && info.has_invariant_tsc()
+    {
+        write!(w, " constant_tsc")?
+    };
+
+    if let Some(info) = cpuid.get_extended_feature_info() {
+        if info.has_fsgsbase() {
+            write!(w, " fsgsbase")?
+        };
+        if info.has_tsc_adjust_msr() {
+            write!(w, " tsc_adjust")?
+        };
+        if info.has_bmi1() {
+            write!(w, " bmi1")?
+        };
+        if info.has_hle() {
+            write!(w, " hle")?
+        };
+        if info.has_avx2() {
+            write!(w, " avx2")?
+        };
+        if info.has_smep() {
+            write!(w, " smep")?
+        };
+        if info.has_bmi2() {
+            write!(w, " bmi2")?
+        };
+        if info.has_rep_movsb_stosb() {
+            write!(w, " erms")?
+        };
+        if info.has_invpcid() {
+            write!(w, " invpcid")?
+        };
+        if info.has_rtm() {
+            write!(w, " rtm")?
+        };
+        //if info.has_qm() { write!(w, " qm")? };
+        if info.has_fpu_cs_ds_deprecated() {
+            write!(w, " fpu_seg")?
+        };
+        if info.has_mpx() {
+            write!(w, " mpx")?
+        };
+    }
+
+    writeln!(w)?;
+
+    Ok(())
+}

src/arch/x86_shared/device/hpet.rs

@@ -0,0 +1,125 @@
+//! <https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/software-developers-hpet-spec-1-0a.pdf>
+
+use super::pit;
+use crate::acpi::hpet::Hpet;
+use core::time::Duration;
+
+const LEG_RT_CNF: u64 = 2;
+const ENABLE_CNF: u64 = 1;
+
+const TN_VAL_SET_CNF: u64 = 0x40;
+const TN_TYPE_CNF: u64 = 0x08;
+const TN_INT_ENB_CNF: u64 = 0x04;
+
+pub(crate) const CAPABILITY_OFFSET: usize = 0x00;
+const GENERAL_CONFIG_OFFSET: usize = 0x10;
+const GENERAL_INTERRUPT_OFFSET: usize = 0x20;
+pub(crate) const MAIN_COUNTER_OFFSET: usize = 0xF0;
+// const NUM_TIMER_CAP_MASK: u64 = 0x0f00;
+const LEG_RT_CAP: u64 = 0x8000;
+const T0_CONFIG_CAPABILITY_OFFSET: usize = 0x100;
+pub(crate) const T0_COMPARATOR_OFFSET: usize = 0x108;
+
+const PER_INT_CAP: u64 = 0x10;
+
+pub unsafe fn init(hpet: &mut Hpet) -> bool {
+    unsafe {
+        debug!("HPET @ {:#x}", { hpet.base_address.address });
+        debug_caps(hpet);
+
+        trace!("HPET Before Init");
+        debug_config(hpet);
+
+        // Disable HPET
+        {
+            let mut config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET);
+            config_word &= !(LEG_RT_CNF | ENABLE_CNF);
+            hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word);
+        }
+
+        let capability = hpet.read_u64(CAPABILITY_OFFSET);
+        if capability & LEG_RT_CAP == 0 {
+            warn!("HPET missing capability LEG_RT_CAP");
+            return false;
+        }
+
+        let period_fs = capability >> 32;
+        let divisor = (pit::RATE as u64 * 1_000_000) / period_fs;
+
+        let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
+        if t0_capabilities & PER_INT_CAP == 0 {
+            warn!("HPET T0 missing capability PER_INT_CAP");
+            return false;
+        }
+
+        let counter = hpet.read_u64(MAIN_COUNTER_OFFSET);
+
+        let t0_config_word: u64 = TN_VAL_SET_CNF | TN_TYPE_CNF | TN_INT_ENB_CNF;
+        hpet.write_u64(T0_CONFIG_CAPABILITY_OFFSET, t0_config_word);
+        // set accumulator value
+        hpet.write_u64(T0_COMPARATOR_OFFSET, counter + divisor);
+        // set interval
+        hpet.write_u64(T0_COMPARATOR_OFFSET, divisor);
+
+        // Enable interrupts from the HPET
+        {
+            let mut config_word: u64 = hpet.read_u64(GENERAL_CONFIG_OFFSET);
+            config_word |= LEG_RT_CNF | ENABLE_CNF;
+            hpet.write_u64(GENERAL_CONFIG_OFFSET, config_word);
+        }
+
+        trace!("HPET After Init");
+        debug_config(hpet);
+
+        true
+    }
+}
+
+unsafe fn debug_caps(hpet: &mut Hpet) {
+    unsafe {
+        let capability = hpet.read_u64(CAPABILITY_OFFSET);
+        trace!("  caps: {:#x}", capability);
+        trace!(
+            "    clock period: {:?}",
+            Duration::from_nanos((capability >> 32) / 1_000_000)
+        );
+        trace!(
+            "    ID: {:#x} revision: {}",
+            (capability >> 16) as u16,
+            capability as u8
+        );
+        trace!(
+            "    LEG_RT_CAP: {} COUNT_SIZE_CAP: {}",
+            capability & (1 << 15) == (1 << 15),
+            capability & (1 << 13) == (1 << 13)
+        );
+        // The NUM_TIM_CAP field contains the index of the last timer.
+        // Add 1 to get the amount of timers.
+        trace!("    timers: {}", ((capability >> 8) as u8 & 0x1F) + 1);
+
+        let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
+        trace!(
+            "  T0 interrupt routing: {:#x}",
+            (t0_capabilities >> 32) as u32
+        );
+    }
+}
+
+unsafe fn debug_config(hpet: &mut Hpet) {
+    unsafe {
+        let config_word = hpet.read_u64(GENERAL_CONFIG_OFFSET);
+        trace!("  config: {:#x}", config_word);
+
+        let interrupt_status = hpet.read_u64(GENERAL_INTERRUPT_OFFSET);
+        trace!("  interrupt status: {:#x}", interrupt_status);
+
+        let counter = hpet.read_u64(MAIN_COUNTER_OFFSET);
+        trace!("  counter: {:#x}", counter);
+
+        let t0_capabilities = hpet.read_u64(T0_CONFIG_CAPABILITY_OFFSET);
+        trace!("  T0 flags: {:#x}", t0_capabilities as u32);
+
+        let t0_comparator = hpet.read_u64(T0_COMPARATOR_OFFSET);
+        trace!("  T0 comparator: {:#x}", t0_comparator);
+    }
+}

src/arch/x86_shared/device/ioapic.rs

@@ -0,0 +1,427 @@
+use core::{cell::SyncUnsafeCell, fmt, ptr};
+
+use alloc::vec::Vec;
+use spin::Mutex;
+
+use super::{local_apic::ApicId, pic};
+use crate::{
+    acpi::madt::{self, Madt, MadtEntry, MadtIntSrcOverride, MadtIoApic},
+    arch::{cpuid::cpuid, interrupt::irq},
+    memory::{map_device_memory, PhysicalAddress},
+};
+
+pub struct IoApicRegs {
+    pointer: *const u32,
+}
+impl IoApicRegs {
+    fn ioregsel(&self) -> *const u32 {
+        self.pointer
+    }
+    fn iowin(&self) -> *const u32 {
+        // offset 0x10
+        unsafe { self.pointer.offset(4) }
+    }
+    fn write_ioregsel(&mut self, value: u32) {
+        unsafe { ptr::write_volatile::<u32>(self.ioregsel().cast_mut(), value) }
+    }
+    fn read_iowin(&self) -> u32 {
+        unsafe { ptr::read_volatile::<u32>(self.iowin()) }
+    }
+    fn write_iowin(&mut self, value: u32) {
+        unsafe { ptr::write_volatile::<u32>(self.iowin().cast_mut(), value) }
+    }
+    fn read_reg(&mut self, reg: u8) -> u32 {
+        self.write_ioregsel(reg.into());
+        self.read_iowin()
+    }
+    fn write_reg(&mut self, reg: u8, value: u32) {
+        self.write_ioregsel(reg.into());
+        self.write_iowin(value);
+    }
+    pub fn read_ioapicid(&mut self) -> u32 {
+        self.read_reg(0x00)
+    }
+    pub fn read_ioapicver(&mut self) -> u32 {
+        self.read_reg(0x01)
+    }
+    pub fn read_ioredtbl(&mut self, idx: u8) -> u64 {
+        assert!(idx < 24);
+        let lo = self.read_reg(0x10 + idx * 2);
+        let hi = self.read_reg(0x10 + idx * 2 + 1);
+
+        u64::from(lo) | (u64::from(hi) << 32)
+    }
+    pub fn write_ioredtbl(&mut self, idx: u8, value: u64) {
+        assert!(idx < 24);
+
+        let lo = value as u32;
+        let hi = (value >> 32) as u32;
+
+        self.write_reg(0x10 + idx * 2, lo);
+        self.write_reg(0x10 + idx * 2 + 1, hi);
+    }
+
+    pub fn max_redirection_table_entries(&mut self) -> u8 {
+        let ver = self.read_ioapicver();
+        ((ver & 0x00FF_0000) >> 16) as u8
+    }
+    #[allow(dead_code)]
+    pub fn id(&mut self) -> u8 {
+        let id_reg = self.read_ioapicid();
+        ((id_reg & 0x0F00_0000) >> 24) as u8
+    }
+}
+pub struct IoApic {
+    regs: Mutex<IoApicRegs>,
+    gsi_start: u32,
+    count: u8,
+}
+unsafe impl Send for IoApic {}
+unsafe impl Sync for IoApic {}
+impl IoApic {
+    #[allow(dead_code)]
+    pub fn new(regs_base: *const u32, gsi_start: u32) -> Self {
+        let mut regs = IoApicRegs { pointer: regs_base };
+        let count = regs.max_redirection_table_entries();
+
+        Self {
+            regs: Mutex::new(regs),
+            gsi_start,
+            count,
+        }
+    }
+    /// Map an interrupt vector to a physical local APIC ID of a processor (thus physical mode).
+    #[allow(dead_code)]
+    pub fn map(&self, idx: u8, info: MapInfo) {
+        self.regs.lock().write_ioredtbl(idx, info.as_raw())
+    }
+    pub fn set_mask(&self, gsi: u32, mask: bool) {
+        let idx = (gsi - self.gsi_start) as u8;
+        let mut guard = self.regs.lock();
+
+        let mut reg = guard.read_ioredtbl(idx);
+        reg &= !(1 << 16);
+        reg |= u64::from(mask) << 16;
+        guard.write_ioredtbl(idx, reg);
+    }
+}
+
+#[repr(u8)]
+#[derive(Clone, Copy, Debug)]
+pub enum ApicTriggerMode {
+    Edge = 0,
+    Level = 1,
+}
+#[repr(u8)]
+#[derive(Clone, Copy, Debug)]
+pub enum ApicPolarity {
+    ActiveHigh = 0,
+    ActiveLow = 1,
+}
+#[repr(u8)]
+#[derive(Clone, Copy, Debug)]
+#[allow(unused)]
+pub enum DestinationMode {
+    Physical = 0,
+    Logical = 1,
+}
+#[repr(u8)]
+#[derive(Clone, Copy, Debug)]
+#[allow(unused)]
+pub enum DeliveryMode {
+    Fixed = 0b000,
+    LowestPriority = 0b001,
+    Smi = 0b010,
+    Nmi = 0b100,
+    Init = 0b101,
+    ExtInt = 0b111,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct MapInfo {
+    pub dest: ApicId,
+    pub mask: bool,
+    pub trigger_mode: ApicTriggerMode,
+    pub polarity: ApicPolarity,
+    pub dest_mode: DestinationMode,
+    pub delivery_mode: DeliveryMode,
+    pub vector: u8,
+}
+
+impl MapInfo {
+    pub fn as_raw(&self) -> u64 {
+        assert!(self.vector >= 0x20);
+        assert!(self.vector <= 0xFE);
+
+        // TODO: Check for reserved fields.
+
+        (u64::from(self.dest.get()) << 56)
+            | (u64::from(self.mask) << 16)
+            | ((self.trigger_mode as u64) << 15)
+            | ((self.polarity as u64) << 13)
+            | ((self.dest_mode as u64) << 11)
+            | ((self.delivery_mode as u64) << 8)
+            | u64::from(self.vector)
+    }
+}
+
+impl fmt::Debug for IoApic {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        struct RedirTable<'a>(&'a Mutex<IoApicRegs>);
+
+        impl fmt::Debug for RedirTable<'_> {
+            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                let mut guard = self.0.lock();
+
+                let count = guard.max_redirection_table_entries();
+                f.debug_list()
+                    .entries((0..count).map(|i| guard.read_ioredtbl(i)))
+                    .finish()
+            }
+        }
+
+        f.debug_struct("IoApic")
+            .field("redir_table", &RedirTable(&self.regs))
+            .field("gsi_start", &self.gsi_start)
+            .field("count", &self.count)
+            .finish()
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum TriggerMode {
+    ConformsToSpecs,
+    Edge,
+    Level,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum Polarity {
+    ConformsToSpecs,
+    ActiveHigh,
+    ActiveLow,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct Override {
+    bus_irq: u8,
+    gsi: u32,
+
+    trigger_mode: TriggerMode,
+    polarity: Polarity,
+}
+
+// static mut because only the AP initializes the I/O Apic, and when that is done, it's solely
+// accessed immutably.
+static IOAPICS: SyncUnsafeCell<Option<Vec<IoApic>>> = SyncUnsafeCell::new(None);
+
+// static mut for the same reason as above
+static SRC_OVERRIDES: SyncUnsafeCell<Option<Vec<Override>>> = SyncUnsafeCell::new(None);
+
+pub fn ioapics() -> &'static [IoApic] {
+    unsafe { &*IOAPICS.get() }
+        .as_ref()
+        .map_or(&[], |vector| &vector[..])
+}
+pub fn src_overrides() -> &'static [Override] {
+    unsafe { &*SRC_OVERRIDES.get() }
+        .as_ref()
+        .map_or(&[], |vector| &vector[..])
+}
+
+pub unsafe fn handle_ioapic(madt_ioapic: &'static MadtIoApic) {
+    unsafe {
+        // map the I/O APIC registers
+        let virt = map_device_memory(PhysicalAddress::new(madt_ioapic.address as usize), 4096);
+
+        let ioapic_registers = virt.data() as *const u32;
+        let ioapic = IoApic::new(ioapic_registers, madt_ioapic.gsi_base);
+
+        assert_eq!(
+            ioapic.regs.lock().id(),
+            madt_ioapic.id,
+            "mismatched ACPI MADT I/O APIC ID, and the ID reported by the I/O APIC"
+        );
+
+        (*IOAPICS.get()).get_or_insert_with(Vec::new).push(ioapic);
+    }
+}
+pub unsafe fn handle_src_override(src_override: &'static MadtIntSrcOverride) {
+    unsafe {
+        let flags = src_override.flags;
+
+        let polarity_raw = (flags & 0x0003) as u8;
+        let trigger_mode_raw = ((flags & 0x000C) >> 2) as u8;
+
+        let polarity = match polarity_raw {
+            0b00 => Polarity::ConformsToSpecs,
+            0b01 => Polarity::ActiveHigh,
+            0b10 => return, // reserved
+            0b11 => Polarity::ActiveLow,
+
+            _ => unreachable!(),
+        };
+
+        let trigger_mode = match trigger_mode_raw {
+            0b00 => TriggerMode::ConformsToSpecs,
+            0b01 => TriggerMode::Edge,
+            0b10 => return, // reserved
+            0b11 => TriggerMode::Level,
+            _ => unreachable!(),
+        };
+
+        let over = Override {
+            bus_irq: src_override.irq_source,
+            gsi: src_override.gsi_base,
+            polarity,
+            trigger_mode,
+        };
+        (*SRC_OVERRIDES.get())
+            .get_or_insert_with(Vec::new)
+            .push(over);
+    }
+}
+
+#[allow(dead_code)]
+pub unsafe fn init() {
+    unsafe {
+        let bsp_apic_id = ApicId::new(u32::from(
+            cpuid().get_feature_info().unwrap().initial_local_apic_id(),
+        )); // TODO: remove unwraps
+
+        // search the madt for all IOAPICs.
+        if cfg!(feature = "acpi") {
+            let madt: &'static Madt = match madt::madt() {
+                Some(m) => m,
+                // TODO: Parse MP tables too.
+                None => return,
+            };
+            if madt.flags & madt::FLAG_PCAT != 0 {
+                pic::disable();
+            }
+
+            // find all I/O APICs (usually one).
+
+            for entry in madt.iter() {
+                match entry {
+                    MadtEntry::IoApic(ioapic) => handle_ioapic(ioapic),
+                    MadtEntry::IntSrcOverride(src_override) => handle_src_override(src_override),
+                    _ => (),
+                }
+            }
+        }
+        println!(
+            "I/O APICs: {:?}, overrides: {:?}",
+            ioapics(),
+            src_overrides()
+        );
+
+        // map the legacy PC-compatible IRQs (0-15) to 32-47, just like we did with 8259 PIC (if it
+        // wouldn't have been disabled due to this I/O APIC)
+        for legacy_irq in 0..=15 {
+            let (gsi, trigger_mode, polarity) = match get_override(legacy_irq) {
+                Some(over) => (over.gsi, over.trigger_mode, over.polarity),
+                None => {
+                    if src_overrides()
+                        .iter()
+                        .any(|over| over.gsi == u32::from(legacy_irq) && over.bus_irq != legacy_irq)
+                        && !src_overrides()
+                            .iter()
+                            .any(|over| over.bus_irq == legacy_irq)
+                    {
+                        // there's an IRQ conflict, making this legacy IRQ inaccessible.
+                        continue;
+                    }
+                    (
+                        legacy_irq.into(),
+                        TriggerMode::ConformsToSpecs,
+                        Polarity::ConformsToSpecs,
+                    )
+                }
+            };
+            let apic = match find_ioapic(gsi) {
+                Some(ioapic) => ioapic,
+                None => {
+                    println!("Unable to find a suitable APIC for legacy IRQ {} (GSI {}). It will not be mapped.", legacy_irq, gsi);
+                    continue;
+                }
+            };
+            let redir_tbl_index = (gsi - apic.gsi_start) as u8;
+
+            let map_info = MapInfo {
+                // only send to the BSP
+                dest: bsp_apic_id,
+                dest_mode: DestinationMode::Physical,
+                delivery_mode: DeliveryMode::Fixed,
+                mask: false,
+                polarity: match polarity {
+                    Polarity::ActiveHigh => ApicPolarity::ActiveHigh,
+                    Polarity::ActiveLow => ApicPolarity::ActiveLow,
+                    Polarity::ConformsToSpecs => ApicPolarity::ActiveHigh,
+                },
+                trigger_mode: match trigger_mode {
+                    TriggerMode::Edge => ApicTriggerMode::Edge,
+                    TriggerMode::Level => ApicTriggerMode::Level,
+                    TriggerMode::ConformsToSpecs => ApicTriggerMode::Edge,
+                },
+                vector: 32 + legacy_irq,
+            };
+            apic.map(redir_tbl_index, map_info);
+        }
+        println!(
+            "I/O APICs: {:?}, overrides: {:?}",
+            ioapics(),
+            src_overrides()
+        );
+        irq::set_irq_method(irq::IrqMethod::Apic);
+
+        // tell the firmware that we're using APIC rather than the default 8259 PIC.
+
+        // FIXME: With ACPI moved to userspace, we should instead allow userspace to check whether the
+        // IOAPIC has been initialized, and then subsequently let some ACPI driver call the AML from
+        // userspace.
+
+        /*#[cfg(feature = "acpi")]
+        {
+            let method = {
+                let namespace_guard = crate::acpi::ACPI_TABLE.namespace.read();
+                if let Some(value) = namespace_guard.as_ref().unwrap().get("\\_PIC") {
+                    value.get_as_method().ok()
+                } else {
+                    None
+                }
+            };
+            if let Some(m) = method {
+                m.execute("\\_PIC".into(), vec!(crate::acpi::aml::AmlValue::Integer(1)));
+            }
+        }*/
+    }
+}
+fn get_override(irq: u8) -> Option<&'static Override> {
+    src_overrides().iter().find(|over| over.bus_irq == irq)
+}
+fn resolve(irq: u8) -> u32 {
+    get_override(irq).map_or(u32::from(irq), |over| over.gsi)
+}
+fn find_ioapic(gsi: u32) -> Option<&'static IoApic> {
+    ioapics()
+        .iter()
+        .find(|apic| gsi >= apic.gsi_start && gsi < apic.gsi_start + u32::from(apic.count))
+}
+
+pub unsafe fn mask(irq: u8) {
+    let gsi = resolve(irq);
+    let apic = match find_ioapic(gsi) {
+        Some(a) => a,
+        None => return,
+    };
+    apic.set_mask(gsi, true);
+}
+pub unsafe fn unmask(irq: u8) {
+    let gsi = resolve(irq);
+    let apic = match find_ioapic(gsi) {
+        Some(a) => a,
+        None => return,
+    };
+    apic.set_mask(gsi, false);
+}

src/arch/x86_shared/device/local_apic.rs

@@ -0,0 +1,272 @@
+use core::{
+    cell::SyncUnsafeCell,
+    ptr::{read_volatile, write_volatile},
+};
+use x86::msr::*;
+
+use crate::{
+    arch::{cpuid::cpuid, ipi::IpiKind},
+    memory::{map_device_memory, PhysicalAddress},
+    percpu::PercpuBlock,
+};
+
+#[derive(Clone, Copy, Debug)]
+pub struct ApicId(u32);
+
+impl ApicId {
+    pub fn new(inner: u32) -> Self {
+        Self(inner)
+    }
+
+    pub fn get(&self) -> u32 {
+        self.0
+    }
+}
+
+static LOCAL_APIC: SyncUnsafeCell<LocalApic> = SyncUnsafeCell::new(LocalApic {
+    address: 0,
+    x2: false,
+});
+pub unsafe fn the_local_apic() -> &'static mut LocalApic {
+    unsafe { &mut *LOCAL_APIC.get() }
+}
+
+pub unsafe fn init() {
+    unsafe {
+        the_local_apic().init();
+    }
+}
+
+pub unsafe fn init_ap() {
+    unsafe {
+        the_local_apic().init_ap();
+    }
+}
+
+/// Local APIC
+pub struct LocalApic {
+    pub address: usize,
+    pub x2: bool,
+}
+
+impl LocalApic {
+    unsafe fn init(&mut self) {
+        unsafe {
+            let physaddr = PhysicalAddress::new(rdmsr(IA32_APIC_BASE) as usize & 0xFFFF_0000);
+
+            self.x2 = cpuid()
+                .get_feature_info()
+                .is_some_and(|feature_info| feature_info.has_x2apic());
+
+            if !self.x2 {
+                debug!("Detected xAPIC at {:#x}", physaddr.data());
+                self.address = map_device_memory(physaddr, 4096).data();
+            } else {
+                debug!("Detected x2APIC");
+            }
+
+            self.init_ap();
+        }
+    }
+
+    unsafe fn init_ap(&mut self) {
+        unsafe {
+            if self.x2 {
+                wrmsr(IA32_APIC_BASE, rdmsr(IA32_APIC_BASE) | (1 << 10));
+                wrmsr(IA32_X2APIC_SIVR, 0x100);
+            } else {
+                self.write(0xF0, 0x100);
+            }
+            self.setup_error_int();
+            //self.setup_timer();
+
+            PercpuBlock::current()
+                .misc_arch_info
+                .apic_id_opt
+                .set(Some(self.id()));
+        }
+    }
+
+    unsafe fn read(&self, reg: u32) -> u32 {
+        debug_assert!(!self.x2);
+        unsafe { read_volatile((self.address + reg as usize) as *const u32) }
+    }
+
+    unsafe fn write(&mut self, reg: u32, value: u32) {
+        debug_assert!(!self.x2);
+        unsafe {
+            write_volatile((self.address + reg as usize) as *mut u32, value);
+        }
+    }
+
+    pub fn id(&self) -> ApicId {
+        ApicId::new(if self.x2 {
+            unsafe { rdmsr(IA32_X2APIC_APICID) as u32 }
+        } else {
+            unsafe { self.read(0x20) }
+        })
+    }
+
+    pub fn version(&self) -> u32 {
+        if self.x2 {
+            unsafe { rdmsr(IA32_X2APIC_VERSION) as u32 }
+        } else {
+            unsafe { self.read(0x30) }
+        }
+    }
+
+    pub fn icr(&self) -> u64 {
+        if self.x2 {
+            unsafe { rdmsr(IA32_X2APIC_ICR) }
+        } else {
+            unsafe { ((self.read(0x310) as u64) << 32) | self.read(0x300) as u64 }
+        }
+    }
+
+    pub fn set_icr(&mut self, value: u64) {
+        if self.x2 {
+            unsafe {
+                wrmsr(IA32_X2APIC_ICR, value);
+            }
+        } else {
+            unsafe {
+                const PENDING: u32 = 1 << 12;
+                while self.read(0x300) & PENDING == PENDING {
+                    core::hint::spin_loop();
+                }
+                self.write(0x310, (value >> 32) as u32);
+                self.write(0x300, value as u32);
+                while self.read(0x300) & PENDING == PENDING {
+                    core::hint::spin_loop();
+                }
+            }
+        }
+    }
+
+    pub fn ipi(&mut self, apic_id: ApicId, kind: IpiKind) {
+        let shift = if self.x2 { 32 } else { 56 };
+        self.set_icr((u64::from(apic_id.get()) << shift) | 0x40 | kind as u64);
+    }
+    pub fn ipi_nmi(&mut self, apic_id: ApicId) {
+        let shift = if self.x2 { 32 } else { 56 };
+        self.set_icr((u64::from(apic_id.get()) << shift) | (1 << 14) | (0b100 << 8));
+    }
+
+    pub unsafe fn eoi(&mut self) {
+        unsafe {
+            if self.x2 {
+                wrmsr(IA32_X2APIC_EOI, 0);
+            } else {
+                self.write(0xB0, 0);
+            }
+        }
+    }
+    /// Reads the Error Status Register.
+    pub unsafe fn esr(&mut self) -> u32 {
+        unsafe {
+            if self.x2 {
+                // update the ESR to the current state of the local apic.
+                wrmsr(IA32_X2APIC_ESR, 0);
+                // read the updated value
+                rdmsr(IA32_X2APIC_ESR) as u32
+            } else {
+                self.write(0x280, 0);
+                self.read(0x280)
+            }
+        }
+    }
+    pub unsafe fn lvt_timer(&mut self) -> u32 {
+        unsafe {
+            if self.x2 {
+                rdmsr(IA32_X2APIC_LVT_TIMER) as u32
+            } else {
+                self.read(0x320)
+            }
+        }
+    }
+    pub unsafe fn set_lvt_timer(&mut self, value: u32) {
+        unsafe {
+            if self.x2 {
+                wrmsr(IA32_X2APIC_LVT_TIMER, u64::from(value));
+            } else {
+                self.write(0x320, value);
+            }
+        }
+    }
+    pub unsafe fn init_count(&mut self) -> u32 {
+        unsafe {
+            if self.x2 {
+                rdmsr(IA32_X2APIC_INIT_COUNT) as u32
+            } else {
+                self.read(0x380)
+            }
+        }
+    }
+    pub unsafe fn set_init_count(&mut self, initial_count: u32) {
+        unsafe {
+            if self.x2 {
+                wrmsr(IA32_X2APIC_INIT_COUNT, u64::from(initial_count));
+            } else {
+                self.write(0x380, initial_count);
+            }
+        }
+    }
+    pub unsafe fn cur_count(&mut self) -> u32 {
+        unsafe {
+            if self.x2 {
+                rdmsr(IA32_X2APIC_CUR_COUNT) as u32
+            } else {
+                self.read(0x390)
+            }
+        }
+    }
+    pub unsafe fn div_conf(&mut self) -> u32 {
+        unsafe {
+            if self.x2 {
+                rdmsr(IA32_X2APIC_DIV_CONF) as u32
+            } else {
+                self.read(0x3E0)
+            }
+        }
+    }
+    pub unsafe fn set_div_conf(&mut self, div_conf: u32) {
+        unsafe {
+            if self.x2 {
+                wrmsr(IA32_X2APIC_DIV_CONF, u64::from(div_conf));
+            } else {
+                self.write(0x3E0, div_conf);
+            }
+        }
+    }
+    pub unsafe fn lvt_error(&mut self) -> u32 {
+        unsafe {
+            if self.x2 {
+                rdmsr(IA32_X2APIC_LVT_ERROR) as u32
+            } else {
+                self.read(0x370)
+            }
+        }
+    }
+    pub unsafe fn set_lvt_error(&mut self, lvt_error: u32) {
+        unsafe {
+            if self.x2 {
+                wrmsr(IA32_X2APIC_LVT_ERROR, u64::from(lvt_error));
+            } else {
+                self.write(0x370, lvt_error);
+            }
+        }
+    }
+    unsafe fn setup_error_int(&mut self) {
+        unsafe {
+            let vector = 49u32;
+            self.set_lvt_error(vector);
+        }
+    }
+}
+
+#[repr(u8)]
+pub enum LvtTimerMode {
+    OneShot = 0b00,
+    Periodic = 0b01,
+    TscDeadline = 0b10,
+}

src/arch/x86_shared/device/mod.rs

@@ -0,0 +1,94 @@
+use core::cell::Cell;
+
+pub mod cpu;
+pub mod hpet;
+pub mod ioapic;
+pub mod local_apic;
+pub mod pic;
+pub mod pit;
+pub mod serial;
+#[cfg(feature = "system76_ec_debug")]
+pub mod system76_ec;
+
+#[cfg(feature = "x86_kvm_pv")]
+pub mod tsc;
+
+pub unsafe fn init() {
+    unsafe {
+        pic::init();
+        local_apic::init();
+
+        // Run here for the side effect of printing if KVM was used to avoid interleaved logs.
+        tsc::get_kvm_support();
+    }
+}
+pub unsafe fn init_after_acpi() {
+    // this will disable the IOAPIC if needed.
+    //ioapic::init(mapper);
+}
+
+unsafe fn init_hpet() -> bool {
+    if cfg!(not(feature = "acpi")) {
+        return false;
+    }
+
+    unsafe {
+        use crate::acpi::ACPI_TABLE;
+        match *ACPI_TABLE.hpet.write() {
+            Some(ref mut hpet) => {
+                if cfg!(target_arch = "x86") {
+                    //TODO: fix HPET on i686
+                    warn!("HPET found but implemented on i686");
+                    return false;
+                }
+                hpet::init(hpet)
+            }
+            _ => false,
+        }
+    }
+}
+
+pub unsafe fn init_noncore() {
+    unsafe {
+        debug!("Initializing system timer");
+
+        #[cfg(feature = "x86_kvm_pv")]
+        if tsc::init() {
+            debug!("TSC used as system clock source");
+        }
+
+        if init_hpet() {
+            debug!("HPET used as system timer");
+        } else {
+            pit::init();
+            debug!("PIT used as system timer");
+        }
+
+        debug!("Finished initializing devices");
+    }
+}
+
+pub unsafe fn init_ap() {
+    unsafe {
+        local_apic::init_ap();
+
+        #[cfg(feature = "x86_kvm_pv")]
+        tsc::init();
+    }
+}
+
+pub struct ArchPercpuMisc {
+    pub apic_id_opt: Cell<Option<local_apic::ApicId>>,
+    #[cfg(feature = "x86_kvm_pv")]
+    pub tsc_info: tsc::TscPercpu,
+}
+
+impl ArchPercpuMisc {
+    pub const fn default() -> Self {
+        Self {
+            apic_id_opt: Cell::new(None),
+            #[cfg(feature = "x86_kvm_pv")]
+            tsc_info: tsc::TscPercpu::default(),
+        }
+    }
+}

src/arch/x86_shared/device/pic.rs

@@ -0,0 +1,98 @@
+use core::cell::SyncUnsafeCell;
+
+use crate::{
+    arch::interrupt::irq,
+    syscall::io::{Io, Pio},
+};
+
+static MASTER: SyncUnsafeCell<Pic> = SyncUnsafeCell::new(Pic::new(0x20));
+static SLAVE: SyncUnsafeCell<Pic> = SyncUnsafeCell::new(Pic::new(0xA0));
+
+// SAFETY: must be main thread
+pub unsafe fn master<'a>() -> &'a mut Pic {
+    unsafe { &mut *MASTER.get() }
+}
+// SAFETY: must be main thread
+pub unsafe fn slave<'a>() -> &'a mut Pic {
+    unsafe { &mut *SLAVE.get() }
+}
+
+pub unsafe fn init() {
+    unsafe {
+        let master = master();
+        let slave = slave();
+
+        // Start initialization
+        master.cmd.write(0x11);
+        slave.cmd.write(0x11);
+
+        // Set offsets
+        master.data.write(0x20);
+        slave.data.write(0x28);
+
+        // Set up cascade
+        master.data.write(4);
+        slave.data.write(2);
+
+        // Set up interrupt mode (1 is 8086/88 mode, 2 is auto EOI)
+        master.data.write(1);
+        slave.data.write(1);
+
+        // Unmask interrupts
+        master.data.write(0);
+        slave.data.write(0);
+
+        // Ack remaining interrupts
+        master.ack();
+        slave.ack();
+
+        // probably already set to PIC, but double-check
+        irq::set_irq_method(irq::IrqMethod::Pic);
+    }
+}
+
+pub unsafe fn disable() {
+    unsafe {
+        master().data.write(0xFF);
+        slave().data.write(0xFF);
+    }
+}
+
+pub struct Pic {
+    cmd: Pio<u8>,
+    data: Pio<u8>,
+}
+
+impl Pic {
+    pub const fn new(port: u16) -> Pic {
+        Pic {
+            cmd: Pio::new(port),
+            data: Pio::new(port + 1),
+        }
+    }
+
+    pub fn ack(&mut self) {
+        self.cmd.write(0x20);
+    }
+
+    pub fn mask_set(&mut self, irq: u8) {
+        assert!(irq < 8);
+
+        let mut mask = self.data.read();
+        mask |= 1 << irq;
+        self.data.write(mask);
+    }
+
+    pub fn mask_clear(&mut self, irq: u8) {
+        assert!(irq < 8);
+
+        let mut mask = self.data.read();
+        mask &= !(1 << irq);
+        self.data.write(mask);
+    }
+    /// A bitmap of all currently servicing IRQs. Spurious IRQs will not have this bit set
+    pub fn isr(&mut self) -> u8 {
+        self.cmd.write(0x0A);
+        self.cmd.read() // note that cmd is read, rather than data
+    }
+}

src/arch/x86_shared/device/pit.rs

@@ -0,0 +1,50 @@
+use core::cell::SyncUnsafeCell;
+
+use crate::syscall::io::{Io, Pio};
+
+static CHAN0: SyncUnsafeCell<Pio<u8>> = SyncUnsafeCell::new(Pio::new(0x40));
+//pub static mut CHAN1: Pio<u8> = Pio::new(0x41);
+//pub static mut CHAN2: Pio<u8> = Pio::new(0x42);
+static COMMAND: SyncUnsafeCell<Pio<u8>> = SyncUnsafeCell::new(Pio::new(0x43));
+
+// SAFETY: must be externally syncd
+pub unsafe fn chan0<'a>() -> &'a mut Pio<u8> {
+    unsafe { &mut *CHAN0.get() }
+}
+// SAFETY: must be externally syncd
+pub unsafe fn command<'a>() -> &'a mut Pio<u8> {
+    unsafe { &mut *COMMAND.get() }
+}
+
+const SELECT_CHAN0: u8 = 0b00 << 6;
+const ACCESS_LATCH: u8 = 0b00 << 4;
+const ACCESS_LOHI: u8 = 0b11 << 4;
+const MODE_2: u8 = 0b010 << 1;
+
+// 1 / (1.193182 MHz) = 838,095,110 femtoseconds ~= 838.095 ns
+pub const PERIOD_FS: u128 = 838_095_110;
+
+// 4847 / (1.193182 MHz) = 4,062,247 ns ~= 4.1 ms or 246 Hz
+pub const CHAN0_DIVISOR: u16 = 4847;
+
+// Calculated interrupt period in nanoseconds based on divisor and period
+pub const RATE: u128 = (CHAN0_DIVISOR as u128 * PERIOD_FS) / 1_000_000;
+
+pub unsafe fn init() {
+    unsafe {
+        command().write(SELECT_CHAN0 | ACCESS_LOHI | MODE_2);
+        chan0().write(CHAN0_DIVISOR as u8);
+        chan0().write((CHAN0_DIVISOR >> 8) as u8);
+    }
+}
+
+pub unsafe fn read() -> u16 {
+    unsafe {
+        command().write(SELECT_CHAN0 | ACCESS_LATCH);
+        let low = chan0().read();
+        let high = chan0().read();
+        let counter = ((high as u16) << 8) | (low as u16);
+        // Counter is inverted, subtract from CHAN0_DIVISOR
+        CHAN0_DIVISOR.saturating_sub(counter)
+    }
+}

src/arch/x86_shared/device/serial.rs

@@ -0,0 +1,48 @@
+use crate::{
+    devices::{serial::SerialKind, uart_16550::SerialPort},
+    memory::map_device_memory,
+    syscall::io::{Mmio, Pio},
+};
+use spin::Mutex;
+
+pub static COM1: Mutex<SerialKind> = Mutex::new(SerialKind::NotPresent);
+pub static COM2: Mutex<SerialKind> = Mutex::new(SerialKind::NotPresent);
+
+pub static LPSS: Mutex<SerialKind> = Mutex::new(SerialKind::NotPresent);
+
+pub unsafe fn init() {
+    #[cfg(feature = "system76_ec_debug")]
+    super::system76_ec::init();
+
+    if cfg!(not(feature = "serial_debug")) {
+        // FIXME remove serial_debug feature once ACPI SPCR is respected on UEFI boots.
+        return;
+    }
+
+    let mut com1 = SerialPort::<Pio<u8>>::new(0x3F8);
+    if com1.init().is_ok() {
+        *COM1.lock() = SerialKind::Ns16550Pio(com1);
+    }
+    let mut com2 = SerialPort::<Pio<u8>>::new(0x2F8);
+    if com2.init().is_ok() {
+        *COM2.lock() = SerialKind::Ns16550Pio(com2);
+    }
+
+    // FIXME remove explicit LPSS handling once ACPI SPCR is supported
+    if cfg!(not(feature = "lpss_debug")) {
+        return;
+    }
+
+    let virt = unsafe {
+        map_device_memory(
+            // TODO: Make this configurable
+            crate::memory::PhysicalAddress::new(0xFE032000),
+            4,
+        )
+    };
+
+    let lpss = unsafe { SerialPort::<Mmio<u32>>::new(virt.data()) };
+    if lpss.init().is_ok() {
+        *LPSS.lock() = SerialKind::Ns16550u32(lpss);
+    }
+}

src/arch/x86_shared/device/system76_ec.rs

@@ -0,0 +1,89 @@
+use spin::Mutex;
+use syscall::io::{Io, Pio};
+
+pub static SYSTEM76_EC: Mutex<Option<System76Ec>> = Mutex::new(None);
+
+pub fn init() {
+    *SYSTEM76_EC.lock() = System76Ec::new();
+}
+
+pub struct System76Ec {
+    base: u16,
+}
+
+impl System76Ec {
+    pub fn new() -> Option<Self> {
+        let mut system76_ec = Self { base: 0x0E00 };
+        if system76_ec.probe() {
+            Some(system76_ec)
+        } else {
+            None
+        }
+    }
+
+    #[inline(always)]
+    pub fn read(&mut self, addr: u8) -> u8 {
+        Pio::<u8>::new(self.base + addr as u16).read()
+    }
+
+    #[inline(always)]
+    pub fn write(&mut self, addr: u8, data: u8) {
+        Pio::<u8>::new(self.base + addr as u16).write(data)
+    }
+
+    pub fn probe(&mut self) -> bool {
+        // Send probe command
+        self.write(0, 1);
+
+        // Wait for response
+        let mut timeout = 1_000_000;
+        while timeout > 0 {
+            if self.read(0) == 0 {
+                break;
+            }
+            timeout -= 1;
+        }
+        if timeout == 0 {
+            return false;
+        }
+
+        // Return false on command error
+        if self.read(1) != 0 {
+            return false;
+        }
+
+        // Must receive 0x76, 0xEC as signature
+        self.read(2) == 0x76 && self.read(3) == 0xEC
+    }
+
+    pub fn flush(&mut self) {
+        // Send command
+        self.write(0, 4);
+
+        // TODO: timeout
+        while self.read(0) != 0 {}
+
+        // Clear length
+        self.write(3, 0);
+    }
+
+    pub fn print(&mut self, byte: u8) {
+        // Read length
+        let len = self.read(3);
+        // Write data at offset
+        self.write(len + 4, byte);
+        // Update length
+        self.write(3, len + 1);
+
+        // If we hit the end of the buffer, or were given a newline, flush
+        if byte == b'\n' || len >= 128 {
+            self.flush();
+        }
+    }
+
+    pub fn print_slice(&mut self, bytes: &[u8]) {
+        for &byte in bytes {
+            self.print(byte);
+        }
+    }
+}

src/arch/x86_shared/device/tsc.rs

@@ -0,0 +1,161 @@
+use core::{cell::Cell, ptr::addr_of};
+
+#[cfg(target_arch = "x86_64")]
+use core::arch::x86_64::__cpuid;
+
+#[cfg(target_arch = "x86")]
+use core::arch::x86::__cpuid;
+
+use rmm::Arch;
+use spin::Once;
+
+use crate::{memory::allocate_frame, percpu::PercpuBlock};
+
+pub struct KvmSupport {
+    max_leaf: u32,
+    supp_feats: KvmFeatureBits,
+}
+bitflags! {
+    // https://www.kernel.org/doc/html/latest/virt/kvm/x86/cpuid.html
+    #[derive(Debug)]
+    struct KvmFeatureBits: u32 {
+        const CLOCKSOURCE = 1 << 0;
+        const CLOCKSOURCE2 = 1 << 3;
+        const CLOCKSOURCE_STABLE = 1 << 24;
+    }
+}
+
+// https://www.kernel.org/doc/html/v5.9/virt/kvm/msr.html
+#[repr(C, packed)]
+#[derive(Clone, Copy, Debug)]
+struct PvclockVcpuTimeInfo {
+    version: u32,
+    pad: u32,
+    tsc_timestamp: u64,
+    system_time: u64,
+    tsc_to_system_mul: u32,
+    tsc_shift: i8,
+    flags: u8,
+    _pad: [u8; 2],
+}
+
+const MSR_KVM_SYSTEM_TIME_NEW: u32 = 0x4b564d01;
+const MSR_KVM_WALL_CLOCK_NEW: u32 = 0x4b564d00;
+
+pub struct TscPercpu {
+    vcpu_page: Cell<*const PvclockVcpuTimeInfo>,
+    prev: Cell<u128>,
+}
+impl TscPercpu {
+    pub const fn default() -> Self {
+        Self {
+            vcpu_page: Cell::new(core::ptr::null()),
+            prev: Cell::new(0),
+        }
+    }
+}
+
+pub fn monotonic_absolute() -> Option<u128> {
+    let inf = &PercpuBlock::current().misc_arch_info.tsc_info;
+    let ptr = inf.vcpu_page.get();
+    if ptr.is_null() {
+        return None;
+    }
+    loop {
+        unsafe {
+            let cur_version = addr_of!((*ptr).version).read_volatile();
+            if cur_version & 1 == 1 {
+                continue;
+            }
+            let elapsed_ticks =
+                x86::time::rdtsc().saturating_sub(addr_of!((*ptr).tsc_timestamp).read_volatile());
+            let tsc_shift = addr_of!((*ptr).tsc_shift).read_volatile();
+            let elapsed = if tsc_shift >= 0 {
+                elapsed_ticks.checked_shl(tsc_shift as u32).unwrap()
+            } else {
+                elapsed_ticks.checked_shr((-tsc_shift) as u32).unwrap()
+            };
+            let system_time = addr_of!((*ptr).system_time).read_volatile();
+            let tsc_to_system_mul = addr_of!((*ptr).tsc_to_system_mul).read_volatile();
+            let new_version = addr_of!((*ptr).version).read_volatile();
+            if new_version != cur_version || new_version & 1 == 1 {
+                continue;
+            }
+            let delta = (u128::from(elapsed) * u128::from(tsc_to_system_mul)) >> 32;
+            let time = u128::from(system_time) + delta;
+            let prev = inf.prev.replace(time);
+            if prev > time {
+                // TODO
+                error!("TSC wraparound ({prev} > {time})");
+                return None;
+            }
+            assert!(prev <= time);
+            return Some(time);
+        }
+    }
+}
+
+pub fn get_kvm_support() -> &'static Option<KvmSupport> {
+    static KVM_SUPPORT: Once<Option<KvmSupport>> = Once::new();
+
+    KVM_SUPPORT.call_once(|| {
+        let res = unsafe { __cpuid(0x4000_0000) };
+        if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] {
+            return None;
+        }
+        let max_leaf = res.eax;
+        if max_leaf < 0x4000_0001 {
+            return None;
+        }
+        let res = unsafe { __cpuid(0x4000_0001) };
+
+        let supp_feats = KvmFeatureBits::from_bits_retain(res.eax);
+
+        debug!("Detected KVM paravirtualization support, features {supp_feats:?}");
+
+        Some(KvmSupport {
+            max_leaf,
+            supp_feats,
+        })
+    })
+}
+
+pub unsafe fn init() -> bool {
+    unsafe {
+        let cpuid = crate::arch::cpuid::cpuid();
+        if !cpuid.get_feature_info().is_some_and(|f| f.has_tsc()) {
+            return false;
+        }
+
+        let kvm_support = get_kvm_support();
+
+        if let Some(kvm_support) = kvm_support
+            && kvm_support
+                .supp_feats
+                .contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE)
+        {
+            let frame = allocate_frame().expect("failed to allocate timer page");
+            x86::msr::wrmsr(MSR_KVM_SYSTEM_TIME_NEW, (frame.base().data() as u64) | 1);
+            let ptr = crate::memory::RmmA::phys_to_virt(frame.base()).data()
+                as *const PvclockVcpuTimeInfo;
+            PercpuBlock::current()
+                .misc_arch_info
+                .tsc_info
+                .vcpu_page
+                .set(ptr);
+
+            /*let tsc_ghz = loop {
+                let val1 = ptr.read_volatile();
+                let val2 = ptr.read_volatile();
+                if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version {
+                    continue;
+                }
+                let val1
+                break tsc_hz / 1_000_000_000;
+            };*/
+            true
+        } else {
+            false
+        }
+    }
+}

src/arch/x86_shared/gdt.rs

@@ -0,0 +1,431 @@
+//! Global descriptor table
+
+use core::ptr;
+
+#[cfg(target_arch = "x86")]
+use x86::bits32::task::TaskStateSegment;
+#[cfg(target_arch = "x86_64")]
+use x86::bits64::task::TaskStateSegment;
+use x86::{
+    dtables::{self, DescriptorTablePointer},
+    segmentation::{self, Descriptor as SegmentDescriptor, SegmentSelector},
+    task, Ring,
+};
+
+use crate::{
+    cpu_set::LogicalCpuId,
+    memory::{RmmA, RmmArch, PAGE_SIZE},
+    percpu::PercpuBlock,
+};
+
+pub const GDT_NULL: usize = 0;
+pub const GDT_KERNEL_CODE: usize = 1;
+pub const GDT_KERNEL_DATA: usize = 2;
+#[cfg(target_arch = "x86")]
+pub const GDT_KERNEL_PERCPU: usize = 3;
+#[cfg(target_arch = "x86_64")]
+pub const GDT_USER_CODE32_UNUSED: usize = 3;
+pub const GDT_USER_DATA: usize = 4;
+pub const GDT_USER_CODE: usize = 5;
+#[cfg(target_arch = "x86")]
+pub const GDT_USER_FS: usize = 6;
+#[cfg(target_arch = "x86")]
+pub const GDT_USER_GS: usize = 7;
+#[cfg(target_arch = "x86")]
+pub const GDT_TSS: usize = 8;
+#[cfg(target_arch = "x86_64")]
+pub const GDT_TSS: usize = 6;
+#[cfg(target_arch = "x86_64")]
+pub const GDT_TSS_HIGH: usize = 7;
+
+pub const GDT_A_PRESENT: u8 = 1 << 7;
+pub const GDT_A_RING_0: u8 = 0 << 5;
+pub const GDT_A_RING_1: u8 = 1 << 5;
+pub const GDT_A_RING_2: u8 = 2 << 5;
+pub const GDT_A_RING_3: u8 = 3 << 5;
+pub const GDT_A_SYSTEM: u8 = 1 << 4;
+pub const GDT_A_EXECUTABLE: u8 = 1 << 3;
+pub const GDT_A_CONFORMING: u8 = 1 << 2;
+pub const GDT_A_PRIVILEGE: u8 = 1 << 1;
+pub const GDT_A_DIRTY: u8 = 1;
+
+pub const GDT_A_TSS_AVAIL: u8 = 0x9;
+pub const GDT_A_TSS_BUSY: u8 = 0xB;
+
+pub const GDT_F_PAGE_SIZE: u8 = 1 << 7;
+pub const GDT_F_PROTECTED_MODE: u8 = 1 << 6;
+pub const GDT_F_LONG_MODE: u8 = 1 << 5;
+
+const IOBITMAP_SIZE: u32 = 65536 / 8;
+
+#[cfg(target_arch = "x86")]
+const SEGMENT_LIMIT: u32 = 0xFFFFF;
+#[cfg(target_arch = "x86_64")]
+const SEGMENT_LIMIT: u32 = 0;
+
+#[cfg(target_arch = "x86")]
+const SEGMENT_FLAGS: u8 = GDT_F_PAGE_SIZE | GDT_F_PROTECTED_MODE;
+#[cfg(target_arch = "x86_64")]
+const SEGMENT_FLAGS: u8 = GDT_F_LONG_MODE;
+
+#[cfg(target_arch = "x86")]
+const SEGMENT_COUNT: usize = 9;
+#[cfg(target_arch = "x86_64")]
+const SEGMENT_COUNT: usize = 8;
+
+// Later copied into the actual GDT with various fields set.
+const BASE_GDT: [GdtEntry; SEGMENT_COUNT] = [
+    // Null
+    GdtEntry::new(0, 0, 0, 0),
+    // Kernel code
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // Kernel data
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // Kernel TLS
+    #[cfg(target_arch = "x86")]
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // Dummy 32-bit user code - apparently necessary for SYSRET. We restrict it to ring 0 anyway.
+    #[cfg(target_arch = "x86_64")]
+    GdtEntry::new(
+        0,
+        0,
+        GDT_A_PRESENT | GDT_A_RING_0 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE,
+        GDT_F_PROTECTED_MODE,
+    ),
+    // User data
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // User code
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_EXECUTABLE | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // User FS (for TLS)
+    #[cfg(target_arch = "x86")]
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // User GS (for TLS)
+    #[cfg(target_arch = "x86")]
+    GdtEntry::new(
+        0,
+        SEGMENT_LIMIT,
+        GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_SYSTEM | GDT_A_PRIVILEGE,
+        SEGMENT_FLAGS,
+    ),
+    // TSS
+    GdtEntry::new(0, 0, GDT_A_PRESENT | GDT_A_RING_3 | GDT_A_TSS_AVAIL, 0),
+    // TSS must be 16 bytes long, twice the normal size
+    #[cfg(target_arch = "x86_64")]
+    GdtEntry::new(0, 0, 0, 0),
+];
+
+#[repr(C, align(16))]
+struct Align([u64; 2]);
+
+#[repr(C, align(4096))]
+pub struct ProcessorControlRegion {
+    // TODO: When both KASLR and KPTI are implemented, the PCR may need to be split into two pages,
+    // such that "secret" kernel addresses are only stored in the protected half.
+    pub self_ref: *mut ProcessorControlRegion,
+
+    pub user_rsp_tmp: usize,
+    // The GDT *must* be stored in the PCR! The paranoid interrupt handler, lacking a reliable way
+    // to correctly obtain GSBASE, uses SGDT to calculate the PCR offset.
+    pub gdt: [GdtEntry; SEGMENT_COUNT],
+    pub percpu: PercpuBlock,
+    _rsvd: Align,
+    pub tss: TaskStateSegment,
+
+    // These two fields are read by the CPU, but not currently modified by the kernel. Instead, the
+    // kernel sets the `iomap_base` field in the TSS, to either point to this bitmap, or outside
+    // the TSS, in which case userspace is not granted port IO access.
+    pub _iobitmap: [u8; IOBITMAP_SIZE as usize],
+    pub _all_ones: u8,
+}
+
+const _: () = {
+    if core::mem::offset_of!(ProcessorControlRegion, tss) % 16 != 0 {
+        panic!("PCR is incorrectly defined, TSS alignment is too small");
+    }
+    if core::mem::offset_of!(ProcessorControlRegion, gdt) % 8 != 0 {
+        panic!("PCR is incorrectly defined, GDT alignment is too small");
+    }
+};
+
+impl ProcessorControlRegion {
+    const fn new_partial_init(cpu_id: LogicalCpuId) -> Self {
+        Self {
+            self_ref: ptr::null_mut(),
+            user_rsp_tmp: 0,
+            gdt: BASE_GDT,
+            percpu: PercpuBlock::init(cpu_id),
+            _rsvd: Align([0; 2]),
+            tss: TaskStateSegment::new(),
+            _iobitmap: [0; IOBITMAP_SIZE as usize],
+            _all_ones: 0xFF,
+        }
+    }
+}
+
+pub unsafe fn pcr() -> *mut ProcessorControlRegion {
+    unsafe {
+        // Primitive benchmarking of RDFSBASE and RDGSBASE in userspace, appears to indicate that
+        // obtaining FSBASE/GSBASE using mov gs:[gs_self_ref] is faster than using the (probably
+        // microcoded) instructions.
+        let mut ret: *mut ProcessorControlRegion;
+        core::arch::asm!("mov {}, gs:[{}]", out(reg) ret, const(core::mem::offset_of!(ProcessorControlRegion, self_ref)));
+        ret
+    }
+}
+
+#[cfg(feature = "pti")]
+pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) {
+    use super::pti::{PTI_CONTEXT_STACK, PTI_CPU_STACK};
+
+    #[cfg(target_arch = "x86")]
+    unsafe {
+        core::ptr::addr_of_mut!((*pcr).tss.ss0).write((GDT_KERNEL_DATA << 3) as u16);
+        core::ptr::addr_of_mut!((*pcr).tss.esp0)
+            .write((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u32);
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    unsafe {
+        core::ptr::addr_of_mut!((*pcr).tss.rsp[0])
+            .write_unaligned((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64);
+    }
+
+    unsafe { PTI_CONTEXT_STACK = stack };
+}
+
+#[cfg(not(feature = "pti"))]
+pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) {
+    #[cfg(target_arch = "x86")]
+    unsafe {
+        core::ptr::addr_of_mut!((*pcr).tss.ss0).write((GDT_KERNEL_DATA << 3) as u16);
+        core::ptr::addr_of_mut!((*pcr).tss.esp0).write(stack as u32);
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    unsafe {
+        // TODO: If this increases performance, read gs:[offset] directly
+        core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64);
+    }
+}
+
+pub unsafe fn set_userspace_io_allowed(pcr: *mut ProcessorControlRegion, allowed: bool) {
+    let offset = if allowed {
+        u16::try_from(size_of::<TaskStateSegment>()).expect("guaranteed to fit in u16")
+    } else {
+        0xFFFF
+    };
+
+    unsafe {
+        #[cfg(target_arch = "x86")]
+        core::ptr::addr_of_mut!((*pcr).tss.iobp_offset).write(offset);
+
+        #[cfg(target_arch = "x86_64")]
+        core::ptr::addr_of_mut!((*pcr).tss.iomap_base).write(offset);
+    }
+}
+
+#[cold]
+fn init_pcr(pcr: &mut ProcessorControlRegion, stack_end: usize) {
+    pcr.self_ref = pcr as *mut _;
+
+    // Setup the GDT.
+    #[cfg(target_arch = "x86")]
+    pcr.gdt[GDT_KERNEL_PERCPU].set_offset(pcr as *const _ as u32);
+
+    #[cfg(target_arch = "x86")]
+    {
+        pcr.tss.iobp_offset = 0xFFFF;
+        let tss = &pcr.tss as *const _ as usize as u32;
+
+        pcr.gdt[GDT_TSS].set_offset(tss);
+        pcr.gdt[GDT_TSS].set_limit(size_of::<TaskStateSegment>() as u32 + IOBITMAP_SIZE as u32);
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    {
+        pcr.tss.iomap_base = 0xFFFF;
+
+        let tss = &mut pcr.tss as *mut TaskStateSegment as usize as u64;
+        let tss_lo = (tss & 0xFFFF_FFFF) as u32;
+        let tss_hi = (tss >> 32) as u32;
+
+        pcr.gdt[GDT_TSS].set_offset(tss_lo);
+        pcr.gdt[GDT_TSS].set_limit(size_of::<TaskStateSegment>() as u32 + IOBITMAP_SIZE);
+
+        // GDT is aligned to 8 bytes
+        #[expect(clippy::cast_ptr_alignment)]
+        unsafe {
+            (&mut pcr.gdt[GDT_TSS_HIGH] as *mut GdtEntry)
+                .cast::<u32>()
+                .write(tss_hi);
+        }
+    }
+
+    // Set the stack pointer to use when coming back from userspace.
+    unsafe {
+        set_tss_stack(pcr, stack_end);
+    }
+}
+
+#[cold]
+pub unsafe fn install_pcr(pcr_ptr: *mut ProcessorControlRegion) {
+    let pcr = unsafe { &mut *pcr_ptr };
+
+    let gdtr: DescriptorTablePointer<SegmentDescriptor> = DescriptorTablePointer {
+        limit: const { (SEGMENT_COUNT * size_of::<GdtEntry>() - 1) as u16 },
+        base: pcr.gdt.as_ptr() as *const SegmentDescriptor,
+    };
+
+    // Load the new GDT, which is correctly located in thread local storage.
+    unsafe { dtables::lgdt(&gdtr) };
+
+    #[cfg(target_arch = "x86")]
+    unsafe {
+        // Reload the segment descriptors
+        segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
+        segmentation::load_ds(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3));
+        segmentation::load_es(SegmentSelector::new(GDT_USER_DATA as u16, Ring::Ring3));
+        segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
+
+        // TODO: Use FS for kernel percpu on i686?
+        segmentation::load_fs(SegmentSelector::new(GDT_USER_FS as u16, Ring::Ring0));
+        segmentation::load_gs(SegmentSelector::new(GDT_KERNEL_PERCPU as u16, Ring::Ring0));
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    unsafe {
+        // Load segments again, possibly resetting FSBASE and GSBASE.
+        segmentation::load_cs(SegmentSelector::new(GDT_KERNEL_CODE as u16, Ring::Ring0));
+        segmentation::load_ss(SegmentSelector::new(GDT_KERNEL_DATA as u16, Ring::Ring0));
+
+        segmentation::load_ds(SegmentSelector::from_raw(0));
+        segmentation::load_es(SegmentSelector::from_raw(0));
+        segmentation::load_fs(SegmentSelector::from_raw(0));
+
+        // What happens when GS is loaded with a NULL selector, is undefined on Intel CPUs. However,
+        // GSBASE is set later.
+        segmentation::load_gs(SegmentSelector::from_raw(0));
+
+        // Ensure that GSBASE always points to the PCR in kernel space.
+        x86::msr::wrmsr(x86::msr::IA32_GS_BASE, pcr as *mut _ as usize as u64);
+
+        // While GSBASE points to the PCR in kernel space, userspace is free to set it to other values.
+        // Zero-initialize userspace's GSBASE. The reason the GSBASE register writes are reversed, is
+        // because entering usermode will entail executing the SWAPGS instruction.
+        x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, 0);
+
+        // Set the userspace FSBASE to zero.
+        x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0);
+    }
+
+    // Load the task register
+    unsafe { task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)) };
+
+    unsafe { crate::percpu::init_tlb_shootdown(pcr.percpu.cpu_id, &mut pcr.percpu) };
+}
+
+/// Initialize GDT and configure percpu for the BSP.
+#[cold]
+pub unsafe fn init_bsp(stack_end: usize) {
+    static mut BSP_PCR: ProcessorControlRegion =
+        ProcessorControlRegion::new_partial_init(LogicalCpuId::BSP);
+
+    init_pcr(unsafe { &mut *ptr::addr_of_mut!(BSP_PCR) }, stack_end);
+
+    unsafe { install_pcr(ptr::addr_of_mut!(BSP_PCR)) };
+}
+
+#[cold]
+pub fn allocate_and_init_pcr(
+    cpu_id: LogicalCpuId,
+    stack_end: usize,
+) -> *mut ProcessorControlRegion {
+    let alloc_order = size_of::<ProcessorControlRegion>()
+        .div_ceil(PAGE_SIZE)
+        .next_power_of_two()
+        .trailing_zeros();
+
+    let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR");
+    let pcr_ptr = RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion;
+    unsafe { core::ptr::write(pcr_ptr, ProcessorControlRegion::new_partial_init(cpu_id)) };
+
+    init_pcr(unsafe { &mut *pcr_ptr }, stack_end);
+
+    pcr_ptr
+}
+
+#[derive(Copy, Clone, Debug)]
+#[repr(C, packed)]
+pub struct GdtEntry {
+    pub limitl: u16,
+    pub offsetl: u16,
+    pub offsetm: u8,
+    pub access: u8,
+    pub flags_limith: u8,
+    pub offseth: u8,
+}
+
+impl GdtEntry {
+    pub const fn new(offset: u32, limit: u32, access: u8, flags: u8) -> Self {
+        GdtEntry {
+            limitl: limit as u16,
+            offsetl: offset as u16,
+            offsetm: (offset >> 16) as u8,
+            access,
+            flags_limith: flags & 0xF0 | ((limit >> 16) as u8) & 0x0F,
+            offseth: (offset >> 24) as u8,
+        }
+    }
+
+    #[cfg(target_arch = "x86")]
+    pub const fn offset(&self) -> u32 {
+        (self.offsetl as u32) | ((self.offsetm as u32) << 16) | ((self.offseth as u32) << 24)
+    }
+
+    pub const fn set_offset(&mut self, offset: u32) {
+        self.offsetl = offset as u16;
+        self.offsetm = (offset >> 16) as u8;
+        self.offseth = (offset >> 24) as u8;
+    }
+
+    pub const fn set_limit(&mut self, limit: u32) {
+        self.limitl = limit as u16;
+        self.flags_limith = self.flags_limith & 0xF0 | ((limit >> 16) as u8) & 0x0F;
+    }
+}
+
+impl PercpuBlock {
+    pub fn current() -> &'static Self {
+        unsafe { &*core::ptr::addr_of!((*pcr()).percpu) }
+    }
+}

src/arch/x86_shared/idt.rs

@@ -0,0 +1,361 @@
+use core::{
+    cell::SyncUnsafeCell,
+    mem,
+    sync::atomic::{AtomicU32, Ordering},
+};
+
+use alloc::boxed::Box;
+use hashbrown::{hash_map::DefaultHashBuilder, HashMap};
+
+use x86::{
+    dtables::{self, DescriptorTablePointer},
+    segmentation::Descriptor as X86IdtEntry,
+};
+
+use crate::{
+    arch::{
+        interrupt::{
+            irq::{__generic_interrupts_end, __generic_interrupts_start},
+            *,
+        },
+        ipi::IpiKind,
+    },
+    cpu_set::LogicalCpuId,
+    memory::PAGE_SIZE,
+};
+
+use spin::RwLock;
+
+#[repr(C)]
+pub struct Idt {
+    pub(crate) entries: [IdtEntry; 256],
+    reservations: [AtomicU32; 8],
+    backup_stack_end: usize,
+}
+
+impl Idt {
+    const fn new() -> Self {
+        Self {
+            entries: [IdtEntry::new(); 256],
+            reservations: [const { AtomicU32::new(0) }; 8],
+            backup_stack_end: 0,
+        }
+    }
+
+    #[inline]
+    fn is_reserved(&self, index: u8) -> bool {
+        let byte_index = index / 32;
+        let bit = index % 32;
+
+        self.reservations[usize::from(byte_index)].load(Ordering::Acquire) & (1 << bit) != 0
+    }
+
+    #[inline]
+    pub(crate) fn set_reserved(&self, index: u8, reserved: bool) {
+        let byte_index = index / 32;
+        let bit = index % 32;
+
+        self.reservations[usize::from(byte_index)]
+            .fetch_or(u32::from(reserved) << bit, Ordering::AcqRel);
+    }
+
+    #[inline]
+    pub(crate) fn set_reserved_mut(&mut self, index: u8, reserved: bool) {
+        let byte_index = index / 32;
+        let bit = index % 32;
+
+        *{ &mut self.reservations[usize::from(byte_index)] }.get_mut() |=
+            u32::from(reserved) << bit;
+    }
+}
+
+// Allocate 64 KiB of stack space for the backup stack.
+const BACKUP_STACK_SIZE: usize = PAGE_SIZE << 4;
+
+static INIT_BSP_IDT: SyncUnsafeCell<Idt> = SyncUnsafeCell::new(Idt::new());
+
+// TODO: VecMap?
+pub(crate) static IDTS: RwLock<HashMap<LogicalCpuId, &'static mut Idt>> =
+    RwLock::new(HashMap::with_hasher(DefaultHashBuilder::new()));
+
+#[inline]
+pub fn is_reserved(cpu_id: LogicalCpuId, index: u8) -> bool {
+    if cpu_id == LogicalCpuId::BSP {
+        return unsafe { (*INIT_BSP_IDT.get()).is_reserved(index) };
+    }
+
+    IDTS.read().get(&cpu_id).unwrap().is_reserved(index)
+}
+
+#[inline]
+pub fn set_reserved(cpu_id: LogicalCpuId, index: u8, reserved: bool) {
+    if cpu_id == LogicalCpuId::BSP {
+        unsafe { (*INIT_BSP_IDT.get()).set_reserved(index, reserved) };
+        return;
+    }
+
+    IDTS.read()
+        .get(&cpu_id)
+        .unwrap()
+        .set_reserved(index, reserved);
+}
+
+pub fn available_irqs_iter(cpu_id: LogicalCpuId) -> impl Iterator<Item = u8> + 'static {
+    (32..=254).filter(move |&index| !is_reserved(cpu_id, index))
+}
+
+fn set_exceptions(idt: &mut [IdtEntry]) {
+    // Set up exceptions
+    idt[0].set_func(exception::divide_by_zero);
+    idt[1].set_func(exception::debug);
+    idt[2].set_func(exception::non_maskable);
+    idt[3].set_func(exception::breakpoint);
+    idt[3].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT);
+    idt[4].set_func(exception::overflow);
+    idt[5].set_func(exception::bound_range);
+    idt[6].set_func(exception::invalid_opcode);
+    idt[7].set_func(exception::device_not_available);
+    idt[8].set_func(exception::double_fault);
+    // 9 no longer available
+    idt[10].set_func(exception::invalid_tss);
+    idt[11].set_func(exception::segment_not_present);
+    idt[12].set_func(exception::stack_segment);
+    idt[13].set_func(exception::protection);
+    idt[14].set_func(exception::page);
+    // 15 reserved
+    idt[16].set_func(exception::fpu_fault);
+    idt[17].set_func(exception::alignment_check);
+    idt[18].set_func(exception::machine_check);
+    idt[19].set_func(exception::simd);
+    idt[20].set_func(exception::virtualization);
+    // 21 through 29 reserved
+    idt[30].set_func(exception::security);
+    // 31 reserved
+}
+
+/// Initializes a fully functional IDT for use before it be moved into the map. This is ONLY called
+/// on the BSP, since the kernel heap is ready for the APs.
+pub unsafe fn init_bsp() {
+    #[repr(C, packed(4096))]
+    struct BackupStack([u8; BACKUP_STACK_SIZE]);
+
+    static INIT_BSP_BACKUP_STACK: SyncUnsafeCell<BackupStack> =
+        SyncUnsafeCell::new(BackupStack([0; BACKUP_STACK_SIZE]));
+
+    unsafe {
+        init_generic(
+            LogicalCpuId::BSP,
+            &mut *INIT_BSP_IDT.get(),
+            INIT_BSP_BACKUP_STACK.get().addr() + BACKUP_STACK_SIZE,
+        );
+
+        install_idt(&mut *INIT_BSP_IDT.get());
+    }
+}
+
+pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt {
+    let mut idts_btree = IDTS.write();
+
+    let idt = idts_btree
+        .entry(cpu_id)
+        .or_insert_with(|| Box::leak(Box::new(Idt::new())));
+
+    use crate::memory::{RmmA, RmmArch};
+    let frames = crate::memory::allocate_p2frame(4)
+        .expect("failed to allocate pages for backup interrupt stack");
+
+    // Physical pages are mapped linearly. So is the linearly mapped virtual memory.
+    let base_address = RmmA::phys_to_virt(frames.base());
+
+    // Stack always grows downwards.
+    let backup_stack_end = base_address.data() + BACKUP_STACK_SIZE;
+
+    init_generic(cpu_id, idt, backup_stack_end);
+
+    *idt
+}
+
+const BACKUP_IST: u8 = 1;
+
+/// Initializes an IDT for any type of processor.
+fn init_generic(cpu_id: LogicalCpuId, idt: &mut Idt, backup_stack_end: usize) {
+    let (current_idt, current_reservations) = (&mut idt.entries, &mut idt.reservations);
+
+    set_exceptions(current_idt);
+
+    // We give Non-Maskable Interrupts, Double Fault, and Machine Check exceptions separate
+    // stacks, since these (unless we are going to set up NMI watchdogs like Linux does) are
+    // considered the most fatal, especially Double Faults which are caused by errors __when
+    // accessing the system IDT__. If that goes wrong, then kernel memory may be partially
+    // corrupt, and we want a separate stack.
+    //
+    // Note that each CPU has its own "backup interrupt stack".
+    idt.backup_stack_end = backup_stack_end;
+    current_idt[2].set_ist(BACKUP_IST);
+    current_idt[8].set_ist(BACKUP_IST);
+    current_idt[18].set_ist(BACKUP_IST);
+
+    assert_eq!(
+        __generic_interrupts_end as usize - __generic_interrupts_start as usize,
+        224 * 8
+    );
+
+    for i in 0..224 {
+        current_idt[i + 32].set_func(unsafe {
+            mem::transmute::<usize, unsafe extern "C" fn()>(
+                __generic_interrupts_start as usize + i * 8,
+            )
+        });
+    }
+
+    // reserve bits 31:0, i.e. the first 32 interrupts, which are reserved for exceptions
+    *current_reservations[0].get_mut() |= 0x0000_0000_FFFF_FFFF;
+
+    if cpu_id == LogicalCpuId::BSP {
+        // Set up IRQs
+        current_idt[32].set_func(irq::pit_stack);
+        current_idt[33].set_func(irq::keyboard);
+        current_idt[34].set_func(irq::cascade);
+        current_idt[35].set_func(irq::com2);
+        current_idt[36].set_func(irq::com1);
+        current_idt[37].set_func(irq::lpt2);
+        current_idt[38].set_func(irq::floppy);
+        current_idt[39].set_func(irq::lpt1);
+        current_idt[40].set_func(irq::rtc);
+        current_idt[41].set_func(irq::pci1);
+        current_idt[42].set_func(irq::pci2);
+        current_idt[43].set_func(irq::pci3);
+        current_idt[44].set_func(irq::mouse);
+        current_idt[45].set_func(irq::fpu);
+        current_idt[46].set_func(irq::ata1);
+        current_idt[47].set_func(irq::ata2);
+        current_idt[48].set_func(irq::lapic_timer);
+        current_idt[49].set_func(irq::lapic_error);
+
+        // reserve bits 49:32, which are for the standard IRQs, and for the local apic timer and error.
+        *current_reservations[1].get_mut() |= 0x0003_FFFF;
+    } else {
+        // TODO: use_default_irqs! but also the legacy IRQs that are only needed on one CPU
+        current_idt[49].set_func(irq::lapic_error);
+
+        // reserve bit 49
+        *current_reservations[1].get_mut() |= 1 << 17;
+    }
+
+    // Set IPI handlers
+    current_idt[IpiKind::Wakeup as usize].set_func(ipi::wakeup);
+    current_idt[IpiKind::Switch as usize].set_func(ipi::switch);
+    current_idt[IpiKind::Tlb as usize].set_func(ipi::tlb);
+    current_idt[IpiKind::Pit as usize].set_func(ipi::pit);
+    idt.set_reserved_mut(IpiKind::Wakeup as u8, true);
+    idt.set_reserved_mut(IpiKind::Switch as u8, true);
+    idt.set_reserved_mut(IpiKind::Tlb as u8, true);
+    idt.set_reserved_mut(IpiKind::Pit as u8, true);
+
+    #[cfg(target_arch = "x86")]
+    {
+        let current_idt = &mut idt.entries;
+        // Set syscall function
+        current_idt[0x80].set_func(syscall::syscall);
+        current_idt[0x80].set_flags(IdtFlags::PRESENT | IdtFlags::RING_3 | IdtFlags::INTERRUPT);
+        idt.set_reserved_mut(0x80, true);
+    }
+
+    #[cfg(feature = "profiling")]
+    crate::profiling::maybe_setup_timer(idt, cpu_id);
+}
+
+pub unsafe fn install_idt(idt_ptr: *mut Idt) {
+    unsafe {
+        let idt = &mut *idt_ptr;
+
+        #[cfg(target_arch = "x86_64")] // TODO: x86
+        {
+            (*crate::arch::gdt::pcr()).tss.ist[usize::from(BACKUP_IST - 1)] =
+                idt.backup_stack_end as u64;
+        }
+
+        let idtr: DescriptorTablePointer<X86IdtEntry> = DescriptorTablePointer {
+            limit: (idt.entries.len() * size_of::<IdtEntry>() - 1) as u16,
+            base: idt.entries.as_ptr() as *const X86IdtEntry,
+        };
+
+        dtables::lidt(&idtr);
+    }
+}
+
+bitflags! {
+    pub struct IdtFlags: u8 {
+        const PRESENT = 1 << 7;
+        const RING_0 = 0 << 5;
+        const RING_1 = 1 << 5;
+        const RING_2 = 2 << 5;
+        const RING_3 = 3 << 5;
+        const SS = 1 << 4;
+        const INTERRUPT = 0xE;
+        const TRAP = 0xF;
+    }
+}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(C, packed)]
+pub struct IdtEntry {
+    offsetl: u16,
+    selector: u16,
+    zero: u8,
+    attribute: u8,
+    offsetm: u16,
+    #[cfg(target_arch = "x86_64")]
+    offseth: u32,
+    #[cfg(target_arch = "x86_64")]
+    _zero2: u32,
+}
+
+impl IdtEntry {
+    pub const fn new() -> IdtEntry {
+        IdtEntry {
+            offsetl: 0,
+            selector: 0,
+            zero: 0,
+            attribute: 0,
+            offsetm: 0,
+            #[cfg(target_arch = "x86_64")]
+            offseth: 0,
+            #[cfg(target_arch = "x86_64")]
+            _zero2: 0,
+        }
+    }
+
+    pub fn set_flags(&mut self, flags: IdtFlags) {
+        self.attribute = flags.bits();
+    }
+
+    pub fn set_ist(&mut self, ist: u8) {
+        assert_eq!(
+            ist & 0x07,
+            ist,
+            "interrupt stack table must be within 0..=7"
+        );
+        self.zero &= 0xF8;
+        self.zero |= ist;
+    }
+
+    pub fn set_offset(&mut self, selector: u16, base: usize) {
+        self.selector = selector;
+        self.offsetl = base as u16;
+        self.offsetm = (base >> 16) as u16;
+        #[cfg(target_arch = "x86_64")]
+        {
+            self.offseth = ((base as u64) >> 32) as u32;
+        }
+    }
+
+    // A function to set the offset more easily
+    pub fn set_func(&mut self, func: unsafe extern "C" fn()) {
+        self.set_flags(IdtFlags::PRESENT | IdtFlags::RING_0 | IdtFlags::INTERRUPT);
+        self.set_offset(
+            (crate::arch::gdt::GDT_KERNEL_CODE as u16) << 3,
+            func as usize,
+        );
+    }
+}

src/arch/x86_shared/interrupt/exception.rs

@@ -0,0 +1,294 @@
+use syscall::Exception;
+use x86::irq::PageFaultError;
+
+use crate::{
+    arch::x86_shared::interrupt,
+    context::signal::excp_handler,
+    memory::{GenericPfFlags, VirtualAddress},
+    ptrace,
+    sync::CleanLockToken,
+    syscall::flag::*,
+};
+
+interrupt_stack!(divide_by_zero, |stack| {
+    println!("Divide by zero");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 0,
+        ..Default::default()
+    });
+});
+
+interrupt_stack!(debug, @paranoid, |stack| {
+    let mut handled = false;
+
+    // Disable singlestep before there is a breakpoint, since the breakpoint
+    // handler might end up setting it again but unless it does we want the
+    // default to be false.
+    #[cfg(target_arch = "x86")]
+    let had_singlestep = stack.iret.eflags & (1 << 8) == 1 << 8;
+    #[cfg(target_arch = "x86_64")]
+    let had_singlestep = stack.iret.rflags & (1 << 8) == 1 << 8;
+    stack.set_singlestep(false);
+
+    let mut token = unsafe { CleanLockToken::new() };
+    if ptrace::breakpoint_callback(PTRACE_STOP_SINGLESTEP, None, &mut token).is_some() {
+        handled = true;
+    } else {
+        // There was no breakpoint, restore original value
+        stack.set_singlestep(had_singlestep);
+    }
+
+    if !handled {
+        println!("Debug trap");
+        stack.dump();
+        excp_handler(Exception {
+            kind: 1,
+            ..Default::default()
+        });
+    }
+});
+
+interrupt_stack!(non_maskable, @paranoid, |stack| {
+    #[cfg(target_arch = "x86_64")]
+    unsafe { crate::profiling::nmi_handler(stack) };
+
+    #[cfg(not(all(target_arch = "x86_64", feature = "profiling")))]
+    {
+        // TODO: This will likely deadlock
+        println!("Non-maskable interrupt");
+        stack.dump();
+    }
+});
+
+interrupt_stack!(breakpoint, |stack| {
+    // The processor lets EIP/RIP point to the instruction *after* int3, so
+    // unhandled breakpoint interrupt don't go in an infinite loop. But we
+    // throw SIGTRAP anyway, so that's not a problem.
+    //
+    // We have the following code to prevent
+    // - RIP from going out of sync with instructions
+    // - The user having to do 2 syscalls to replace the instruction at RIP
+    // - Having more compatibility glue for GDB than necessary
+    //
+    // Let's just follow Linux convention and let RIP be RIP-1, point to the
+    // int3 instruction. After all, it's the sanest thing to do.
+    #[cfg(target_arch = "x86")]
+    {
+        stack.iret.eip -= 1;
+    }
+    #[cfg(target_arch = "x86_64")]
+    {
+        stack.iret.rip -= 1;
+    }
+
+    let mut token = unsafe { CleanLockToken::new() };
+    if ptrace::breakpoint_callback(PTRACE_STOP_BREAKPOINT, None, &mut token).is_none() {
+        println!("Breakpoint trap");
+        stack.dump();
+        excp_handler(Exception {
+            kind: 3,
+            ..Default::default()
+        });
+    }
+});
+
+interrupt_stack!(overflow, |stack| {
+    println!("Overflow trap");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 4,
+        ..Default::default()
+    });
+});
+
+interrupt_stack!(bound_range, |stack| {
+    println!("Bound range exceeded fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 5,
+        ..Default::default()
+    });
+});
+
+interrupt_stack!(invalid_opcode, |stack| {
+    println!("Invalid opcode fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 6,
+        ..Default::default()
+    });
+});
+
+interrupt_stack!(device_not_available, |stack| {
+    println!("Device not available fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 7,
+        ..Default::default()
+    });
+});
+
+interrupt_error!(double_fault, |stack, _code| {
+    println!("Double fault");
+    stack.trace();
+    unsafe {
+        loop {
+            interrupt::disable();
+            interrupt::halt();
+        }
+    }
+});
+
+interrupt_error!(invalid_tss, |stack, code| {
+    println!("Invalid TSS fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 10,
+        code,
+        ..Default::default()
+    });
+});
+
+interrupt_error!(segment_not_present, |stack, code| {
+    println!("Segment not present fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 11,
+        code,
+        ..Default::default()
+    });
+});
+
+interrupt_error!(stack_segment, |stack, code| {
+    println!("Stack segment fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 12,
+        code,
+        ..Default::default()
+    });
+});
+
+interrupt_error!(protection, |stack, code| {
+    println!("Protection fault code={:#0x}", code);
+    stack.trace();
+    excp_handler(Exception {
+        kind: 13,
+        code,
+        ..Default::default()
+    });
+});
+
+interrupt_error!(page, |stack, code| {
+    let cr2 = VirtualAddress::new(unsafe { x86::controlregs::cr2() });
+    let arch_flags = PageFaultError::from_bits_truncate(code as u32);
+    let mut generic_flags = GenericPfFlags::empty();
+
+    generic_flags.set(
+        GenericPfFlags::PRESENT,
+        arch_flags.contains(PageFaultError::P),
+    );
+    generic_flags.set(
+        GenericPfFlags::INVOLVED_WRITE,
+        arch_flags.contains(PageFaultError::WR),
+    );
+    generic_flags.set(
+        GenericPfFlags::USER_NOT_SUPERVISOR,
+        arch_flags.contains(PageFaultError::US),
+    );
+    generic_flags.set(
+        GenericPfFlags::INVL,
+        arch_flags.contains(PageFaultError::RSVD),
+    );
+    generic_flags.set(
+        GenericPfFlags::INSTR_NOT_DATA,
+        arch_flags.contains(PageFaultError::ID),
+    );
+
+    #[cfg(target_arch = "x86")]
+    if crate::memory::page_fault_handler(&mut stack.inner, generic_flags, cr2).is_err() {
+        println!("Page fault: {:>08X} {:#?}", cr2.data(), arch_flags);
+        stack.trace();
+        excp_handler(Exception {
+            kind: 14,
+            code,
+            address: cr2.data(),
+        });
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    if crate::memory::page_fault_handler(stack, generic_flags, cr2).is_err() {
+        println!("Page fault: {:>016X} {:#?}", cr2.data(), arch_flags);
+        stack.trace();
+        excp_handler(Exception {
+            kind: 14,
+            code,
+            address: cr2.data(),
+        });
+    }
+});
+
+interrupt_stack!(fpu_fault, |stack| {
+    println!("FPU floating point fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 16,
+        ..Default::default()
+    });
+});
+
+interrupt_error!(alignment_check, |stack, code| {
+    println!("Alignment check fault");
+    stack.trace();
+    excp_handler(Exception {
+        kind: 17,
+        code,
+        ..Default::default()
+    });
+});
+
+interrupt_stack!(machine_check, @paranoid, |stack| {
+    println!("Machine check fault");
+    stack.trace();
+    unsafe {
+        loop {
+            interrupt::disable();
+            interrupt::halt();
+        }
+    }
+});
+
+interrupt_stack!(simd, |stack| {
+    println!("SIMD floating point fault");
+    let mut mxcsr = 0_usize;
+    unsafe { core::arch::asm!("stmxcsr [{}]", in(reg) core::ptr::addr_of_mut!(mxcsr)) };
+    println!("MXCSR {:#0x}", mxcsr);
+    stack.trace();
+    excp_handler(Exception {
+        kind: 19,
+        ..Default::default()
+    });
+});
+
+interrupt_stack!(virtualization, |stack| {
+    println!("Virtualization fault");
+    stack.trace();
+    unsafe {
+        loop {
+            interrupt::disable();
+            interrupt::halt();
+        }
+    }
+});
+
+interrupt_error!(security, |stack, _code| {
+    println!("Security exception");
+    stack.trace();
+    unsafe {
+        loop {
+            interrupt::disable();
+            interrupt::halt();
+        }
+    }
+});

src/arch/x86_shared/interrupt/ipi.rs

@@ -0,0 +1,28 @@
+use crate::{
+    arch::device::local_apic::the_local_apic, context, percpu::PercpuBlock, sync::CleanLockToken,
+};
+
+interrupt!(wakeup, || {
+    unsafe { the_local_apic().eoi() };
+});
+
+interrupt!(tlb, || {
+    PercpuBlock::current().maybe_handle_tlb_shootdown();
+
+    unsafe { the_local_apic().eoi() };
+});
+
+interrupt!(switch, || {
+    unsafe { the_local_apic().eoi() };
+
+    let mut token = unsafe { CleanLockToken::new() };
+    let _ = context::switch(&mut token);
+});
+
+interrupt!(pit, || {
+    unsafe { the_local_apic().eoi() };
+
+    // Switch after a sufficient amount of time since the last switch.
+    let mut token = unsafe { CleanLockToken::new() };
+    context::switch::tick(&mut token);
+});

src/arch/x86_shared/interrupt/irq.rs

@@ -0,0 +1,352 @@
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+use alloc::vec::Vec;
+
+use crate::{
+    arch::{
+        device::{
+            ioapic, local_apic, pic, pit,
+            serial::{COM1, COM2},
+        },
+        ipi::{ipi, IpiKind, IpiTarget},
+    },
+    context::{self, timeout},
+    percpu::PercpuBlock,
+    scheme::{irq::irq_trigger, serio::serio_input},
+    sync::CleanLockToken,
+    time,
+};
+
+#[repr(u8)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum IrqMethod {
+    Pic = 0,
+    Apic = 1,
+}
+
+static SPURIOUS_COUNT_IRQ7: AtomicUsize = AtomicUsize::new(0);
+static SPURIOUS_COUNT_IRQ15: AtomicUsize = AtomicUsize::new(0);
+
+pub fn spurious_count_irq7() -> usize {
+    SPURIOUS_COUNT_IRQ7.load(Ordering::Relaxed)
+}
+pub fn spurious_count_irq15() -> usize {
+    SPURIOUS_COUNT_IRQ15.load(Ordering::Relaxed)
+}
+pub fn spurious_count() -> usize {
+    spurious_count_irq7() + spurious_count_irq15()
+}
+pub fn spurious_irq_resource(_token: &mut CleanLockToken) -> syscall::Result<Vec<u8>> {
+    match irq_method() {
+        IrqMethod::Apic => Ok(Vec::from(&b"(not implemented for APIC yet)"[..])),
+        IrqMethod::Pic => Ok(format!(
+            "{}\tIRQ7\n{}\tIRQ15\n{}\ttotal\n",
+            spurious_count_irq7(),
+            spurious_count_irq15(),
+            spurious_count()
+        )
+        .into_bytes()),
+    }
+}
+
+static IRQ_METHOD: AtomicUsize = AtomicUsize::new(IrqMethod::Pic as usize);
+
+pub fn set_irq_method(method: IrqMethod) {
+    IRQ_METHOD.store(method as usize, core::sync::atomic::Ordering::Release);
+}
+
+fn irq_method() -> IrqMethod {
+    let raw = IRQ_METHOD.load(core::sync::atomic::Ordering::Acquire);
+
+    match raw {
+        0 => IrqMethod::Pic,
+        1 => IrqMethod::Apic,
+        _ => unreachable!(),
+    }
+}
+
+/// Notify the IRQ scheme that an IRQ has been registered. This should mask the IRQ until the
+/// scheme user unmasks it ("acknowledges" it).
+unsafe fn trigger(irq: u8) {
+    unsafe {
+        match irq_method() {
+            IrqMethod::Pic => {
+                if irq < 16 {
+                    pic_mask(irq)
+                }
+            }
+            IrqMethod::Apic => ioapic_mask(irq),
+        }
+        let mut token = CleanLockToken::new();
+        irq_trigger(irq, &mut token);
+    }
+}
+
+/// Unmask the IRQ. This is called from the IRQ scheme, which does this when a user process has
+/// processed the IRQ.
+pub unsafe fn acknowledge(irq: usize) {
+    unsafe {
+        match irq_method() {
+            IrqMethod::Pic => {
+                if irq < 16 {
+                    pic_unmask(irq)
+                }
+            }
+            IrqMethod::Apic => ioapic_unmask(irq),
+        }
+    }
+}
+
+/// Sends an end-of-interrupt, so that the interrupt controller can go on to the next one.
+pub unsafe fn eoi(irq: u8) {
+    unsafe {
+        PercpuBlock::current().stats.add_irq(irq);
+
+        match irq_method() {
+            IrqMethod::Pic => {
+                if irq < 16 {
+                    pic_eoi(irq)
+                }
+            }
+            IrqMethod::Apic => lapic_eoi(),
+        }
+    }
+}
+
+unsafe fn pic_mask(irq: u8) {
+    unsafe {
+        debug_assert!(irq < 16);
+
+        if irq >= 8 {
+            pic::slave().mask_set(irq - 8);
+        } else {
+            pic::master().mask_set(irq);
+        }
+    }
+}
+
+unsafe fn ioapic_mask(irq: u8) {
+    unsafe {
+        ioapic::mask(irq);
+    }
+}
+
+unsafe fn pic_eoi(irq: u8) {
+    unsafe {
+        debug_assert!(irq < 16);
+
+        if irq >= 8 {
+            pic::master().ack();
+            pic::slave().ack();
+        } else {
+            pic::master().ack();
+        }
+    }
+}
+
+unsafe fn lapic_eoi() {
+    unsafe { local_apic::the_local_apic().eoi() }
+}
+
+unsafe fn pic_unmask(irq: usize) {
+    unsafe {
+        debug_assert!(irq < 16);
+
+        if irq >= 8 {
+            pic::slave().mask_clear(irq as u8 - 8);
+        } else {
+            pic::master().mask_clear(irq as u8);
+        }
+    }
+}
+
+unsafe fn ioapic_unmask(irq: usize) {
+    unsafe {
+        ioapic::unmask(irq as u8);
+    }
+}
+
+interrupt_stack!(pit_stack, |_stack| {
+    // Saves CPU time by not sending IRQ event irq_trigger(0);
+
+    let mut token = unsafe { CleanLockToken::new() };
+    {
+        *time::OFFSET.write(token.token()) += pit::RATE;
+    }
+
+    unsafe { eoi(0) };
+
+    // Wake up other CPUs
+    ipi(IpiKind::Pit, IpiTarget::Other);
+
+    // Any better way of doing this?
+    timeout::trigger(&mut token);
+
+    // Switch after a sufficient amount of time since the last switch.
+    context::switch::tick(&mut token);
+});
+
+interrupt!(keyboard, || {
+    let data: u8;
+    unsafe { core::arch::asm!("in al, 0x60", out("al") data) };
+
+    unsafe { eoi(1) };
+
+    let mut token = unsafe { CleanLockToken::new() };
+    serio_input(0, data, &mut token);
+});
+
+interrupt!(cascade, || {
+    // No need to do any operations on cascade
+    unsafe { eoi(2) };
+});
+
+interrupt!(com2, || {
+    let mut token = unsafe { CleanLockToken::new() };
+    COM2.lock().receive(&mut token);
+    unsafe { eoi(3) };
+});
+
+interrupt!(com1, || {
+    let mut token = unsafe { CleanLockToken::new() };
+    COM1.lock().receive(&mut token);
+    unsafe { eoi(4) };
+});
+
+interrupt!(lpt2, || {
+    unsafe {
+        trigger(5);
+        eoi(5);
+    }
+});
+
+interrupt!(floppy, || {
+    unsafe {
+        trigger(6);
+        eoi(6);
+    }
+});
+
+interrupt!(lpt1, || {
+    unsafe {
+        if irq_method() == IrqMethod::Pic && pic::master().isr() & (1 << 7) == 0 {
+            // the IRQ was spurious, ignore it but increment a counter.
+            SPURIOUS_COUNT_IRQ7.fetch_add(1, Ordering::Relaxed);
+            return;
+        }
+        trigger(7);
+        eoi(7);
+    }
+});
+
+interrupt!(rtc, || {
+    unsafe {
+        trigger(8);
+        eoi(8);
+    }
+});
+
+interrupt!(pci1, || {
+    unsafe {
+        trigger(9);
+        eoi(9);
+    }
+});
+
+interrupt!(pci2, || {
+    unsafe {
+        trigger(10);
+        eoi(10);
+    }
+});
+
+interrupt!(pci3, || {
+    unsafe {
+        trigger(11);
+        eoi(11);
+    }
+});
+
+interrupt!(mouse, || {
+    let data: u8;
+    unsafe { core::arch::asm!("in al, 0x60", out("al") data) };
+
+    unsafe { eoi(12) };
+
+    let mut token = unsafe { CleanLockToken::new() };
+    serio_input(1, data, &mut token);
+});
+
+interrupt!(fpu, || {
+    unsafe {
+        trigger(13);
+        eoi(13);
+    }
+});
+
+interrupt!(ata1, || {
+    unsafe {
+        trigger(14);
+        eoi(14);
+    }
+});
+
+interrupt!(ata2, || {
+    unsafe {
+        if irq_method() == IrqMethod::Pic && pic::slave().isr() & (1 << 7) == 0 {
+            SPURIOUS_COUNT_IRQ15.fetch_add(1, Ordering::Relaxed);
+            pic::master().ack();
+            return;
+        }
+        trigger(15);
+        eoi(15);
+    }
+});
+
+interrupt!(lapic_timer, || {
+    println!("Local apic timer interrupt");
+    unsafe { lapic_eoi() };
+});
+#[cfg(feature = "profiling")]
+interrupt!(aux_timer, || {
+    unsafe { lapic_eoi() };
+    crate::ipi::ipi(IpiKind::Profile, IpiTarget::Other);
+});
+
+interrupt!(lapic_error, || {
+    error!("Local apic internal error: ESR={:#0x}", unsafe {
+        local_apic::the_local_apic().esr()
+    });
+    unsafe { lapic_eoi() };
+});
+
+interrupt_error!(generic_irq, |_stack, code| {
+    let mut token = unsafe { CleanLockToken::new() };
+
+    // The reason why 128 is subtracted and added from the code, is that PUSH imm8 sign-extends the
+    // value, and the longer PUSH imm32 would make the generic_interrupts table twice as large
+    // (containing lots of useless NOPs).
+    irq_trigger((code as i32).wrapping_add(128) as u8, &mut token);
+
+    unsafe { lapic_eoi() };
+});
+
+core::arch::global_asm!("
+    .globl __generic_interrupts_start
+    .globl __generic_interrupts_end
+    .p2align 3
+__generic_interrupts_start:
+    n = 0
+    .rept 224
+    push (n - 128)
+    jmp {}
+    .p2align 3
+    n = n + 1
+    .endr
+__generic_interrupts_end:
+", sym generic_irq);
+
+unsafe extern "C" {
+    pub fn __generic_interrupts_start();
+    pub fn __generic_interrupts_end();
+}

src/arch/x86_shared/interrupt/mod.rs

@@ -0,0 +1,44 @@
+//! Interrupt instructions
+
+pub mod exception;
+pub mod ipi;
+pub mod irq;
+pub mod trace;
+
+pub use super::idt::{available_irqs_iter, is_reserved, set_reserved};
+
+/// Clear interrupts
+#[inline(always)]
+pub unsafe fn disable() {
+    unsafe {
+        core::arch::asm!("cli", options(nomem, nostack));
+    }
+}
+
+/// Set interrupts and halt
+/// This will atomically wait for the next interrupt
+/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_halt() {
+    unsafe {
+        core::arch::asm!("sti; hlt", options(nomem, nostack));
+    }
+}
+
+/// Set interrupts and nop
+/// This will enable interrupts and allow the IF flag to be processed
+/// Simply enabling interrupts does not guarantee that they will trigger, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_nop() {
+    unsafe {
+        core::arch::asm!("sti; nop", options(nomem, nostack));
+    }
+}
+
+/// Halt instruction
+#[inline(always)]
+pub unsafe fn halt() {
+    unsafe {
+        core::arch::asm!("hlt", options(nomem, nostack));
+    }
+}