From cba02a26fa34b3e758a62781d75d49f4dbd80b41 Mon Sep 17 00:00:00 2001 From: Andrey Turkin Date: Tue, 22 Oct 2024 16:53:09 +0300 Subject: [PATCH] Rework irqchip to support risc-v irqs, and add risc-v irq chips handling --- src/arch/aarch64/device/generic_timer.rs | 56 ++-- src/arch/aarch64/device/irqchip/gic.rs | 39 ++- src/arch/aarch64/device/irqchip/gicv3.rs | 39 ++- .../aarch64/device/irqchip/irq_bcm2835.rs | 79 ++--- .../aarch64/device/irqchip/irq_bcm2836.rs | 29 +- src/arch/aarch64/device/irqchip/mod.rs | 21 +- src/arch/aarch64/device/mod.rs | 39 ++- src/arch/aarch64/device/serial.rs | 57 ++-- src/arch/aarch64/interrupt/irq.rs | 33 +-- src/arch/aarch64/interrupt/mod.rs | 2 - src/arch/aarch64/start.rs | 19 +- src/arch/riscv64/device/irqchip/clint.rs | 42 +++ src/arch/riscv64/device/irqchip/clint_sbi.rs | 144 +++++++++ src/arch/riscv64/device/irqchip/hlic.rs | 159 ++++++++++ src/arch/riscv64/device/irqchip/mod.rs | 42 ++- src/arch/riscv64/device/irqchip/plic.rs | 196 ++++++++++++ src/arch/riscv64/device/mod.rs | 14 +- src/arch/riscv64/device/serial.rs | 35 ++- src/arch/riscv64/interrupt/exception.rs | 20 +- src/arch/riscv64/interrupt/mod.rs | 3 - src/arch/riscv64/rmm.rs | 1 - src/arch/riscv64/start.rs | 5 +- src/arch/x86_64/paging/mod.rs | 2 + src/context/arch/riscv64.rs | 4 - src/dtb/irqchip.rs | 279 ++++++++++++------ src/dtb/mod.rs | 7 +- 26 files changed, 1011 insertions(+), 355 deletions(-) create mode 100644 src/arch/riscv64/device/irqchip/clint.rs create mode 100644 src/arch/riscv64/device/irqchip/clint_sbi.rs create mode 100644 src/arch/riscv64/device/irqchip/hlic.rs create mode 100644 src/arch/riscv64/device/irqchip/plic.rs diff --git a/src/arch/aarch64/device/generic_timer.rs b/src/arch/aarch64/device/generic_timer.rs index 517cc06c8f..2250d91d0c 100644 --- a/src/arch/aarch64/device/generic_timer.rs +++ b/src/arch/aarch64/device/generic_timer.rs @@ -1,18 +1,15 @@ use alloc::boxed::Box; -use log::info; +use log::{error, info}; +use super::ic_for_chip; use crate::{ context, context::timeout, device::cpu::registers::control_regs, - dtb::{ - irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, - DTB_BINARY, - }, + dtb::irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, interrupt::irq::trigger, time, }; -use alloc::vec::Vec; use byteorder::{ByteOrder, BE}; use fdt::Fdt; @@ -24,42 +21,33 @@ bitflags! { } } -pub unsafe fn init() { +pub unsafe fn init(fdt: &Fdt) { let mut timer = GenericTimer { clk_freq: 0, reload_count: 0, }; timer.init(); - let data = DTB_BINARY.get().unwrap(); - let fdt = Fdt::new(data).unwrap(); if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) { let interrupts = node.property("interrupts").unwrap(); - let mut intr_data = Vec::new(); - for chunk in interrupts.value.chunks(4) { - let val = BE::read_u32(chunk); - intr_data.push(val); - } - let mut ic_idx = IRQ_CHIP.irq_chip_list.root_idx; - if let Some(interrupt_parent) = node.property("interrupt-parent") { - let phandle = interrupt_parent.as_usize().unwrap() as u32; - let mut i = 0; - while i < IRQ_CHIP.irq_chip_list.chips.len() { - let item = &IRQ_CHIP.irq_chip_list.chips[i]; - if item.phandle == phandle { - ic_idx = i; - break; - } - i += 1; - } - } - //PHYS_NONSECURE_PPI only - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_xlate(&intr_data, 1) + let irq = interrupts + .value + .array_chunks::<4>() + .map(|f| BE::read_u32(f)) + .skip(3) + .next_chunk::<3>() .unwrap(); - info!("generic_timer virq = {}", virq); - register_irq(virq as u32, Box::new(timer)); - IRQ_CHIP.irq_enable(virq as u32); + if let Some(ic_idx) = ic_for_chip(&fdt, &node) { + //PHYS_NONSECURE_PPI only + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(&irq) + .unwrap(); + info!("generic_timer virq = {}", virq); + register_irq(virq as u32, Box::new(timer)); + IRQ_CHIP.irq_enable(virq as u32); + } else { + error!("Failed to find irq parent for generic timer"); + } } } diff --git a/src/arch/aarch64/device/irqchip/gic.rs b/src/arch/aarch64/device/irqchip/gic.rs index cf6bf26c47..b00150bad4 100644 --- a/src/arch/aarch64/device/irqchip/gic.rs +++ b/src/arch/aarch64/device/irqchip/gic.rs @@ -1,7 +1,7 @@ +use super::InterruptController; +use crate::dtb::irqchip::{InterruptHandler, IrqDesc}; use core::ptr::{read_volatile, write_volatile}; use fdt::{node::FdtNode, Fdt}; - -use crate::dtb::irqchip::{InterruptController, IrqDesc}; use log::info; use syscall::{ error::{Error, EINVAL}, @@ -75,6 +75,10 @@ impl GenericInterruptController { } } +impl InterruptHandler for GenericInterruptController { + fn irq_handler(&mut self, _irq: u32) {} +} + impl InterruptController for GenericInterruptController { fn irq_init( &mut self, @@ -82,7 +86,7 @@ impl InterruptController for GenericInterruptController { irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize, - ) -> Result> { + ) -> Result<()> { let (dist_addr, _dist_size, cpu_addr, _cpu_size) = match GenericInterruptController::parse(fdt) { Ok(regs) => regs, @@ -117,7 +121,7 @@ impl InterruptController for GenericInterruptController { info!("gic irq_range = ({}, {})", idx, idx + cnt); self.irq_range = (idx, idx + cnt); *irq_idx = idx + cnt; - Ok(None) + Ok(()) } fn irq_ack(&mut self) -> u32 { unsafe { self.gic_cpu_if.irq_ack() } @@ -131,31 +135,22 @@ impl InterruptController for GenericInterruptController { fn irq_disable(&mut self, irq_num: u32) { unsafe { self.gic_dist_if.irq_disable(irq_num) } } - fn irq_xlate(&mut self, irq_data: &[u32], idx: usize) -> Result { - let mut i = 0; - for chunk in irq_data.chunks(3) { - if i == idx { - let mut off = match chunk[0] { - 0 => chunk[1] as usize + 32, //SPI - 1 => chunk[1] as usize + 16, //PPI, - _ => return Err(Error::new(EINVAL)), - }; - off += self.irq_range.0; - return Ok(off); - } - i += 1; - } - Err(Error::new(EINVAL)) + fn irq_xlate(&self, irq_data: &[u32; 3]) -> Result { + let mut off = match irq_data[0] { + 0 => irq_data[1] as usize + 32, //SPI + 1 => irq_data[1] as usize + 16, //PPI, + _ => return Err(Error::new(EINVAL)), + }; + off += self.irq_range.0; + return Ok(off); } - fn irq_to_virq(&mut self, hwirq: u32) -> Option { + fn irq_to_virq(&self, hwirq: u32) -> Option { if hwirq >= self.gic_dist_if.nirqs { None } else { Some(self.irq_range.0 + hwirq as usize) } } - - fn irq_handler(&mut self, _irq: u32) {} } #[derive(Debug)] diff --git a/src/arch/aarch64/device/irqchip/gicv3.rs b/src/arch/aarch64/device/irqchip/gicv3.rs index cde9d2085f..245220e204 100644 --- a/src/arch/aarch64/device/irqchip/gicv3.rs +++ b/src/arch/aarch64/device/irqchip/gicv3.rs @@ -2,8 +2,8 @@ use alloc::vec::Vec; use core::arch::asm; use fdt::{node::NodeProperty, Fdt}; -use super::gic::GicDistIf; -use crate::dtb::irqchip::{InterruptController, IrqDesc}; +use super::{gic::GicDistIf, InterruptController}; +use crate::dtb::irqchip::{InterruptHandler, IrqDesc}; use syscall::{ error::{Error, EINVAL}, Result, @@ -71,6 +71,10 @@ impl GicV3 { } } +impl InterruptHandler for GicV3 { + fn irq_handler(&mut self, _irq: u32) {} +} + impl InterruptController for GicV3 { fn irq_init( &mut self, @@ -78,7 +82,7 @@ impl InterruptController for GicV3 { irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize, - ) -> Result> { + ) -> Result<()> { self.parse(fdt)?; log::info!("{:X?}", self); @@ -104,7 +108,7 @@ impl InterruptController for GicV3 { log::info!("gic irq_range = ({}, {})", idx, idx + cnt); self.irq_range = (idx, idx + cnt); *irq_idx = idx + cnt; - Ok(None) + Ok(()) } fn irq_ack(&mut self) -> u32 { let irq_num = unsafe { self.gic_cpu_if.irq_ack() }; @@ -119,31 +123,22 @@ impl InterruptController for GicV3 { fn irq_disable(&mut self, irq_num: u32) { unsafe { self.gic_dist_if.irq_disable(irq_num) } } - fn irq_xlate(&mut self, irq_data: &[u32], idx: usize) -> Result { - let mut i = 0; - for chunk in irq_data.chunks(3) { - if i == idx { - let mut off = match chunk[0] { - 0 => chunk[1] as usize + 32, //SPI - 1 => chunk[1] as usize + 16, //PPI, - _ => return Err(Error::new(EINVAL)), - }; - off += self.irq_range.0; - return Ok(off); - } - i += 1; - } - Err(Error::new(EINVAL)) + fn irq_xlate(&self, irq_data: &[u32; 3]) -> Result { + let mut off = match irq_data[0] { + 0 => irq_data[1] as usize + 32, //SPI + 1 => irq_data[1] as usize + 16, //PPI, + _ => return Err(Error::new(EINVAL)), + }; + off += self.irq_range.0; + return Ok(off); } - fn irq_to_virq(&mut self, hwirq: u32) -> Option { + fn irq_to_virq(&self, hwirq: u32) -> Option { if hwirq >= self.gic_dist_if.nirqs { None } else { Some(self.irq_range.0 + hwirq as usize) } } - - fn irq_handler(&mut self, _irq: u32) {} } #[derive(Debug)] diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs index 27ef2c3b7c..462996ef65 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2835.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2835.rs @@ -1,12 +1,10 @@ -use alloc::vec::Vec; -use core::ptr::{read_volatile, write_volatile}; - use byteorder::{ByteOrder, BE}; +use core::ptr::{read_volatile, write_volatile}; use fdt::{node::FdtNode, Fdt}; - use log::{debug, error, info}; -use crate::dtb::irqchip::{InterruptController, InterruptHandler, IrqDesc, IRQ_CHIP}; +use super::InterruptController; +use crate::dtb::irqchip::{InterruptHandler, IrqDesc, IRQ_CHIP}; use syscall::{ error::{Error, EINVAL}, Result, @@ -81,25 +79,17 @@ impl Bcm2835ArmInterruptController { if let Some(interrupt_parent) = node.property("interrupt-parent") { let phandle = interrupt_parent.as_usize().unwrap() as u32; let interrupts = node.property("interrupts").unwrap(); - let mut intr_data = Vec::new(); - for chunk in interrupts.value.chunks(4) { - let val = BE::read_u32(chunk); - intr_data.push(val); - } - let mut ic_idx = IRQ_CHIP.irq_chip_list.root_idx; - let mut i = 0; - while i < IRQ_CHIP.irq_chip_list.chips.len() { - let item = &IRQ_CHIP.irq_chip_list.chips[i]; - if item.phandle == phandle { - ic_idx = i; - break; - } - i += 1; - } + let irq = interrupts + .value + .array_chunks::<4>() + .map(|f| BE::read_u32(f)) + .next_chunk::<3>() + .unwrap(); + let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap(); //PHYS_NONSECURE_PPI only let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] .ic - .irq_xlate(&intr_data, 0) + .irq_xlate(&irq) .unwrap(); info!("bcm2835arm_ctrl virq = {}", virq); ret_virq = Some(virq); @@ -134,8 +124,8 @@ impl InterruptController for Bcm2835ArmInterruptController { irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize, - ) -> Result> { - let (base, _size, virq) = match Bcm2835ArmInterruptController::parse(fdt) { + ) -> Result<()> { + let (base, _size, _virq) = match Bcm2835ArmInterruptController::parse(fdt) { Ok((a, b, c)) => (a, b, c), Err(_) => return Err(Error::new(EINVAL)), }; @@ -160,7 +150,7 @@ impl InterruptController for Bcm2835ArmInterruptController { *irq_idx = idx + cnt; } - Ok(virq) + Ok(()) } fn irq_ack(&mut self) -> u32 { @@ -259,46 +249,23 @@ impl InterruptController for Bcm2835ArmInterruptController { _ => return, } } - fn irq_xlate(&mut self, irq_data: &[u32], idx: usize) -> Result { + fn irq_xlate(&self, irq_data: &[u32; 3]) -> Result { //assert interrupt-cells == 0x2 - let mut i = 0; - //assert interrupt-cells == 0x2 - for chunk in irq_data.chunks(2) { - if i == idx { - let bank = chunk[0] as usize; - let irq = chunk[1] as usize; - //TODO: check bank && irq - let hwirq = bank << 5 | irq; - let off = hwirq + self.irq_range.0; - return Ok(off); - } - i += 1; - } - Err(Error::new(EINVAL)) + let bank = irq_data[0] as usize; + let irq = irq_data[1] as usize; + //TODO: check bank && irq + let hwirq = bank << 5 | irq; + let off = hwirq + self.irq_range.0; + return Ok(off); } - fn irq_to_virq(&mut self, hwirq: u32) -> Option { + + fn irq_to_virq(&self, hwirq: u32) -> Option { if hwirq > 95 { None } else { Some(self.irq_range.0 + hwirq as usize) } } - - fn irq_handler(&mut self, _irq: u32) { - unsafe { - let irq = self.irq_ack(); - if let Some(virq) = self.irq_to_virq(irq) - && virq < 1024 - { - if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler { - handler.irq_handler(virq as u32); - } - } else { - error!("unexpected irq num {}", irq); - } - self.irq_eoi(irq); - } - } } impl InterruptHandler for Bcm2835ArmInterruptController { diff --git a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs index a16f5e7e74..866cffc759 100644 --- a/src/arch/aarch64/device/irqchip/irq_bcm2836.rs +++ b/src/arch/aarch64/device/irqchip/irq_bcm2836.rs @@ -1,10 +1,10 @@ +use super::InterruptController; +use crate::dtb::irqchip::{InterruptHandler, IrqDesc}; use core::{ arch::asm, ptr::{read_volatile, write_volatile}, }; use fdt::{node::FdtNode, Fdt}; - -use crate::dtb::irqchip::{InterruptController, IrqDesc}; use log::{debug, info}; use syscall::{ error::{Error, EINVAL}, @@ -102,6 +102,10 @@ impl Bcm2836ArmInterruptController { } } +impl InterruptHandler for Bcm2836ArmInterruptController { + fn irq_handler(&mut self, _irq: u32) {} +} + impl InterruptController for Bcm2836ArmInterruptController { fn irq_init( &mut self, @@ -109,7 +113,7 @@ impl InterruptController for Bcm2836ArmInterruptController { irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize, - ) -> Result> { + ) -> Result<()> { let (base, _size) = match Bcm2836ArmInterruptController::parse(fdt) { Ok((a, b)) => (a, b), Err(_) => return Err(Error::new(EINVAL)), @@ -138,7 +142,7 @@ impl InterruptController for Bcm2836ArmInterruptController { *irq_idx = idx + cnt; } - Ok(None) + Ok(()) } fn irq_ack(&mut self) -> u32 { @@ -190,25 +194,16 @@ impl InterruptController for Bcm2836ArmInterruptController { } } } - fn irq_xlate(&mut self, irq_data: &[u32], idx: usize) -> Result { - let mut i = 0; + fn irq_xlate(&self, irq_data: &[u32; 3]) -> Result { //assert interrupt-cells == 0x2 - for chunk in irq_data.chunks(2) { - if i == idx { - let off = chunk[0] as usize + self.irq_range.0; - return Ok(off); - } - i += 1; - } - Err(Error::new(EINVAL)) + let off = irq_data[0] as usize + self.irq_range.0; + return Ok(off); } - fn irq_to_virq(&mut self, hwirq: u32) -> Option { + fn irq_to_virq(&self, hwirq: u32) -> Option { if hwirq > LOCAL_IRQ_LAST { None } else { Some(self.irq_range.0 + hwirq as usize) } } - - fn irq_handler(&mut self, _irq: u32) {} } diff --git a/src/arch/aarch64/device/irqchip/mod.rs b/src/arch/aarch64/device/irqchip/mod.rs index 256a0d4d58..1795105ded 100644 --- a/src/arch/aarch64/device/irqchip/mod.rs +++ b/src/arch/aarch64/device/irqchip/mod.rs @@ -1,12 +1,13 @@ -use crate::dtb::irqchip::InterruptController; +use crate::dtb::irqchip::{InterruptController, IRQ_CHIP}; use alloc::boxed::Box; +use fdt::{node::FdtNode, Fdt}; mod gic; mod gicv3; mod irq_bcm2835; mod irq_bcm2836; -pub fn new_irqchip(ic_str: &str) -> Option> { +pub(crate) fn new_irqchip(ic_str: &str) -> Option> { if ic_str.contains("arm,gic-v3") { Some(Box::new(gicv3::GicV3::new())) } else if ic_str.contains("arm,cortex-a15-gic") { @@ -20,3 +21,19 @@ pub fn new_irqchip(ic_str: &str) -> Option> { None } } + +pub(crate) fn ic_for_chip(fdt: &Fdt, node: &FdtNode) -> Option { + if let Some(_) = node.property("interrupts-extended") { + log::error!("multi-parented device not supported"); + None + } else if let Some(irqc_phandle) = node + .property("interrupt-parent") + .or(fdt.root().property("interrupt-parent")) + .and_then(|f| f.as_usize()) + { + unsafe { IRQ_CHIP.phandle_to_ic_idx(irqc_phandle as u32) } + } else { + log::error!("no irq parent found"); + None + } +} diff --git a/src/arch/aarch64/device/mod.rs b/src/arch/aarch64/device/mod.rs index 00abdf6167..ac38fa0d11 100644 --- a/src/arch/aarch64/device/mod.rs +++ b/src/arch/aarch64/device/mod.rs @@ -1,4 +1,6 @@ -use crate::{dtb, dtb::DTB_BINARY, info}; +use crate::info; +use core::sync::atomic::{AtomicUsize, Ordering}; +use fdt::Fdt; pub mod cpu; pub mod generic_timer; @@ -7,18 +9,35 @@ pub mod rtc; pub mod serial; pub mod uart_pl011; -pub unsafe fn init() { - info!("IRQCHIP INIT"); - let data = DTB_BINARY.get().unwrap(); - let fdt = fdt::Fdt::new(data).unwrap(); - dtb::irqchip::init(&fdt); - info!("GIT INIT"); - generic_timer::init(); +use crate::dtb::irqchip::IRQ_CHIP; +use irqchip::ic_for_chip; + +pub static ROOT_IC_IDX: AtomicUsize = AtomicUsize::new(0); + +unsafe fn init_root_ic(fdt: &Fdt) { + let root_irqc_phandle = fdt + .root() + .property("interrupt-parent") + .unwrap() + .as_usize() + .unwrap(); + let ic_idx = IRQ_CHIP + .phandle_to_ic_idx(root_irqc_phandle as u32) + .unwrap(); + ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed); } -pub unsafe fn init_noncore() { +pub unsafe fn init(fdt: &Fdt) { + info!("IRQCHIP INIT"); + crate::dtb::irqchip::init(&fdt); + init_root_ic(&fdt); + info!("GIT INIT"); + generic_timer::init(fdt); +} + +pub unsafe fn init_noncore(fdt: &Fdt) { info!("SERIAL INIT"); - serial::init(); + serial::init(fdt); info!("RTC INIT"); rtc::init(); } diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs index 329d1bee57..a9f6e9584e 100644 --- a/src/arch/aarch64/device/serial.rs +++ b/src/arch/aarch64/device/serial.rs @@ -3,15 +3,16 @@ use spin::Mutex; use crate::{device::uart_pl011::SerialPort, interrupt::irq::trigger}; -use crate::dtb::{ - diag_uart_range, - irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, - DTB_BINARY, +use crate::{ + arch::device::irqchip::ic_for_chip, + dtb::{ + diag_uart_range, + irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, + }, }; -use alloc::vec::Vec; use byteorder::{ByteOrder, BE}; use fdt::Fdt; -use log::info; +use log::{error, info}; pub static COM1: Mutex> = Mutex::new(None); @@ -45,36 +46,26 @@ pub unsafe fn init_early(dtb: &Fdt) { } } -pub unsafe fn init() { - let data = DTB_BINARY.get().unwrap(); - let fdt = Fdt::new(data).unwrap(); +pub unsafe fn init(fdt: &Fdt) { if let Some(node) = fdt.find_compatible(&["arm,pl011"]) { let interrupts = node.property("interrupts").unwrap(); - let mut intr_data = Vec::new(); - for chunk in interrupts.value.chunks(4) { - let val = BE::read_u32(chunk); - intr_data.push(val); - } - let mut ic_idx = IRQ_CHIP.irq_chip_list.root_idx; - if let Some(interrupt_parent) = node.property("interrupt-parent") { - let phandle = interrupt_parent.as_usize().unwrap() as u32; - let mut i = 0; - while i < IRQ_CHIP.irq_chip_list.chips.len() { - let item = &IRQ_CHIP.irq_chip_list.chips[i]; - if item.phandle == phandle { - ic_idx = i; - break; - } - i += 1; - } - } - let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_xlate(&intr_data, 0) + let irq = interrupts + .value + .array_chunks::<4>() + .map(|f| BE::read_u32(f)) + .next_chunk::<3>() .unwrap(); - info!("serial_port virq = {}", virq); - register_irq(virq as u32, Box::new(Com1Irq {})); - IRQ_CHIP.irq_enable(virq as u32); + if let Some(ic_idx) = ic_for_chip(&fdt, &node) { + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(&irq) + .unwrap(); + info!("serial_port virq = {}", virq); + register_irq(virq as u32, Box::new(Com1Irq {})); + IRQ_CHIP.irq_enable(virq as u32); + } else { + error!("serial port irq parent not found"); + } } if let Some(ref mut serial_port) = *COM1.lock() { serial_port.enable_irq(); diff --git a/src/arch/aarch64/interrupt/irq.rs b/src/arch/aarch64/interrupt/irq.rs index dab968a6b4..f7a31a6e75 100644 --- a/src/arch/aarch64/interrupt/irq.rs +++ b/src/arch/aarch64/interrupt/irq.rs @@ -1,34 +1,29 @@ -use crate::dtb::irqchip::IRQ_CHIP; +use crate::{arch::device::ROOT_IC_IDX, dtb::irqchip::IRQ_CHIP}; +use core::sync::atomic::Ordering; + +unsafe fn irq_ack() -> (u32, Option) { + let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic; + let irq = ic.irq_ack(); + (irq, ic.irq_to_virq(irq)) +} exception_stack!(irq_at_el0, |_stack| { - let irq = IRQ_CHIP.irq_ack(); - if let Some(virq) = IRQ_CHIP.irq_to_virq(irq) + let (irq, virq) = irq_ack(); + if let Some(virq) = virq && virq < 1024 { - if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler { - handler.irq_handler(virq as u32); - } else if let Some(ic_idx) = IRQ_CHIP.irq_desc[virq].basic.child_ic_idx { - IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_handler(virq as u32); - } + IRQ_CHIP.trigger_virq(virq as u32); } else { println!("unexpected irq num {}", irq); } }); exception_stack!(irq_at_el1, |_stack| { - let irq = IRQ_CHIP.irq_ack(); - if let Some(virq) = IRQ_CHIP.irq_to_virq(irq) + let (irq, virq) = irq_ack(); + if let Some(virq) = virq && virq < 1024 { - if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler { - handler.irq_handler(virq as u32); - } else if let Some(ic_idx) = IRQ_CHIP.irq_desc[virq].basic.child_ic_idx { - IRQ_CHIP.irq_chip_list.chips[ic_idx] - .ic - .irq_handler(virq as u32); - } + IRQ_CHIP.trigger_virq(virq as u32); } else { println!("unexpected irq num {}", irq); } diff --git a/src/arch/aarch64/interrupt/mod.rs b/src/arch/aarch64/interrupt/mod.rs index f3bf58b5e8..64acba3290 100644 --- a/src/arch/aarch64/interrupt/mod.rs +++ b/src/arch/aarch64/interrupt/mod.rs @@ -10,8 +10,6 @@ pub mod irq; pub mod syscall; pub mod trace; -use crate::cpu_set::LogicalCpuId; - pub use self::handler::InterruptStack; /// Clear interrupts diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs index 98acef991c..290fa6eb9d 100644 --- a/src/arch/aarch64/start.rs +++ b/src/arch/aarch64/start.rs @@ -69,11 +69,10 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { } else { None }; + let dtb = dtb.unwrap(); - if let Some(dt) = &dtb { - // Try to find serial port prior to logging - device::serial::init_early(dt); - } + // Try to find serial port prior to logging + device::serial::init_early(&dtb); // Convert env to slice let env = slice::from_raw_parts( @@ -138,12 +137,10 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { tmp = out(reg) _, ); - if let Some(dt) = &dtb { - //in uefi boot mode, ignore memory node, just read the device memory range - //register_memory_ranges(dt); + //in uefi boot mode, ignore memory node, just read the device memory range + //register_memory_ranges(&dtb); - register_dev_memory_ranges(dt); - } + register_dev_memory_ranges(&dtb); register_bootloader_areas(args.areas_base, args.areas_size); @@ -202,10 +199,10 @@ pub unsafe extern "C" fn kstart(args_ptr: *const KernelArgs) -> ! { dtb::init(Some((crate::PHYS_OFFSET + args.dtb_base, args.dtb_size))); // Initialize devices - device::init(); + device::init(&dtb); // Initialize all of the non-core devices not otherwise needed to complete initialization - device::init_noncore(); + device::init_noncore(&dtb); BSP_READY.store(true, Ordering::SeqCst); diff --git a/src/arch/riscv64/device/irqchip/clint.rs b/src/arch/riscv64/device/irqchip/clint.rs new file mode 100644 index 0000000000..560d86df0b --- /dev/null +++ b/src/arch/riscv64/device/irqchip/clint.rs @@ -0,0 +1,42 @@ +use spin::Mutex; +use syscall::{Io, Mmio}; +use crate::context::switch::tick; + +#[repr(packed(4))] +#[repr(C)] +struct ClintRegs { + /// per-hart MSIP registers + /// bit 0: trigger IPI for the hart + msip: [Mmio; 4095], // +0000 -- 3fff + _rsrv1: u32, + /// per-hart MTIMECMP registers + /// timer interrupt trigger threshold + mtimecmp: [Mmio; 4095], // +4000 - bff7 + mtime: Mmio // current time +} + +pub struct Clint { + regs: &'static mut ClintRegs, + freq: u64 +} + +pub static CLINT: Mutex> = Mutex::new(None); + +impl Clint { + pub fn new(addr: *mut u8, size: usize, freq: usize) -> Self { + assert!(size >= core::mem::size_of::()); + Self { + regs: unsafe { (addr as *mut ClintRegs).as_mut().unwrap() }, + freq: freq as u64 + } + } + + pub fn init(self: &mut Self) { + (*self.regs).mtimecmp[0].write((*self.regs).mtime.read() + self.freq / 100); + } + + pub fn timer_irq(self: &mut Self, hart: usize) { + (*self.regs).mtimecmp[hart].write((*self.regs).mtimecmp[hart].read() + self.freq / 100); + tick(); + } +} diff --git a/src/arch/riscv64/device/irqchip/clint_sbi.rs b/src/arch/riscv64/device/irqchip/clint_sbi.rs new file mode 100644 index 0000000000..4fecd3af60 --- /dev/null +++ b/src/arch/riscv64/device/irqchip/clint_sbi.rs @@ -0,0 +1,144 @@ +use crate::{ + arch::riscv64::sbi::SBI, + dtb::irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, +}; +use alloc::{boxed::Box, vec::Vec}; +use byteorder::{ByteOrder, BE}; +use core::{arch::asm, cmp::max}; +use fdt::node::FdtNode; +use spin::Mutex; + +// This is a Core-Local Interruptor (CLINT). A single device directly routed into each HLIC +// It is responsible for local timer and IPI interrupts +// An example DTS: +// /soc/ +// clint@2000000/ +// interrupts-extended = <&hlic0 3>, <&hlic0 7>, <&hlic1 3>, <&hlic1 7>, +// <&hlic2 3>, <&hlic2 7>, <&hlic3 3>, <&hlic3 7>; +// reg = <0x200000000 0x10000>; +// compatible = "sifive,clint0", "riscv,clint0"; + +pub struct Clint { + freq: u64, + next_event: Vec, +} + +pub static CLINT: Mutex> = Mutex::new(None); +const TICKS_PER_SECOND: u64 = 100; +const IRQ_IPI: usize = 0; +const IRQ_TIMER: usize = 1; + +struct ClintConnector { + hart_id: usize, + irq: usize, +} + +impl InterruptHandler for ClintConnector { + fn irq_handler(&mut self, _irq: u32) { + CLINT + .lock() + .as_mut() + .unwrap() + .irq_handler(self.hart_id, self.irq); + } +} + +fn map_interrupt(irq: u32) -> u32 { + match irq { + 3 => 1, // map M-mode IPI to S-mode IPI + 7 => 5, // map M-mode timer to S-mode timer + x => x, + } +} + +impl Clint { + pub fn new(freq: usize, node: &FdtNode) -> Self { + // TODO IPI + // let reg = clint_node.reg().unwrap().next().unwrap(); + // reg.starting_address.add(crate::PHYS_OFFSET) as *mut u8; + // reg.size.unwrap(); + + let mut me = Self { + freq: freq as u64, + next_event: Vec::new(), + }; + let mut interrupts = node + .property("interrupts-extended") + .unwrap() + .value + .chunks(4) + .map(|x| BE::read_u32(x)); + let mut hart_id = 0; + while let Ok([phandle1, irq0, phandle2, irq1]) = interrupts.next_chunk::<4>() { + assert_eq!( + phandle1, phandle2, + "Invalid interrupts-extended property for CLINT" + ); + let hlic = unsafe { + IRQ_CHIP + .irq_chip_list + .chips + .iter() + .find(|x| x.phandle == phandle1) + .expect("Couldn't find HLIC in irqchip list for CLINT") + }; + + // FIXME dirty hack map M-mode interrupts (handled by SBI) to S-mode interrupts we get from SBI + // Why aren't S-mode interrupts in the DTB already? + let irq0 = map_interrupt(irq0); + let irq1 = map_interrupt(irq1); + + let virq0 = hlic + .ic + .irq_xlate(&[irq0, 0, 0]) + .expect("Couldn't get virq 0 from HLIC"); + let virq1 = hlic + .ic + .irq_xlate(&[irq1, 0, 0]) + .expect("Couldn't get virq 1 from HLIC"); + register_irq(virq0 as u32, Box::new(ClintConnector { hart_id, irq: 0 })); + register_irq(virq1 as u32, Box::new(ClintConnector { hart_id, irq: 1 })); + hart_id += 1; + } + me.next_event.resize_with(hart_id, || 0); + me + } + + pub(crate) fn irq_handler(self: &mut Self, hart_id: usize, irq: usize) { + match irq { + IRQ_IPI => { + println!("IPI interrupt at {}", hart_id); + } + IRQ_TIMER => { + let mtime: usize; + unsafe { + asm!( + "rdtime t0", + lateout("t0") mtime + ) + }; + + self.next_event[hart_id] = + max(self.next_event[hart_id], mtime as u64) + self.freq / TICKS_PER_SECOND; + SBI.set_timer(self.next_event[hart_id]) + .expect("SBI timer cannot be set!"); + } + _ => { + panic!("Unexpected CLINT irq") + } + } + } + + pub fn init(self: &mut Self, hart: usize) { + let mtime: usize; + unsafe { + asm!( + "rdtime t0", + lateout("t0") mtime + ) + }; + self.next_event[hart] = mtime as u64 + (self.freq / TICKS_PER_SECOND); + SBI.set_timer(self.next_event[hart]) + .expect("SBI timer cannot be set!"); + } +} diff --git a/src/arch/riscv64/device/irqchip/hlic.rs b/src/arch/riscv64/device/irqchip/hlic.rs new file mode 100644 index 0000000000..c3d66845d1 --- /dev/null +++ b/src/arch/riscv64/device/irqchip/hlic.rs @@ -0,0 +1,159 @@ +use crate::dtb::irqchip::{InterruptController, InterruptHandler, IrqDesc, IRQ_CHIP}; +use alloc::vec::Vec; +use core::arch::asm; +use fdt::{node::NodeProperty, Fdt}; +use syscall::{Error, ENOENT}; + +// This is a hart-local interrupt controller, a root of irqchip tree +// An example DTS: +// /cpus/ +// cpu@1/ +// interrupt-controller/ +// #interrupt-cells = 0x00000001 +// interrupt-controller = +// compatible = "riscv,cpu-intc" +// phandle = 0x00000006 + +fn acknowledge(interrupt: usize) { + unsafe { + asm!( + "csrc sip, t0", + in("t0") 1usize << interrupt, + options(nostack) + ) + } +} + +pub unsafe fn interrupt(hart: usize, interrupt: usize) { + assert!( + hart < CPU_INTERRUPT_HANDLERS.len(), + "Unexpected hart in interrupt routine" + ); + acknowledge(interrupt); + let ic_idx = CPU_INTERRUPT_HANDLERS[hart].unwrap_or_else(|| { + panic!( + "No hlic connected to hart {} yet interrupt {} occurred", + hart, interrupt + ) + }); + let virq = IRQ_CHIP + .irq_to_virq(ic_idx, interrupt as u32) + .unwrap_or_else(|| panic!("HLIC doesn't know of interrupt {}", interrupt)); + if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler { + handler.irq_handler(virq as u32); + } else if let Some(ic_idx) = IRQ_CHIP.irq_desc[virq].basic.child_ic_idx { + IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_handler(virq as u32); + } else { + panic!( + "Unconnected interrupt {} occurred on hlic connected to hart {}", + interrupt, hart + ); + } +} + +pub fn init() { + unsafe { + asm!( + "csrs sie, t0", + in("t0") (0xFFFF), + options(nostack) + ) + } +} + +static mut CPU_INTERRUPT_HANDLERS: Vec> = Vec::new(); + +pub struct Hlic { + virq_base: usize, +} + +impl Hlic { + pub(crate) fn new() -> Self { + return Self { virq_base: 0 }; + } +} +impl InterruptHandler for Hlic { + fn irq_handler(&mut self, irq: u32) { + assert!(irq < 16, "Unsupported HLIC interrupt raised!"); + unsafe { + IRQ_CHIP.trigger_virq(self.virq_base as u32 + irq); + } + } +} + +impl InterruptController for Hlic { + fn irq_init( + &mut self, + fdt: &Fdt, + irq_desc: &mut [IrqDesc; 1024], + ic_idx: usize, + irq_idx: &mut usize, + ) -> syscall::Result<()> { + let desc = unsafe { &IRQ_CHIP.irq_chip_list.chips[ic_idx] }; + let cpu_node = fdt + .find_all_nodes("/cpus/cpu") + .find(|x| { + x.children().any(|x| { + x.property("phandle").and_then(NodeProperty::as_usize) + == Some(desc.phandle as usize) + }) + }) + .expect("Could not find CPU node for HLIC controller"); + let hart = cpu_node.property("reg").unwrap().as_usize().unwrap(); + unsafe { + if CPU_INTERRUPT_HANDLERS.len() <= hart { + CPU_INTERRUPT_HANDLERS.resize(hart + 1, None); + } + assert!( + CPU_INTERRUPT_HANDLERS[hart].replace(ic_idx).is_none(), + "Conflicting HLIC interrupt handler found" + ); + } + self.virq_base = *irq_idx; + for i in 0..16 { + irq_desc[self.virq_base + i].basic.ic_idx = ic_idx; + irq_desc[self.virq_base + i].basic.ic_irq = i as u32; + } + *irq_idx += 16; + Ok(()) + } + + fn irq_ack(&mut self) -> u32 { + panic!("Cannot ack HLIC interrupt"); + } + + fn irq_eoi(&mut self, _irq_num: u32) {} + + fn irq_enable(&mut self, _irq_num: u32) { + // This would require IPI to a correct core + // Not bothering with this, all interrupts are enabled at all times + } + + fn irq_disable(&mut self, _irq_num: u32) { + // This would require IPI to a correct core + // Not bothering with this, all interrupts are enabled at all times + } + + fn irq_xlate(&self, irq_data: &[u32; 3]) -> syscall::Result { + let irq = irq_data[0]; + match irq { + 0..=0xF => Ok(self.virq_base + irq as usize), + _ => Err(Error::new(ENOENT)), + } + } + + fn irq_to_virq(&self, hwirq: u32) -> Option { + if hwirq > 0 && hwirq <= 0xF { + Some(self.virq_base + hwirq as usize) + } else { + None + } + } +} + +pub fn irqchip_for_hart(hart: usize) -> Option { + let value = unsafe { CPU_INTERRUPT_HANDLERS.get(hart) }?; + *value +} diff --git a/src/arch/riscv64/device/irqchip/mod.rs b/src/arch/riscv64/device/irqchip/mod.rs index bb93c393ca..b84709a69a 100644 --- a/src/arch/riscv64/device/irqchip/mod.rs +++ b/src/arch/riscv64/device/irqchip/mod.rs @@ -1,6 +1,44 @@ +use self::clint::Clint; use crate::dtb::irqchip::InterruptController; use alloc::boxed::Box; +use fdt::Fdt; -pub fn new_irqchip(_ic_str: &str) -> Option> { - None +pub(crate) mod hlic; +mod plic; + +#[path = "clint_sbi.rs"] +mod clint; + +// pub mod clint; // actual clint.rs off limits if SBI is present + +pub fn new_irqchip(ic_str: &str) -> Option> { + if ic_str.contains("riscv,cpu-intc") { + Some(Box::new(hlic::Hlic::new())) + } else if ic_str.contains("riscv,plic0") { + Some(Box::new(plic::Plic::new())) + } else { + log::warn!("no driver for interrupt controller {:?}", ic_str); + None + } +} + +pub unsafe fn init_clint(fdt: &Fdt) { + let cpus = fdt.find_node("/cpus").unwrap(); + let clock_freq = cpus + .property("timebase-frequency") + .unwrap() + .as_usize() + .unwrap(); + + let clint_node = fdt.find_node("/soc/clint").unwrap(); + assert!(clint_node + .compatible() + .unwrap() + .all() + .find(|x| ((*x).eq("riscv,clint0"))) + .is_some()); + + let clint = Clint::new(clock_freq, &clint_node); + *clint::CLINT.lock() = Some(clint); + clint::CLINT.lock().as_mut().unwrap().init(0); } diff --git a/src/arch/riscv64/device/irqchip/plic.rs b/src/arch/riscv64/device/irqchip/plic.rs new file mode 100644 index 0000000000..3bab54403a --- /dev/null +++ b/src/arch/riscv64/device/irqchip/plic.rs @@ -0,0 +1,196 @@ +use crate::{ + arch::{device::irqchip::hlic, start::BOOT_HART_ID}, + dtb::irqchip::{InterruptController, InterruptHandler, IrqDesc, IRQ_CHIP}, +}; +use core::{mem, num::NonZero, sync::atomic::Ordering}; +use fdt::Fdt; +use log::{error, info}; +use syscall::{Error, Io, Mmio, ENODEV}; + +#[repr(packed(4))] +#[repr(C)] +struct InterruptThresholdRegs { + threshold: Mmio, + claim_complete: Mmio, + _rsrv: [u32; 1022], +} + +static MAX_CONTEXTS: usize = 64; + +#[repr(packed(4))] +#[repr(C)] +struct PlicRegs { + /// source priorities + source_priority: [Mmio; 1024], // +0000 -- 0fff + // pending interrupts + pending: [Mmio; 1024], // +1000 -- 1fff + // per-context interrupt enable + enable: [[Mmio; 32]; 16320], // +2000 - 1f'ffff + // per-context priority threshold and acknowledge + thresholds: [InterruptThresholdRegs; 64], // specced at +20'0000 - 0fff'ffff for 15872 contexts + // but actual memory allotted in firmware is much lower +} + +const _: () = assert!(0x1000 == mem::offset_of!(PlicRegs, pending)); +const _: () = assert!(0x2000 == mem::offset_of!(PlicRegs, enable)); +const _: () = assert!(0x20_0000 == mem::offset_of!(PlicRegs, thresholds)); +const _: () = assert!(0x1000 == mem::size_of::()); + +impl PlicRegs { + pub fn set_priority(self: &mut Self, irq: usize, priority: usize) { + assert!(irq > 0 && irq <= 1023 && priority < 8); + self.source_priority[irq].write(priority as u32); + } + + pub fn pending(self: &Self, irq_lane: usize) -> u32 { + assert!(irq_lane < 32); + self.pending[irq_lane].read() + } + + pub fn enable(self: &mut Self, context: usize, irq: NonZero, enable: bool) { + assert!(irq.get() <= 1023 && context < MAX_CONTEXTS); + let irq_lane = irq.get() / 32; + let irq = irq.get() % 32; + self.enable[context][irq_lane].writef(1u32 << irq, enable); + } + + pub fn set_priority_threshold(self: &mut Self, context: usize, priority: usize) { + assert!(context < MAX_CONTEXTS && priority <= 7); + self.thresholds[context].threshold.write(priority as u32); + } + + pub fn claim(self: &mut Self, context: usize) -> Option> { + assert!(context < MAX_CONTEXTS); + let claim = self.thresholds[context].claim_complete.read(); + NonZero::new(claim as usize) + } + + pub fn complete(self: &mut Self, context: usize, claim: NonZero) { + assert!(context < MAX_CONTEXTS); + self.thresholds[context] + .claim_complete + .write(claim.get() as u32); + } +} + +pub struct Plic { + regs: *mut PlicRegs, + ndev: usize, + virq_base: usize, + context: usize, +} + +impl Plic { + pub fn new() -> Self { + Self { + regs: 0 as *mut PlicRegs, + ndev: 0, + virq_base: 0, + context: 0, + } + } +} +impl InterruptHandler for Plic { + fn irq_handler(&mut self, _irq: u32) { + unsafe { + let irq = self.irq_ack(); + //println!("PLIC interrupt {}", irq); + if let Some(virq) = self.irq_to_virq(irq) { + IRQ_CHIP.trigger_virq(virq as u32); + } else { + error!("unexpected irq num {}", irq); + self.irq_eoi(irq); + } + } + //println!("PLIC interrupt done"); + } +} + +impl InterruptController for Plic { + fn irq_init( + &mut self, + fdt: &Fdt, + irq_desc: &mut [IrqDesc; 1024], + ic_idx: usize, + irq_idx: &mut usize, + ) -> syscall::Result<()> { + let desc = unsafe { &IRQ_CHIP.irq_chip_list.chips[ic_idx] }; + let my_node = fdt.find_phandle(desc.phandle).unwrap(); + + // MMIO region + let reg = my_node.reg().unwrap().next().unwrap(); + // Specifies how many external interrupts are supported by this controller. + let ndev = my_node + .property("riscv,ndev") + .and_then(|x| x.as_usize()) + .unwrap(); + + unsafe { + self.regs = reg.starting_address.add(crate::PHYS_OFFSET) as *mut PlicRegs; + } + self.ndev = ndev; + + self.virq_base = *irq_idx; + for i in 0..ndev { + irq_desc[self.virq_base + i].basic.ic_idx = ic_idx; + irq_desc[self.virq_base + i].basic.ic_irq = i as u32; + } + *irq_idx += ndev; + + // route all interrupts to boot HART + // TODO spread irqs over all the cores when we have them? + let hlic_ic_idx = hlic::irqchip_for_hart(BOOT_HART_ID.load(Ordering::Relaxed)) + .expect("Could not find HLIC irqchip for the boot hart while initing PLIC"); + self.context = desc + .parents + .iter() + .position(|x| x.parent_interrupt[0] != u32::MAX && x.parent == hlic_ic_idx) + .unwrap(); + info!("PLIC: using context {}", self.context); + + let regs = unsafe { self.regs.as_mut().unwrap() }; + regs.set_priority_threshold(self.context, 0); + + Ok(()) + } + + fn irq_ack(&mut self) -> u32 { + let regs = unsafe { self.regs.as_mut().unwrap() }; + regs.claim(self.context).unwrap().get() as u32 + } + + fn irq_eoi(&mut self, irq_num: u32) { + let regs = unsafe { self.regs.as_mut().unwrap() }; + regs.complete(self.context, NonZero::new(irq_num as usize).unwrap()); + } + + fn irq_enable(&mut self, irq_num: u32) { + assert!(irq_num > 0 && irq_num as usize <= self.ndev); + let regs = unsafe { self.regs.as_mut().unwrap() }; + regs.set_priority(irq_num as usize, 1); + regs.enable(self.context, NonZero::new(irq_num as usize).unwrap(), true); + } + + fn irq_disable(&mut self, irq_num: u32) { + assert!(irq_num > 0 && irq_num as usize <= self.ndev); + let regs = unsafe { self.regs.as_mut().unwrap() }; + regs.set_priority(irq_num as usize, 1); + regs.enable(self.context, NonZero::new(irq_num as usize).unwrap(), false); + } + + fn irq_xlate(&self, irq_data: &[u32; 3]) -> syscall::Result { + if (irq_data[0] as usize) < self.ndev { + Ok(self.virq_base + irq_data[0] as usize) + } else { + Err(Error::new(ENODEV)) + } + } + + fn irq_to_virq(&self, hwirq: u32) -> Option { + if (hwirq as usize) < self.ndev { + Some(self.virq_base + hwirq as usize) + } else { + None + } + } +} diff --git a/src/arch/riscv64/device/mod.rs b/src/arch/riscv64/device/mod.rs index 496249d394..de2fa976d8 100644 --- a/src/arch/riscv64/device/mod.rs +++ b/src/arch/riscv64/device/mod.rs @@ -1,13 +1,18 @@ -use crate::{arch::time, dtb::DTB_BINARY}; +use crate::{ + arch::{device::irqchip::hlic, time}, + dtb::DTB_BINARY, +}; use fdt::{ node::{FdtNode, NodeProperty}, Fdt, }; pub mod cpu; -pub mod irqchip; +pub(crate) mod irqchip; pub mod serial; +use crate::arch::device::irqchip::init_clint; + fn string_property(name: &str) -> bool { name == "compatible" || name == "model" @@ -59,12 +64,16 @@ unsafe fn init_intc(cpu: &FdtNode) { .find(|x| x.name == "interrupt-controller") .unwrap(); assert_eq!(intc_node.compatible().unwrap().first(), "riscv,cpu-intc"); + // This controller is hardwired into interrupt handler code and has no Mmios + hlic::init(); // enable interrupts at HLIC level } pub unsafe fn init() { let data = DTB_BINARY.get().unwrap(); let fdt = Fdt::new(data).unwrap(); + crate::dtb::irqchip::init(&fdt); + let cpu = fdt.find_node(format!("/cpus/cpu@{}", 0).as_str()).unwrap(); init_intc(&cpu); init_time(&fdt); @@ -84,6 +93,7 @@ pub unsafe fn init_noncore() { let data = DTB_BINARY.get().unwrap(); let fdt = Fdt::new(data).unwrap(); + init_clint(&fdt); serial::init(&fdt); } diff --git a/src/arch/riscv64/device/serial.rs b/src/arch/riscv64/device/serial.rs index 041b0f2e22..155145d723 100644 --- a/src/arch/riscv64/device/serial.rs +++ b/src/arch/riscv64/device/serial.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, vec::Vec}; +use alloc::boxed::Box; use byteorder::{ByteOrder, BE}; use fdt::Fdt; use log::info; @@ -7,7 +7,10 @@ use syscall::Mmio; use crate::{ devices::uart_16550, - dtb::{diag_uart_range, DTB_BINARY}, + dtb::{ + diag_uart_range, + irqchip::{register_irq, InterruptHandler, IRQ_CHIP}, + }, scheme::{ debug::{debug_input, debug_notify}, irq::irq_trigger, @@ -33,6 +36,18 @@ pub static COM1: Mutex> = Mutex::new(None); pub struct Com1Irq {} +impl InterruptHandler for Com1Irq { + fn irq_handler(&mut self, irq: u32) { + if let Some(ref mut serial_port) = *COM1.lock() { + serial_port.receive(); + }; + unsafe { + irq_trigger(irq as u8); + IRQ_CHIP.irq_eoi(irq); + } + } +} + pub unsafe fn init_early(dtb: &Fdt) { if COM1.lock().is_some() { // Hardcoded UART @@ -58,8 +73,6 @@ pub unsafe fn init_early(dtb: &Fdt) { } pub unsafe fn init(fdt: &Fdt) -> Option<()> { - let data = DTB_BINARY.get().unwrap(); - let fdt = Fdt::new(data).unwrap(); if let Some(node) = fdt.find_compatible(&["ns16550a"]) { let interrupts = node.property("interrupts").unwrap(); let mut intr_data: [u32; 3] = [0, 0, 0]; @@ -70,8 +83,20 @@ pub unsafe fn init(fdt: &Fdt) -> Option<()> { let val = BE::read_u32(chunk); intr_data[idx] = val; } + + let interrupt_parent = node.interrupt_parent()?; + let phandle = interrupt_parent.property("phandle")?.as_usize()? as u32; + let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle)?; + + let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx] + .ic + .irq_xlate(&intr_data) + .unwrap(); + info!("serial_port virq = {}", virq); + register_irq(virq as u32, Box::new(Com1Irq {})); + IRQ_CHIP.irq_enable(virq as u32); } - if let Some(ref mut serial_port) = *COM1.lock() { + if let Some(ref mut _serial_port) = *COM1.lock() { // serial_port.enable_irq(); // FIXME receive int is enabled by default in 16550. Disable by default? } Some(()) diff --git a/src/arch/riscv64/interrupt/exception.rs b/src/arch/riscv64/interrupt/exception.rs index 85ce556223..ed135f0512 100644 --- a/src/arch/riscv64/interrupt/exception.rs +++ b/src/arch/riscv64/interrupt/exception.rs @@ -1,11 +1,14 @@ -use core::{ - arch::global_asm, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::{arch::global_asm, sync::atomic::Ordering}; use log::{error, info}; use rmm::VirtualAddress; -use crate::{memory::GenericPfFlags, panic::stack_trace, ptrace, syscall, syscall::flag::*}; +use crate::{ + arch::{device::irqchip, start::BOOT_HART_ID}, + memory::GenericPfFlags, + panic::stack_trace, + ptrace, syscall, + syscall::flag::*, +}; const BREAKPOINT: usize = 3; const USERMODE_ECALL: usize = 8; @@ -133,7 +136,12 @@ unsafe fn handle_system_exception(scause: usize, regs: &InterruptStack) { loop {} } -unsafe fn handle_interrupt(interrupt: usize) {} +unsafe fn handle_interrupt(interrupt: usize) { + // FIXME retrieve from percpu area + // For now all the interrupts go to boot hart so this suffices... + let hart: usize = BOOT_HART_ID.load(Ordering::Relaxed); + irqchip::hlic::interrupt(hart, interrupt); +} unsafe fn handle_user_exception(scause: usize, regs: &mut InterruptStack) { if scause == USERMODE_ECALL { diff --git a/src/arch/riscv64/interrupt/mod.rs b/src/arch/riscv64/interrupt/mod.rs index a44bb15024..16b8c8b281 100644 --- a/src/arch/riscv64/interrupt/mod.rs +++ b/src/arch/riscv64/interrupt/mod.rs @@ -7,9 +7,6 @@ mod exception; pub mod syscall; pub mod trace; -pub use self::trace::StackTrace; - -use crate::cpu_set::LogicalCpuId; pub use handler::InterruptStack; pub fn bsp_apic_id() -> Option { diff --git a/src/arch/riscv64/rmm.rs b/src/arch/riscv64/rmm.rs index 2c1388e5b2..a4be789a32 100644 --- a/src/arch/riscv64/rmm.rs +++ b/src/arch/riscv64/rmm.rs @@ -1,4 +1,3 @@ -use super::CurrentRmmArch as RmmA; use rmm::{Arch, PageFlags, VirtualAddress}; pub struct KernelMapper { diff --git a/src/arch/riscv64/start.rs b/src/arch/riscv64/start.rs index 8014c1b86d..e29ac32cfc 100644 --- a/src/arch/riscv64/start.rs +++ b/src/arch/riscv64/start.rs @@ -20,10 +20,7 @@ use crate::{ #[cfg(feature = "graphical_debug")] use crate::devices::graphical_debug; -use crate::{ - devices::graphical_debug::{DebugDisplay, DEBUG_DISPLAY, FRAMEBUFFER}, - dtb::register_dev_memory_ranges, -}; +use crate::dtb::register_dev_memory_ranges; /// Test of zero values in BSS. static mut BSS_TEST_ZERO: usize = 0; diff --git a/src/arch/x86_64/paging/mod.rs b/src/arch/x86_64/paging/mod.rs index 441da03c35..e892cb5577 100644 --- a/src/arch/x86_64/paging/mod.rs +++ b/src/arch/x86_64/paging/mod.rs @@ -23,6 +23,8 @@ pub mod entry { pub mod mapper; +pub const ENTRY_COUNT: usize = RmmA::PAGE_ENTRIES; + /// Size of pages pub const PAGE_SIZE: usize = RmmA::PAGE_SIZE; pub const PAGE_MASK: usize = RmmA::PAGE_OFFSET_MASK; diff --git a/src/context/arch/riscv64.rs b/src/context/arch/riscv64.rs index 8b761b730f..298ab83278 100644 --- a/src/context/arch/riscv64.rs +++ b/src/context/arch/riscv64.rs @@ -8,18 +8,14 @@ use crate::{ percpu::PercpuBlock, syscall::FloatRegisters, }; -use alloc::sync::Arc; use core::{mem::offset_of, sync::atomic::AtomicBool}; use rmm::{Arch, TableKind, VirtualAddress}; use spin::Once; -use spinning_top::RwSpinlock; use syscall::{error::*, EnvRegisters}; pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false); pub const KFX_ALIGN: usize = 16; -pub const KSTACK_SIZE: usize = 65536; -pub const KSTACK_ALIGN: usize = 16; #[derive(Clone, Debug, Default)] pub struct Context { diff --git a/src/dtb/irqchip.rs b/src/dtb/irqchip.rs index ececd51a07..6b1f8968e7 100644 --- a/src/dtb/irqchip.rs +++ b/src/dtb/irqchip.rs @@ -1,48 +1,47 @@ use super::travel_interrupt_ctrl; -use crate::{ - arch, arch::device::irqchip::new_irqchip, cpu_set::LogicalCpuId, scheme::irq::irq_trigger, -}; +use crate::{arch::device::irqchip::new_irqchip, cpu_set::LogicalCpuId, scheme::irq::irq_trigger}; use alloc::{boxed::Box, vec::Vec}; use byteorder::{ByteOrder, BE}; use fdt::{node::NodeProperty, Fdt}; use log::{debug, error}; -use syscall::{Error, Result, ENOENT}; +use syscall::{Error, Result}; pub trait InterruptHandler { fn irq_handler(&mut self, irq: u32); } -pub trait InterruptController { +pub trait InterruptController: InterruptHandler { fn irq_init( &mut self, fdt: &Fdt, irq_desc: &mut [IrqDesc; 1024], ic_idx: usize, irq_idx: &mut usize, - ) -> Result>; + ) -> Result<()>; fn irq_ack(&mut self) -> u32; fn irq_eoi(&mut self, irq_num: u32); fn irq_enable(&mut self, irq_num: u32); #[allow(unused)] fn irq_disable(&mut self, irq_num: u32); - fn irq_xlate(&mut self, irq_data: &[u32], idx: usize) -> Result; - fn irq_to_virq(&mut self, hwirq: u32) -> Option; - fn irq_handler(&mut self, irq: u32); + fn irq_xlate(&self, irq_data: &[u32; 3]) -> Result; + fn irq_to_virq(&self, hwirq: u32) -> Option; +} + +pub struct IrqConnection { + pub parent_phandle: u32, + pub parent: usize, // parent idx in chiplist + pub parent_interrupt: [u32; 3], } pub struct IrqChipItem { pub phandle: u32, - pub parent_phandle: Option, - pub parent: Option, //parent idx in chiplist - pub childs: Vec, //child idx in chiplist - pub interrupts: Vec, + pub parents: Vec, + pub children: Vec, // child idx in chiplist pub ic: Box, } pub struct IrqChipList { pub chips: Vec, - pub root_phandle: u32, - pub root_idx: usize, } pub struct IrqDescItem { @@ -60,20 +59,11 @@ pub struct IrqDesc { impl IrqChipList { fn init_inner1(&mut self, fdt: &Fdt) { - let root_node = fdt.root(); - let mut idx = 0; - let intr = root_node.property("interrupt-parent").unwrap(); - - let root_intr_parent = intr.as_usize().unwrap() as u32; - debug!("root parent = 0x{:08x}", root_intr_parent); - self.root_phandle = root_intr_parent; for node in fdt.all_nodes() { if node.property("interrupt-controller").is_some() { let compatible = node.property("compatible").unwrap().as_str().unwrap(); let phandle = node.property("phandle").unwrap().as_usize().unwrap() as u32; let intr_cells = node.interrupt_cells().unwrap(); - let _intr = node.property("interrupt-parent"); - let _intr_data = node.property("interrupts"); debug!( "{}, compatible = {}, #interrupt-cells = 0x{:08x}, phandle = 0x{:08x}", @@ -81,72 +71,146 @@ impl IrqChipList { ); let mut item = IrqChipItem { phandle, - parent_phandle: None, - parent: None, - childs: Vec::new(), - interrupts: Vec::new(), + parents: Vec::new(), + children: Vec::new(), ic: new_irqchip(compatible).unwrap(), }; - if let Some(intr) = _intr { - if let Some(intr_data) = _intr_data { - let intr = intr.as_usize().unwrap() as u32; - debug!("interrupt-parent = 0x{:08x}", intr); - item.parent_phandle = Some(intr); - debug!("interrupts begin:"); - for chunk in intr_data.value.chunks(4) { - debug!("0x{:08x}, ", BE::read_u32(chunk)); - item.interrupts.push(BE::read_u32(chunk)); - } - debug!("interrupts end"); + + fn interrupt_address( + iter: &mut impl Iterator, + interrupt_cells: usize, + ) -> Option<[u32; 3]> { + match interrupt_cells { + 1 if let Some(a) = iter.next() => Some([a, 0, 0]), + 2 if let Ok([a, b]) = iter.next_chunk() => Some([a, b, 0]), + 3 => iter.next_chunk::<3>().ok(), + _ => None, } } - if item.phandle == root_intr_parent { - self.root_idx = idx as usize; + + if let Some(parent) = node.interrupt_parent() + && let Some(intr_data) = node.property("interrupts") + { + // FIXME use interrupts() helper when fixed (see gh#12) + let parent_interrupt_cells = parent.interrupt_cells().unwrap(); + let parent_phandle = parent + .property("phandle") + .and_then(NodeProperty::as_usize) + .unwrap() as u32; + debug!("interrupt-parent = 0x{:08x}", parent_phandle); + debug!("interrupts begin:"); + let mut intr_data = intr_data.value.chunks(4).map(|x| BE::read_u32(x)); + while let Some(parent_interrupt) = + interrupt_address(&mut intr_data, parent_interrupt_cells) + { + debug!("{:?}, ", parent_interrupt); + item.parents.push(IrqConnection { + parent_phandle, + parent: 0, + parent_interrupt, + }); + } + debug!("interrupts end"); + } else if let Some(intr_data) = node.property("interrupts-extended") { + // FIXME use the helper when fixed (see gh#37) + // Shouldn't matter much since ARM seems to not use extended interrupt and + // RISC-V seems to not use 3-sized interrupt addresses + let mut intr_data = intr_data.value.chunks(4).map(|x| BE::read_u32(x)); + while let Some(parent_phandle) = intr_data.next() + && let Some(parent) = fdt.find_phandle(parent_phandle) + && let Some(parent_interrupt_cells) = parent.interrupt_cells() + && let Some(parent_interrupt) = + interrupt_address(&mut intr_data, parent_interrupt_cells) + { + debug!("{:?}, ", parent_interrupt); + item.parents.push(IrqConnection { + parent_phandle, + parent: 0, + parent_interrupt, + }); + } } self.chips.push(item); - - idx += 1; } } } - fn init_inner2(&mut self) { - let mut x = 0; + fn init_inner2(&mut self) -> Vec { + let mut roots = Vec::new(); - while x < self.chips.len() { - let mut y = 0; - while y < self.chips.len() { - if x == y { - y += 1; - continue; - } - if let Some(pp) = self.chips[y].parent_phandle - && pp == self.chips[x].phandle - { - self.chips[y].parent = Some(x); - self.chips[x].childs.push(y); - } - y += 1; + for child_i in 0..self.chips.len() { + let child = &mut self.chips[child_i]; + let phandle = child.phandle; + + if child.parents.is_empty() { + roots.push(child_i); + continue; + } + + for conn_i in 0..child.parents.len() { + let parent_phandle = self.chips[child_i].parents[conn_i].parent_phandle; + let parent_i = self + .chips + .iter() + .position(|x| parent_phandle == x.phandle) + .unwrap_or_else(|| { + panic!( + "Cannot find parent intc {} (connection from {})", + parent_phandle, phandle + ) + }); + self.chips[child_i].parents[conn_i].parent = parent_i; + let parent = &mut self.chips[parent_i]; + if !parent.children.contains(&child_i) { + parent.children.push(child_i); + } } - x += 1; } + roots } - fn init_inner3(&mut self, fdt: &fdt::Fdt, irq_desc: &mut [IrqDesc; 1024]) { + fn init_inner3(&mut self, fdt: &Fdt, irq_desc: &mut [IrqDesc; 1024], mut queue: Vec) { //run init - let mut queue = Vec::new(); let mut irq_idx: usize = 0; - queue.push(self.root_idx); - while !queue.is_empty() { - let cur_idx = queue.pop().unwrap(); - queue.extend_from_slice(&self.chips[cur_idx].childs); - let virq = self.chips[cur_idx] - .ic - .irq_init(fdt, irq_desc, cur_idx, &mut irq_idx); - if let Ok(Some(virq)) = virq { - irq_desc[virq].basic.child_ic_idx = Some(cur_idx); + let mut queue_idx = 0; + while queue_idx < queue.len() { + let cur_idx = queue[queue_idx]; + let cur_chip = &mut self.chips[cur_idx]; + for child in &cur_chip.children { + if let Some(child_pos) = queue.iter().position(|x| *child == *x) { + assert!( + child_pos > queue_idx, + "IRQ chip tree has a cycle with phandle {} in it", + cur_chip.phandle + ); + } else { + queue.push(*child); + } } + cur_chip + .ic + .irq_init(fdt, irq_desc, cur_idx, &mut irq_idx) + .expect("Failed to initialize irq chip"); + + let cur_chip = &self.chips[cur_idx]; + for connection in &cur_chip.parents { + debug_assert!(queue[0..queue_idx].contains(&connection.parent)); + if connection.parent_interrupt[0] != u32::MAX { + let parent = &self.chips[connection.parent]; + if let Ok(virq) = parent.ic.irq_xlate(&connection.parent_interrupt) { + // assert is unused + irq_desc[virq].basic.child_ic_idx = Some(cur_idx); + } else { + error!( + "Cannot connect irq chip {} to parent irq {} : {:?}", + cur_chip.phandle, parent.phandle, connection.parent_interrupt + ); + } + } + } + + queue_idx += 1; } } } @@ -158,12 +222,6 @@ pub struct IrqChipCore { } impl IrqChipCore { - pub fn irq_ack(&mut self) -> u32 { - self.irq_chip_list.chips[self.irq_chip_list.root_idx] - .ic - .irq_ack() - } - pub fn irq_eoi(&mut self, virq: u32) { let irq_desc = &self.irq_desc[virq as usize]; let ic_idx = irq_desc.basic.ic_idx; @@ -189,24 +247,49 @@ impl IrqChipCore { self.irq_chip_list.chips[ic_idx].ic.irq_disable(hwirq) } - pub fn irq_to_virq(&mut self, hwirq: u32) -> Option { - self.irq_chip_list.chips[self.irq_chip_list.root_idx] - .ic - .irq_to_virq(hwirq) + #[cfg(target_arch = "riscv64")] + pub fn irq_to_virq(&self, ic_idx: usize, hwirq: u32) -> Option { + self.irq_chip_list.chips[ic_idx].ic.irq_to_virq(hwirq) + } + + #[cfg(target_arch = "riscv64")] + pub fn irq_xlate(&self, ic_idx: usize, irq_data: &[u32; 3]) -> Result { + self.irq_chip_list.chips[ic_idx].ic.irq_xlate(irq_data) + } + + pub fn trigger_virq(&mut self, virq: u32) { + if virq < 1024 { + let desc = &mut self.irq_desc[virq as usize]; + if let Some(handler) = &mut desc.handler { + handler.irq_handler(virq); + } else if let Some(ic_idx) = desc.basic.child_ic_idx { + self.irq_chip_list.chips[ic_idx].ic.irq_handler(virq); + } else { + irq_trigger(virq as u8); + } + } } pub fn init(&mut self, fdt: &Fdt) { - for i in 0..1024 { - self.irq_desc[i].basic.idx = i; + for (i, desc) in self.irq_desc.iter_mut().enumerate() { + desc.basic.idx = i; } self.irq_chip_list.init_inner1(fdt); - self.irq_chip_list.init_inner2(); - self.irq_chip_list.init_inner3(fdt, &mut self.irq_desc); + let roots = self.irq_chip_list.init_inner2(); + self.irq_chip_list + .init_inner3(fdt, &mut self.irq_desc, roots); + } + + pub fn phandle_to_ic_idx(&self, phandle: u32) -> Option { + self.irq_chip_list + .chips + .iter() + .position(|x| x.phandle == phandle) } } -pub unsafe fn acknowledge(_irq: usize) { - // TODO +pub unsafe fn acknowledge(irq: usize) { + IRQ_CHIP.irq_eoi(irq as u32); } const INIT_HANDLER: Option> = None; @@ -221,11 +304,7 @@ const INIT_IRQ_DESC: IrqDesc = IrqDesc { handler: INIT_HANDLER, }; pub static mut IRQ_CHIP: IrqChipCore = IrqChipCore { - irq_chip_list: IrqChipList { - chips: Vec::new(), - root_phandle: 0, - root_idx: 0, - }, + irq_chip_list: IrqChipList { chips: Vec::new() }, irq_desc: [INIT_IRQ_DESC; 1024], }; @@ -253,14 +332,20 @@ pub fn register_irq(virq: u32, handler: Box) { } #[inline] -pub fn is_reserved(_cpu_id: LogicalCpuId, _index: u8) -> bool { - //TODO - true +pub fn is_reserved(_cpu_id: LogicalCpuId, index: u8) -> bool { + unsafe { IRQ_CHIP.irq_desc[index as usize].basic.used } } #[inline] -pub fn set_reserved(_cpu_id: LogicalCpuId, _index: u8, _reserved: bool) { - //TODO +pub fn set_reserved(_cpu_id: LogicalCpuId, index: u8, reserved: bool) { + unsafe { + IRQ_CHIP.irq_desc[index as usize].basic.used = reserved; + if reserved { + IRQ_CHIP.irq_enable(index as u32); + } else { + IRQ_CHIP.irq_enable(index as u32); + } + } } pub fn available_irqs_iter(_cpu_id: LogicalCpuId) -> impl Iterator + 'static { diff --git a/src/dtb/mod.rs b/src/dtb/mod.rs index 2d216e827c..8751e2a6bb 100644 --- a/src/dtb/mod.rs +++ b/src/dtb/mod.rs @@ -28,12 +28,13 @@ pub unsafe fn init(dtb: Option<(usize, usize)>) { } pub fn travel_interrupt_ctrl(fdt: &Fdt) { - let root_intr_parent = fdt + if let Some(root_intr_parent) = fdt .root() .property("interrupt-parent") .and_then(NodeProperty::as_usize) - .unwrap(); - debug!("root parent = 0x{:08x}", root_intr_parent); + { + debug!("root parent = 0x{:08x}", root_intr_parent); + } for node in fdt.all_nodes() { if node.property("interrupt-controller").is_some() { let compatible = node.property("compatible").unwrap().as_str().unwrap();