diff --git a/audio/ihdad/src/main.rs b/audio/ihdad/src/main.rs index e9a91590ec..078b12265d 100755 --- a/audio/ihdad/src/main.rs +++ b/audio/ihdad/src/main.rs @@ -12,11 +12,8 @@ use syscall::{Packet, SchemeBlockMut}; use event::{user_data, EventQueue}; #[cfg(target_arch = "x86_64")] -use pcid_interface::irq_helpers::allocate_single_interrupt_vector_for_msi; -use pcid_interface::irq_helpers::read_bsp_apic_id; -use pcid_interface::{ - MsiSetFeatureInfo, PciFeature, PciFeatureInfo, PciFunctionHandle, SetFeatureInfo, -}; +use pcid_interface::irq_helpers::allocate_first_msi_interrupt_on_bsp; +use pcid_interface::PciFunctionHandle; pub mod hda; @@ -35,33 +32,9 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { log::debug!("PCI FEATURES: {:?}", all_pci_features); let has_msi = all_pci_features.iter().any(|feature| feature.is_msi()); - let has_msix = all_pci_features.iter().any(|feature| feature.is_msix()); - if has_msi && !has_msix { - let capability = match pcid_handle.feature_info(PciFeature::Msi) { - PciFeatureInfo::Msi(s) => s, - PciFeatureInfo::MsiX(_) => panic!(), - }; - // TODO: Allow allocation of up to 32 vectors. - - // TODO: Find a way to abstract this away, potantially as a helper module for - // pcid_interface, so that this can be shared between nvmed, xhcid, ixgebd, etc.. - - let destination_id = read_bsp_apic_id().expect("ihdad: failed to read BSP apic id"); - let (msg_addr_and_data, interrupt_handle) = - allocate_single_interrupt_vector_for_msi(destination_id); - - let set_feature_info = MsiSetFeatureInfo { - multi_message_enable: Some(0), - message_address_and_data: Some(msg_addr_and_data), - mask_bits: None, - }; - pcid_handle.set_feature_info(SetFeatureInfo::Msi(set_feature_info)); - - pcid_handle.enable_feature(PciFeature::Msi); - log::debug!("Enabled MSI"); - - interrupt_handle + if has_msi { + allocate_first_msi_interrupt_on_bsp(pcid_handle) } else if let Some(irq) = pci_config.func.legacy_interrupt_line { log::debug!("Legacy IRQ {}", irq); diff --git a/net/rtl8139d/src/main.rs b/net/rtl8139d/src/main.rs index c0272136c7..aaf4a9bd70 100644 --- a/net/rtl8139d/src/main.rs +++ b/net/rtl8139d/src/main.rs @@ -1,18 +1,15 @@ use std::fs::File; use std::io::{Read, Write}; use std::os::unix::io::AsRawFd; -use std::ptr::NonNull; -use std::rc::Rc; use driver_network::NetworkScheme; use event::{user_data, EventQueue}; -#[cfg(target_arch = "x86_64")] -use pcid_interface::irq_helpers::allocate_single_interrupt_vector_for_msi; use pcid_interface::irq_helpers::read_bsp_apic_id; -use pcid_interface::msi::{MsixInfo, MsixTableEntry}; -use pcid_interface::{ - MappedBar, MsiSetFeatureInfo, PciFeature, PciFeatureInfo, PciFunctionHandle, SetFeatureInfo, +#[cfg(target_arch = "x86_64")] +use pcid_interface::irq_helpers::{ + allocate_first_msi_interrupt_on_bsp, allocate_single_interrupt_vector_for_msi, }; +use pcid_interface::{PciFeature, PciFeatureInfo, PciFunctionHandle}; pub mod device; @@ -28,21 +25,6 @@ where } } -pub struct MappedMsixRegs { - pub virt_table_base: NonNull, - pub info: MsixInfo, -} - -impl MappedMsixRegs { - pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry { - &mut *self.virt_table_base.as_ptr().offset(k as isize) - } - pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry { - assert!(k < self.info.table_size as usize); - unsafe { self.table_entry_pointer_unchecked(k) } - } -} - #[cfg(target_arch = "x86_64")] fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { let pci_config = pcid_handle.config(); @@ -53,49 +35,12 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { let has_msi = all_pci_features.iter().any(|feature| feature.is_msi()); let has_msix = all_pci_features.iter().any(|feature| feature.is_msix()); - if has_msi && !has_msix { - let capability = match pcid_handle.feature_info(PciFeature::Msi) { - PciFeatureInfo::Msi(s) => s, - PciFeatureInfo::MsiX(_) => panic!(), - }; - // TODO: Allow allocation of up to 32 vectors. - - // TODO: Find a way to abstract this away, potantially as a helper module for - // pcid_interface, so that this can be shared between nvmed, xhcid, ixgebd, etc.. - - let destination_id = read_bsp_apic_id().expect("rtl8139d: failed to read BSP apic id"); - let (msg_addr_and_data, interrupt_handle) = - allocate_single_interrupt_vector_for_msi(destination_id); - - let set_feature_info = MsiSetFeatureInfo { - multi_message_enable: Some(0), - message_address_and_data: Some(msg_addr_and_data), - mask_bits: None, - }; - pcid_handle.set_feature_info(SetFeatureInfo::Msi(set_feature_info)); - - pcid_handle.enable_feature(PciFeature::Msi); - log::info!("Enabled MSI"); - - interrupt_handle - } else if has_msix { + if has_msix { let msix_info = match pcid_handle.feature_info(PciFeature::MsiX) { PciFeatureInfo::Msi(_) => panic!(), PciFeatureInfo::MsiX(s) => s, }; - msix_info.validate(pci_config.func.bars); - - let bar_address = unsafe { pcid_handle.map_bar(msix_info.table_bar) } - .ptr - .as_ptr() as usize; - - let virt_table_base = - (bar_address + msix_info.table_offset as usize) as *mut MsixTableEntry; - - let mut info = MappedMsixRegs { - virt_table_base: NonNull::new(virt_table_base).unwrap(), - info: msix_info, - }; + let mut info = unsafe { msix_info.map_and_mask_all(pcid_handle) }; // Allocate one msi vector. @@ -103,7 +48,6 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { // primary interrupter let k = 0; - assert_eq!(std::mem::size_of::(), 16); let table_entry_pointer = info.table_entry_pointer(k); let destination_id = read_bsp_apic_id().expect("rtl8139d: failed to read BSP apic id"); @@ -119,6 +63,8 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { log::info!("Enabled MSI-X"); method + } else if has_msi { + allocate_first_msi_interrupt_on_bsp(pcid_handle) } else if let Some(irq) = pci_config.func.legacy_interrupt_line { // legacy INTx# interrupt pins. irq.irq_handle("rtl8139d") diff --git a/net/rtl8168d/src/main.rs b/net/rtl8168d/src/main.rs index 44150572f7..27094eb481 100644 --- a/net/rtl8168d/src/main.rs +++ b/net/rtl8168d/src/main.rs @@ -1,18 +1,15 @@ use std::fs::File; use std::io::{Read, Write}; use std::os::unix::io::AsRawFd; -use std::ptr::NonNull; -use std::rc::Rc; use driver_network::NetworkScheme; use event::{user_data, EventQueue}; -#[cfg(target_arch = "x86_64")] -use pcid_interface::irq_helpers::allocate_single_interrupt_vector_for_msi; use pcid_interface::irq_helpers::read_bsp_apic_id; -use pcid_interface::msi::{MsixInfo, MsixTableEntry}; -use pcid_interface::{ - MappedBar, MsiSetFeatureInfo, PciFeature, PciFeatureInfo, PciFunctionHandle, SetFeatureInfo, +#[cfg(target_arch = "x86_64")] +use pcid_interface::irq_helpers::{ + allocate_first_msi_interrupt_on_bsp, allocate_single_interrupt_vector_for_msi, }; +use pcid_interface::{PciFeature, PciFeatureInfo, PciFunctionHandle}; pub mod device; @@ -28,21 +25,6 @@ where } } -pub struct MappedMsixRegs { - pub virt_table_base: NonNull, - pub info: MsixInfo, -} - -impl MappedMsixRegs { - pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry { - &mut *self.virt_table_base.as_ptr().offset(k as isize) - } - pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry { - assert!(k < self.info.table_size as usize); - unsafe { self.table_entry_pointer_unchecked(k) } - } -} - #[cfg(target_arch = "x86_64")] fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { let pci_config = pcid_handle.config(); @@ -53,49 +35,12 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { let has_msi = all_pci_features.iter().any(|feature| feature.is_msi()); let has_msix = all_pci_features.iter().any(|feature| feature.is_msix()); - if has_msi && !has_msix { - let capability = match pcid_handle.feature_info(PciFeature::Msi) { - PciFeatureInfo::Msi(s) => s, - PciFeatureInfo::MsiX(_) => panic!(), - }; - // TODO: Allow allocation of up to 32 vectors. - - // TODO: Find a way to abstract this away, potantially as a helper module for - // pcid_interface, so that this can be shared between nvmed, xhcid, ixgebd, etc.. - - let destination_id = read_bsp_apic_id().expect("rtl8168d: failed to read BSP apic id"); - let (msg_addr_and_data, interrupt_handle) = - allocate_single_interrupt_vector_for_msi(destination_id); - - let set_feature_info = MsiSetFeatureInfo { - multi_message_enable: Some(0), - message_address_and_data: Some(msg_addr_and_data), - mask_bits: None, - }; - pcid_handle.set_feature_info(SetFeatureInfo::Msi(set_feature_info)); - - pcid_handle.enable_feature(PciFeature::Msi); - log::info!("Enabled MSI"); - - interrupt_handle - } else if has_msix { + if has_msix { let msix_info = match pcid_handle.feature_info(PciFeature::MsiX) { PciFeatureInfo::Msi(_) => panic!(), PciFeatureInfo::MsiX(s) => s, }; - msix_info.validate(pci_config.func.bars); - - let bar_address = unsafe { pcid_handle.map_bar(msix_info.table_bar) } - .ptr - .as_ptr() as usize; - - let virt_table_base = - (bar_address + msix_info.table_offset as usize) as *mut MsixTableEntry; - - let mut info = MappedMsixRegs { - virt_table_base: NonNull::new(virt_table_base).unwrap(), - info: msix_info, - }; + let mut info = unsafe { msix_info.map_and_mask_all(pcid_handle) }; // Allocate one msi vector. @@ -103,7 +48,6 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { // primary interrupter let k = 0; - assert_eq!(std::mem::size_of::(), 16); let table_entry_pointer = info.table_entry_pointer(k); let destination_id = read_bsp_apic_id().expect("rtl8168d: failed to read BSP apic id"); @@ -119,6 +63,8 @@ fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> File { log::info!("Enabled MSI-X"); method + } else if has_msi { + allocate_first_msi_interrupt_on_bsp(pcid_handle) } else if let Some(irq) = pci_config.func.legacy_interrupt_line { // legacy INTx# interrupt pins. irq.irq_handle("rtl8168d") diff --git a/pcid/src/driver_handler.rs b/pcid/src/driver_handler.rs index 435c225937..f70a7f6da3 100644 --- a/pcid/src/driver_handler.rs +++ b/pcid/src/driver_handler.rs @@ -7,7 +7,7 @@ use crate::cfg_access::Pcie; pub struct DriverHandler<'a> { func: PciFunction, endpoint_header: &'a mut EndpointHeader, - capabilities: Vec, + capabilities: &'a mut [PciCapability], pcie: &'a Pcie, } @@ -16,7 +16,7 @@ impl<'a> DriverHandler<'a> { pub fn new( func: PciFunction, endpoint_header: &'a mut EndpointHeader, - capabilities: Vec, + capabilities: &'a mut [PciCapability], pcie: &'a Pcie, ) -> Self { DriverHandler { @@ -36,8 +36,11 @@ impl<'a> DriverHandler<'a> { #[forbid(non_exhaustive_omitted_patterns)] match request { PcidClientRequest::EnableDevice => { - self.func.legacy_interrupt_line = - crate::enable_function(&self.pcie, &mut self.endpoint_header); + self.func.legacy_interrupt_line = crate::enable_function( + &self.pcie, + &mut self.endpoint_header, + &mut self.capabilities, + ); PcidClientResponse::EnabledDevice } diff --git a/pcid/src/driver_interface/irq_helpers.rs b/pcid/src/driver_interface/irq_helpers.rs index 6c7b010485..6a91bd1000 100644 --- a/pcid/src/driver_interface/irq_helpers.rs +++ b/pcid/src/driver_interface/irq_helpers.rs @@ -202,3 +202,28 @@ pub fn allocate_single_interrupt_vector_for_msi(cpu_id: usize) -> (MsiAddrAndDat interrupt_handle, ) } + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub fn allocate_first_msi_interrupt_on_bsp( + pcid_handle: &mut crate::driver_interface::PciFunctionHandle, +) -> File { + use crate::driver_interface::{MsiSetFeatureInfo, PciFeature, SetFeatureInfo}; + + // TODO: Allow allocation of up to 32 vectors. + + let destination_id = read_bsp_apic_id().expect("failed to read BSP apic id"); + let (msg_addr_and_data, interrupt_handle) = + allocate_single_interrupt_vector_for_msi(destination_id); + + let set_feature_info = MsiSetFeatureInfo { + multi_message_enable: Some(0), + message_address_and_data: Some(msg_addr_and_data), + mask_bits: None, + }; + pcid_handle.set_feature_info(SetFeatureInfo::Msi(set_feature_info)); + + pcid_handle.enable_feature(PciFeature::Msi); + log::info!("Enabled MSI"); + + interrupt_handle +} diff --git a/pcid/src/driver_interface/msi.rs b/pcid/src/driver_interface/msi.rs index 822ac0976c..0ca68ec556 100644 --- a/pcid/src/driver_interface/msi.rs +++ b/pcid/src/driver_interface/msi.rs @@ -1,6 +1,8 @@ use std::fmt; +use std::ptr::NonNull; use crate::driver_interface::PciBar; +use crate::PciFunctionHandle; use common::io::{Io, Mmio}; use serde::{Deserialize, Serialize}; @@ -32,7 +34,32 @@ pub struct MsixInfo { } impl MsixInfo { - pub fn validate(&self, bars: [PciBar; 6]) { + pub unsafe fn map_and_mask_all(self, pcid_handle: &mut PciFunctionHandle) -> MappedMsixRegs { + self.validate(pcid_handle.config().func.bars); + + let virt_table_base = unsafe { + pcid_handle + .map_bar(self.table_bar) + .ptr + .as_ptr() + .byte_add(self.table_offset as usize) + }; + + let mut info = MappedMsixRegs { + virt_table_base: NonNull::new(virt_table_base.cast::()).unwrap(), + info: self, + }; + + // Mask all interrupts in case some earlier driver/os already unmasked them (according to + // the PCI Local Bus spec 3.0, they are masked after system reset). + for i in 0..info.info.table_size { + info.table_entry_pointer(i.into()).mask(); + } + + info + } + + fn validate(&self, bars: [PciBar; 6]) { if self.table_bar > 5 { panic!( "MSI-X Table BIR contained a reserved enum value: {}", @@ -78,6 +105,21 @@ impl MsixInfo { } } +pub struct MappedMsixRegs { + pub virt_table_base: NonNull, + pub info: MsixInfo, +} +impl MappedMsixRegs { + pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry { + &mut *self.virt_table_base.as_ptr().add(k) + } + + pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry { + assert!(k < self.info.table_size as usize); + unsafe { self.table_entry_pointer_unchecked(k) } + } +} + #[repr(C, packed)] pub struct MsixTableEntry { pub addr_lo: Mmio, @@ -86,6 +128,10 @@ pub struct MsixTableEntry { pub vec_ctl: Mmio, } +const _: () = { + assert!(size_of::() == 16); +}; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub mod x86 { #[repr(u8)] diff --git a/pcid/src/main.rs b/pcid/src/main.rs index 2035ac497b..bbfb8ba301 100644 --- a/pcid/src/main.rs +++ b/pcid/src/main.rs @@ -109,6 +109,7 @@ fn handle_parsed_header( fn enable_function( pcie: &Pcie, endpoint_header: &mut EndpointHeader, + capabilities: &mut [PciCapability], ) -> Option { // Enable bus mastering, memory space, and I/O space endpoint_header.update_command(pcie, |cmd| { @@ -117,6 +118,19 @@ fn enable_function( | CommandRegister::IO_ENABLE }); + // Disable MSI and MSI-X in case a previous driver instance enabled them. + for capability in capabilities { + match capability { + PciCapability::Msi(capability) => { + capability.set_enabled(false, pcie); + } + PciCapability::MsiX(capability) => { + capability.set_enabled(false, pcie); + } + _ => {} + } + } + // Set IRQ line to 9 if not set let mut irq = 0xFF; let mut interrupt_pin = 0xFF; diff --git a/pcid/src/scheme.rs b/pcid/src/scheme.rs index 135a067b73..dcf3a26a3f 100644 --- a/pcid/src/scheme.rs +++ b/pcid/src/scheme.rs @@ -242,8 +242,11 @@ impl PciScheme { if func.enabled { return Err(Error::new(ENOLCK)); } - func.inner.legacy_interrupt_line = - crate::enable_function(&self.pcie, &mut func.endpoint_header); + func.inner.legacy_interrupt_line = crate::enable_function( + &self.pcie, + &mut func.endpoint_header, + &mut func.capabilities, + ); func.enabled = true; Handle::Channel { addr, @@ -287,7 +290,7 @@ impl PciScheme { let response = crate::driver_handler::DriverHandler::new( func.inner.clone(), &mut func.endpoint_header, - func.capabilities.clone(), + &mut func.capabilities, &*pci_state, ) .respond(request); diff --git a/storage/nvmed/src/main.rs b/storage/nvmed/src/main.rs index 6966570a52..56e307769d 100644 --- a/storage/nvmed/src/main.rs +++ b/storage/nvmed/src/main.rs @@ -1,14 +1,10 @@ -#![cfg_attr(target_arch = "aarch64", feature(stdarch_arm_hints))] // Required for yield instruction -#![cfg_attr(target_arch = "riscv64", feature(riscv_ext_intrinsics))] // Required for pause instruction - use std::cell::RefCell; use std::fs::File; use std::io::{self, Read, Write}; use std::os::fd::AsRawFd; -use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; -use std::{slice, usize}; +use std::usize; use driver_block::{Disk, DiskScheme}; use pcid_interface::{PciFeature, PciFeatureInfo, PciFunction, PciFunctionHandle}; @@ -39,35 +35,17 @@ fn get_int_method( // TODO: Allocate more than one vector when possible and useful. if has_msix { // Extended message signaled interrupts. - use self::nvme::MappedMsixRegs; - use pcid_interface::msi::MsixTableEntry; let msix_info = match pcid_handle.feature_info(PciFeature::MsiX) { PciFeatureInfo::MsiX(msix) => msix, _ => unreachable!(), }; - msix_info.validate(function.bars); - fn bar_base(pcid_handle: &mut PciFunctionHandle, bir: u8) -> Result> { - Ok(unsafe { pcid_handle.map_bar(bir) }.ptr) - } - let table_bar_base: *mut u8 = bar_base(pcid_handle, msix_info.table_bar)?.as_ptr(); - let table_base = unsafe { table_bar_base.offset(msix_info.table_offset as isize) }; - - let vector_count = msix_info.table_size; - let table_entries: &'static mut [MsixTableEntry] = unsafe { - slice::from_raw_parts_mut(table_base as *mut MsixTableEntry, vector_count as usize) - }; - - // Mask all interrupts in case some earlier driver/os already unmasked them (according to - // the PCI Local Bus spec 3.0, they are masked after system reset). - for table_entry in table_entries.iter_mut() { - table_entry.mask(); - } + let mut info = unsafe { msix_info.map_and_mask_all(pcid_handle) }; pcid_handle.enable_feature(PciFeature::MsiX); let (msix_vector_number, irq_handle) = { - let entry: &mut MsixTableEntry = &mut table_entries[0]; + let entry = info.table_entry_pointer(0); let bsp_cpu_id = irq_helpers::read_bsp_apic_id().expect("nvmed: failed to read APIC ID"); @@ -79,10 +57,7 @@ fn get_int_method( (0, irq_handle) }; - let interrupt_method = InterruptMethod::MsiX(MappedMsixRegs { - info: msix_info, - table: table_entries, - }); + let interrupt_method = InterruptMethod::MsiX(info); let interrupt_sources = InterruptSources::MsiX(std::iter::once((msix_vector_number, irq_handle)).collect()); @@ -95,22 +70,8 @@ fn get_int_method( _ => unreachable!(), }; - let (msi_vector_number, irq_handle) = { - use pcid_interface::{MsiSetFeatureInfo, SetFeatureInfo}; - - let bsp_cpu_id = - irq_helpers::read_bsp_apic_id().expect("nvmed: failed to read BSP APIC ID"); - let (msg_addr_and_data, irq_handle) = - irq_helpers::allocate_single_interrupt_vector_for_msi(bsp_cpu_id); - - pcid_handle.set_feature_info(SetFeatureInfo::Msi(MsiSetFeatureInfo { - message_address_and_data: Some(msg_addr_and_data), - multi_message_enable: Some(0), // enable 2^0=1 vectors - mask_bits: None, - })); - - (0, irq_handle) - }; + let msi_vector_number = 0; + let irq_handle = irq_helpers::allocate_first_msi_interrupt_on_bsp(pcid_handle); let interrupt_method = InterruptMethod::Msi { msi_info, diff --git a/storage/nvmed/src/nvme/mod.rs b/storage/nvmed/src/nvme/mod.rs index 513814c9d0..312a0eb3e8 100644 --- a/storage/nvmed/src/nvme/mod.rs +++ b/storage/nvmed/src/nvme/mod.rs @@ -21,33 +21,9 @@ pub mod queues; use self::executor::NvmeExecutor; pub use self::queues::{NvmeCmd, NvmeCmdQueue, NvmeComp, NvmeCompQueue}; -use pcid_interface::msi::{MsiInfo, MsixInfo, MsixTableEntry}; +use pcid_interface::msi::{MappedMsixRegs, MsiInfo}; use pcid_interface::PciFunctionHandle; -#[cfg(target_arch = "aarch64")] -#[inline(always)] -pub(crate) unsafe fn pause() { - std::arch::aarch64::__yield(); -} - -#[cfg(target_arch = "x86")] -#[inline(always)] -pub(crate) unsafe fn pause() { - std::arch::x86::_mm_pause(); -} - -#[cfg(target_arch = "x86_64")] -#[inline(always)] -pub(crate) unsafe fn pause() { - std::arch::x86_64::_mm_pause(); -} - -#[cfg(target_arch = "riscv64")] -#[inline(always)] -pub(crate) unsafe fn pause() { - std::arch::riscv64::pause(); -} - /// Used in conjunction with `InterruptMethod`, primarily by the CQ executor. #[derive(Debug)] pub enum InterruptSources { @@ -133,11 +109,6 @@ impl InterruptMethod { } } -pub struct MappedMsixRegs { - pub info: MsixInfo, - pub table: &'static mut [MsixTableEntry], -} - #[repr(C, packed)] pub struct NvmeRegs { /// Controller Capabilities @@ -304,7 +275,7 @@ impl Nvme { let csts = self.regs.get_mut().csts.read(); log::trace!("CSTS: {:X}", csts); if csts & 1 == 1 { - pause(); + std::hint::spin_loop(); } else { break; } @@ -316,7 +287,7 @@ impl Nvme { self.regs.get_mut().intmc.write(0x0000_0001); } &mut InterruptMethod::MsiX(ref mut cfg) => { - cfg.table[0].unmask(); + cfg.table_entry_pointer(0).unmask(); } } @@ -369,7 +340,7 @@ impl Nvme { let csts = self.regs.get_mut().csts.read(); log::debug!("CSTS: {:X}", csts); if csts & 1 == 0 { - pause(); + std::hint::spin_loop(); } else { break; } @@ -442,10 +413,7 @@ impl Nvme { } &mut InterruptMethod::MsiX(ref mut cfg) => { for (vector, mask) in vectors { - cfg.table - .get_mut(vector as usize) - .expect("nvmed: internal error: MSI-X vector out of range") - .set_masked(mask); + cfg.table_entry_pointer(vector.into()).set_masked(mask); } } } diff --git a/storage/nvmed/src/nvme/queues.rs b/storage/nvmed/src/nvme/queues.rs index 9d9cf2e2cb..a3712aeb74 100644 --- a/storage/nvmed/src/nvme/queues.rs +++ b/storage/nvmed/src/nvme/queues.rs @@ -87,7 +87,7 @@ impl NvmeCompQueue { return some; } else { unsafe { - super::pause(); + std::hint::spin_loop(); } } } diff --git a/virtio-core/src/arch/x86.rs b/virtio-core/src/arch/x86.rs index 7275c63e41..751f268e07 100644 --- a/virtio-core/src/arch/x86.rs +++ b/virtio-core/src/arch/x86.rs @@ -1,32 +1,19 @@ use crate::transport::Error; use pcid_interface::irq_helpers::{allocate_single_interrupt_vector_for_msi, read_bsp_apic_id}; -use pcid_interface::msi::MsixTableEntry; -use std::{fs::File, ptr::NonNull}; +use std::fs::File; -use crate::{probe::MappedMsixRegs, MSIX_PRIMARY_VECTOR}; +use crate::MSIX_PRIMARY_VECTOR; use pcid_interface::*; pub fn enable_msix(pcid_handle: &mut PciFunctionHandle) -> Result { - let pci_config = pcid_handle.config(); - // Extended message signaled interrupts. let msix_info = match pcid_handle.feature_info(PciFeature::MsiX) { PciFeatureInfo::MsiX(capability) => capability, _ => unreachable!(), }; - msix_info.validate(pci_config.func.bars); - - let bar_address = unsafe { pcid_handle.map_bar(msix_info.table_bar) } - .ptr - .as_ptr() as usize; - let virt_table_base = (bar_address + msix_info.table_offset as usize) as *mut MsixTableEntry; - - let mut info = MappedMsixRegs { - virt_table_base: NonNull::new(virt_table_base).unwrap(), - info: msix_info, - }; + let mut info = unsafe { msix_info.map_and_mask_all(pcid_handle) }; // Allocate the primary MSI vector. // FIXME allow the driver to register multiple MSI-X vectors diff --git a/virtio-core/src/probe.rs b/virtio-core/src/probe.rs index be3977d96b..da95234636 100644 --- a/virtio-core/src/probe.rs +++ b/virtio-core/src/probe.rs @@ -1,8 +1,6 @@ use std::fs::File; -use std::ptr::NonNull; use std::sync::Arc; -use pcid_interface::msi::{MsixInfo, MsixTableEntry}; use pcid_interface::*; use crate::spec::*; @@ -20,23 +18,6 @@ pub struct Device { unsafe impl Send for Device {} unsafe impl Sync for Device {} -pub struct MappedMsixRegs { - pub virt_table_base: NonNull, - pub info: MsixInfo, -} - -impl MappedMsixRegs { - pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry { - &mut *self.virt_table_base.as_ptr().add(k) - } - - pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry { - assert!(k < self.info.table_size as usize); - unsafe { self.table_entry_pointer_unchecked(k) } - } -} - -static_assertions::const_assert_eq!(std::mem::size_of::(), 16); pub const MSIX_PRIMARY_VECTOR: u16 = 0; diff --git a/xhcid/src/main.rs b/xhcid/src/main.rs index 800614e2e0..74b83a3ff0 100644 --- a/xhcid/src/main.rs +++ b/xhcid/src/main.rs @@ -26,16 +26,14 @@ extern crate bitflags; use std::fs::File; -use std::ptr::NonNull; use std::sync::{Arc, Mutex}; -#[cfg(target_arch = "x86_64")] -use pcid_interface::irq_helpers::allocate_single_interrupt_vector_for_msi; use pcid_interface::irq_helpers::read_bsp_apic_id; -use pcid_interface::msi::MsixTableEntry; -use pcid_interface::{ - MsiSetFeatureInfo, PciFeature, PciFeatureInfo, PciFunctionHandle, SetFeatureInfo, +#[cfg(target_arch = "x86_64")] +use pcid_interface::irq_helpers::{ + allocate_first_msi_interrupt_on_bsp, allocate_single_interrupt_vector_for_msi, }; +use pcid_interface::{PciFeature, PciFeatureInfo, PciFunctionHandle}; use redox_scheme::{RequestKind, SignalBehavior, Socket}; @@ -50,10 +48,7 @@ mod usb; mod xhci; #[cfg(target_arch = "x86_64")] -fn get_int_method( - pcid_handle: &mut PciFunctionHandle, - bar0_address: usize, -) -> (Option, InterruptMethod) { +fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> (Option, InterruptMethod) { let pci_config = pcid_handle.config(); let all_pci_features = pcid_handle.fetch_all_features(); @@ -62,46 +57,12 @@ fn get_int_method( let has_msi = all_pci_features.iter().any(|feature| feature.is_msi()); let has_msix = all_pci_features.iter().any(|feature| feature.is_msix()); - if has_msi && !has_msix { - let mut capability = match pcid_handle.feature_info(PciFeature::Msi) { - PciFeatureInfo::Msi(s) => s, - PciFeatureInfo::MsiX(_) => panic!(), - }; - // TODO: Allow allocation of up to 32 vectors. - - // TODO: Find a way to abstract this away, potantially as a helper module for - // pcid_interface, so that this can be shared between nvmed, xhcid, ixgebd, etc.. - - let destination_id = read_bsp_apic_id().expect("xhcid: failed to read BSP apic id"); - let (msg_addr_and_data, interrupt_handle) = - allocate_single_interrupt_vector_for_msi(destination_id); - - let set_feature_info = MsiSetFeatureInfo { - multi_message_enable: Some(0), - message_address_and_data: Some(msg_addr_and_data), - mask_bits: None, - }; - pcid_handle.set_feature_info(SetFeatureInfo::Msi(set_feature_info)); - - pcid_handle.enable_feature(PciFeature::Msi); - log::debug!("Enabled MSI"); - - (Some(interrupt_handle), InterruptMethod::Msi) - } else if has_msix { + if has_msix { let msix_info = match pcid_handle.feature_info(PciFeature::MsiX) { PciFeatureInfo::Msi(_) => panic!(), PciFeatureInfo::MsiX(s) => s, }; - msix_info.validate(pci_config.func.bars); - - assert_eq!(msix_info.table_bar, 0); - let virt_table_base = - (bar0_address + msix_info.table_offset as usize) as *mut MsixTableEntry; - - let mut info = xhci::MappedMsixRegs { - virt_table_base: NonNull::new(virt_table_base).unwrap(), - info: msix_info, - }; + let mut info = unsafe { msix_info.map_and_mask_all(pcid_handle) }; // Allocate one msi vector. @@ -109,7 +70,6 @@ fn get_int_method( // primary interrupter let k = 0; - assert_eq!(std::mem::size_of::(), 16); let table_entry_pointer = info.table_entry_pointer(k); let destination_id = read_bsp_apic_id().expect("xhcid: failed to read BSP apic id"); @@ -128,6 +88,9 @@ fn get_int_method( log::debug!("Enabled MSI-X"); method + } else if has_msi { + let interrupt_handle = allocate_first_msi_interrupt_on_bsp(pcid_handle); + (Some(interrupt_handle), InterruptMethod::Msi) } else if let Some(irq) = pci_config.func.legacy_interrupt_line { log::debug!("Legacy IRQ {}", irq); @@ -141,10 +104,7 @@ fn get_int_method( //TODO: MSI on non-x86_64? #[cfg(not(target_arch = "x86_64"))] -fn get_int_method( - pcid_handle: &mut PciFunctionHandle, - address: usize, -) -> (Option, InterruptMethod) { +fn get_int_method(pcid_handle: &mut PciFunctionHandle) -> (Option, InterruptMethod) { let pci_config = pcid_handle.config(); if let Some(irq) = pci_config.func.legacy_interrupt_line { @@ -179,7 +139,7 @@ fn daemon(daemon: redox_daemon::Daemon) -> ! { let address = unsafe { pcid_handle.map_bar(0) }.ptr.as_ptr() as usize; - let (irq_file, interrupt_method) = (None, InterruptMethod::Polling); //get_int_method(&mut pcid_handle, address); + let (irq_file, interrupt_method) = (None, InterruptMethod::Polling); //get_int_method(&mut pcid_handle); //TODO: Fix interrupts. println!(" + XHCI {}", pci_config.func.display()); diff --git a/xhcid/src/xhci/mod.rs b/xhcid/src/xhci/mod.rs index 3d3798a312..bbbfb9ffbd 100644 --- a/xhcid/src/xhci/mod.rs +++ b/xhcid/src/xhci/mod.rs @@ -12,11 +12,11 @@ use std::collections::BTreeMap; use std::convert::TryFrom; use std::fs::File; -use std::ptr::NonNull; use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex, MutexGuard}; use std::{mem, process, slice, thread}; +use pcid_interface::msi::MappedMsixRegs; use syscall::error::{Error, Result, EBADF, EBADMSG, EIO, ENOENT}; use syscall::{EAGAIN, PAGE_SIZE}; @@ -28,7 +28,6 @@ use serde::Deserialize; use crate::usb; -use pcid_interface::msi::{MsixInfo, MsixTableEntry}; use pcid_interface::PciFunctionHandle; mod capability; @@ -78,20 +77,6 @@ pub enum InterruptMethod { MsiX(Mutex), } -pub struct MappedMsixRegs { - pub virt_table_base: NonNull, - pub info: MsixInfo, -} -impl MappedMsixRegs { - pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry { - &mut *self.virt_table_base.as_ptr().offset(k as isize) - } - pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry { - assert!(k < self.info.table_size as usize); - unsafe { self.table_entry_pointer_unchecked(k) } - } -} - impl Xhci { /// Gets descriptors, before the port state is initiated. async fn get_desc_raw(