diff --git a/virtio-core/src/arch/x86.rs b/virtio-core/src/arch/x86.rs index 08483e9346..56b33d8fd8 100644 --- a/virtio-core/src/arch/x86.rs +++ b/virtio-core/src/arch/x86.rs @@ -1,10 +1,4 @@ -use crate::{ - legacy_transport::LegacyTransport, - reinit, - transport::Error, - utils::VolatileCell, - Device, -}; +use crate::{legacy_transport::LegacyTransport, reinit, transport::Error, Device}; use std::fs::File; use pcid_interface::*; @@ -13,16 +7,14 @@ pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result { panic!("virtio-core: x86 doesn't support enable_msix") } -pub fn probe_legacy_port_transport<'a>( +pub fn probe_legacy_port_transport( pci_header: &PciHeader, pcid_handle: &mut PcidServerHandle, -) -> Result, Error> { +) -> Result { if let PciBar::Port(port) = pci_header.get_bar(0) { unsafe { syscall::iopl(3).expect("virtio: failed to set I/O privilege level") }; log::warn!("virtio: using legacy transport"); - static SHIM: VolatileCell = VolatileCell::new(0); - let transport = LegacyTransport::new(port); // Setup interrupts. @@ -38,7 +30,6 @@ pub fn probe_legacy_port_transport<'a>( let device = Device { transport, irq_handle, - isr: &SHIM, device_space: core::ptr::null_mut(), }; diff --git a/virtio-core/src/arch/x86_64.rs b/virtio-core/src/arch/x86_64.rs index 1909c20aec..e8200da934 100644 --- a/virtio-core/src/arch/x86_64.rs +++ b/virtio-core/src/arch/x86_64.rs @@ -1,13 +1,8 @@ -use crate::{ - reinit, - transport::{Error}, - utils::VolatileCell, - Device, legacy_transport::LegacyTransport, -}; +use crate::{legacy_transport::LegacyTransport, reinit, transport::Error, Device}; -use pcid_interface::msi::{self, MsixTableEntry}; use pcid_interface::irq_helpers::{allocate_single_interrupt_vector, read_bsp_apic_id}; -use std::{ptr::NonNull, fs::File}; +use pcid_interface::msi::{self, MsixTableEntry}; +use std::{fs::File, ptr::NonNull}; use syscall::Io; @@ -16,7 +11,6 @@ use crate::{probe::MsixInfo, MSIX_PRIMARY_VECTOR}; use pcid_interface::*; pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result { - let pci_config = pcid_handle.fetch_config()?; // Extended message signaled interrupts. @@ -66,6 +60,8 @@ pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result { }; // Allocate the primary MSI vector. + // FIXME allow the driver to register multiple MSI-X vectors + // FIXME move this MSI-X registering code into pcid_interface or pcid itself let interrupt_handle = { let table_entry_pointer = info.table_entry_pointer(MSIX_PRIMARY_VECTOR as usize); @@ -80,7 +76,8 @@ pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result { .unwrap() .expect("virtio_core: interrupt vector exhaustion"); - let msg_data = msi::x86_64::message_data_edge_triggered(msi::x86_64::DeliveryMode::Fixed, vector); + let msg_data = + msi::x86_64::message_data_edge_triggered(msi::x86_64::DeliveryMode::Fixed, vector); table_entry_pointer.addr_lo.write(addr); table_entry_pointer.addr_hi.write(0); @@ -98,16 +95,14 @@ pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result { Ok(interrupt_handle) } -pub fn probe_legacy_port_transport<'a>( +pub fn probe_legacy_port_transport( pci_header: &PciHeader, pcid_handle: &mut PcidServerHandle, -) -> Result, Error> { +) -> Result { if let PciBar::Port(port) = pci_header.get_bar(0) { unsafe { syscall::iopl(3).expect("virtio: failed to set I/O privilege level") }; log::warn!("virtio: using legacy transport"); - static SHIM: VolatileCell = VolatileCell::new(0); - let transport = LegacyTransport::new(port); // Setup interrupts. @@ -123,7 +118,6 @@ pub fn probe_legacy_port_transport<'a>( let device = Device { transport, irq_handle, - isr: &SHIM, device_space: core::ptr::null_mut(), }; diff --git a/virtio-core/src/legacy_transport.rs b/virtio-core/src/legacy_transport.rs index e0a98e784f..45341b103f 100644 --- a/virtio-core/src/legacy_transport.rs +++ b/virtio-core/src/legacy_transport.rs @@ -144,7 +144,7 @@ impl Transport for LegacyTransport { vector, ); - spawn_irq_thread(irq_handle, &queue)?; + spawn_irq_thread(irq_handle, &queue); Ok(queue) } diff --git a/virtio-core/src/probe.rs b/virtio-core/src/probe.rs index 1509187450..157ffa0f1f 100644 --- a/virtio-core/src/probe.rs +++ b/virtio-core/src/probe.rs @@ -1,25 +1,24 @@ - -use std::ptr::NonNull; use std::fs::File; +use std::ptr::NonNull; use std::sync::Arc; + +use pcid_interface::msi::{MsixCapability, MsixTableEntry}; use pcid_interface::*; -use pcid_interface::msi::{self, MsixTableEntry, MsixCapability}; use crate::spec::*; use crate::transport::{Error, StandardTransport, Transport}; -use crate::utils::{align_down, VolatileCell}; +use crate::utils::align_down; -pub struct Device<'a> { +pub struct Device { pub transport: Arc, pub device_space: *const u8, pub irq_handle: File, - pub isr: &'a VolatileCell, } // FIXME(andypython): `device_space` should not be `Send` nor `Sync`. Take // it out of `Device`. -unsafe impl Send for Device<'_> {} -unsafe impl Sync for Device<'_> {} +unsafe impl Send for Device {} +unsafe impl Sync for Device {} pub struct MsixInfo { pub virt_table_base: NonNull, @@ -60,7 +59,7 @@ fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result { /// /// ## Panics /// This function panics if the device is not a virtio device. -pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result, Error> { +pub fn probe_device(pcid_handle: &mut PcidServerHandle) -> Result { let pci_config = pcid_handle.fetch_config()?; let pci_header = pcid_handle.fetch_header()?; @@ -71,7 +70,6 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result let mut common_addr = None; let mut notify_addr = None; - let mut isr_addr = None; let mut device_addr = None; for capability in pcid_handle @@ -89,7 +87,7 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result let capability = unsafe { &*(capability.data.as_ptr() as *const PciCapability) }; match capability.cfg_type { - CfgType::Common | CfgType::Notify | CfgType::Isr | CfgType::Device => {} + CfgType::Common | CfgType::Notify | CfgType::Device => {} _ => continue, } @@ -131,15 +129,13 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result // SAFETY: The capability type is `Notify`, so its safe to access // the `notify_multiplier` field. - let multiplier = unsafe { capability.notify_multiplier() }; + let multiplier = unsafe { + (&*(capability as *const PciCapability as *const PciCapabilityNotify)) + .notify_off_multiplier() + }; notify_addr = Some((address, multiplier)); } - CfgType::Isr => { - debug_assert!(isr_addr.is_none()); - isr_addr = Some(address); - } - CfgType::Device => { debug_assert!(device_addr.is_none()); device_addr = Some(address); @@ -151,13 +147,10 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result log::trace!("virtio-core::device-probe: {capability:?}"); } - if let ( - Some(common_addr), - Some(isr_addr), - Some(device_addr), - Some((notify_addr, notify_multiplier)), - ) = (common_addr, isr_addr, device_addr, notify_addr) + if let (Some(common_addr), Some(device_addr), Some((notify_addr, notify_multiplier))) = + (common_addr, device_addr, notify_addr) { + // FIXME this is explicitly allowed by the virtio specification to happen assert!( notify_multiplier != 0, "virtio-core::device_probe: device uses the same Queue Notify addresses for all queues" @@ -165,7 +158,6 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result let common = unsafe { &mut *(common_addr as *mut CommonCfg) }; let device_space = unsafe { &mut *(device_addr as *mut u8) }; - let isr = unsafe { &*(isr_addr as *mut VolatileCell) }; let transport = StandardTransport::new( common, @@ -190,7 +182,6 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result transport, device_space, irq_handle, - isr, }; device.transport.reset(); @@ -202,7 +193,7 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result } } -pub fn reinit<'a>(device: &Device<'a>) -> Result<(), Error> { +pub fn reinit(device: &Device) -> Result<(), Error> { // XXX: According to the virtio specification v1.2, setting the ACKNOWLEDGE and DRIVER bits // in `device_status` is required to be done in two steps. device diff --git a/virtio-core/src/spec.rs b/virtio-core/src/spec.rs deleted file mode 100644 index f4a3d79f8c..0000000000 --- a/virtio-core/src/spec.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html - -use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering}; - -use crate::utils::{IncompleteArrayField, VolatileCell}; -use static_assertions::const_assert_eq; - -#[derive(Debug, Copy, Clone)] -#[repr(u8)] -pub enum CfgType { - /// Common Configuration. - Common = 1, - /// Notifications. - Notify = 2, - /// ISR Status. - Isr = 3, - /// Device specific configuration. - Device = 4, - /// PCI configuration access. - PciConfig = 5, - /// Shared memory region. - SharedMemory = 8, - /// Vendor-specific data. - Vendor = 9, -} - -const_assert_eq!(core::mem::size_of::(), 1); - -#[derive(Debug, Copy, Clone)] -#[repr(C, packed)] -pub struct PciCapability { - /// Identifies the structure. - pub cfg_type: CfgType, - /// Where to find it. - pub bar: u8, - /// Pad to a full dword. - pub padding: [u8; 3], - /// Offset within the bar. - pub offset: u32, - /// Length of the structure, in bytes. - pub length: u32, - notify_multiplier: u32, -} - -// The size of `PciCapability` is 13 bytes since the generic -// PCI fields are *not* included. -const_assert_eq!(core::mem::size_of::(), 17); - -impl PciCapability { - /// ## Safety - /// Undefined if accessed from a capability type other than [`CfgType::Notify`]. - pub unsafe fn notify_multiplier(&self) -> u32 { - self.notify_multiplier - } -} - -bitflags::bitflags! { - #[derive(Debug, Copy, Clone, PartialEq)] - #[repr(transparent)] - pub struct DeviceStatusFlags: u8 { - /// Indicates that the guest OS has found the device and recognized it as a - /// valid device. - const ACKNOWLEDGE = 1; - /// Indicates that the guest OS knows how to drive the device. - const DRIVER = 2; - /// Indicates that something went wrong in the guest and it has given up on - /// the device. - const FAILED = 128; - /// Indicates that the driver has acknowledged all the features it understands - /// and feature negotiation is complete. - const FEATURES_OK = 8; - /// Indicates that the driver is set up and ready to drive the device. - const DRIVER_OK = 4; - /// Indicates that the device has experienced an error from which it can’t recover. - const DEVICE_NEEDS_RESET = 64; - } -} - -#[derive(Debug)] -#[repr(C)] -pub struct CommonCfg { - // About the whole device. - pub device_feature_select: VolatileCell, // read-write - pub device_feature: VolatileCell, // read-only for driver - pub driver_feature_select: VolatileCell, // read-write - pub driver_feature: VolatileCell, // read-write - pub msix_config: VolatileCell, // read-write - pub num_queues: VolatileCell, // read-only for driver - pub device_status: VolatileCell, // read-write - pub config_generation: VolatileCell, // read-only for driver - - // About a specific virtqueue. - pub queue_select: VolatileCell, // read-write - pub queue_size: VolatileCell, // read-write - pub queue_msix_vector: VolatileCell, // read-write - pub queue_enable: VolatileCell, // read-write - pub queue_notify_off: VolatileCell, // read-only for driver - pub queue_desc: VolatileCell, // read-write - pub queue_driver: VolatileCell, // read-write - pub queue_device: VolatileCell, // read-write -} - -const_assert_eq!(core::mem::size_of::(), 56); - -bitflags::bitflags! { - #[derive(Debug, Copy, Clone)] - #[repr(transparent)] - pub struct DescriptorFlags: u16 { - /// The next field contains linked buffer index. - const NEXT = 1 << 0; - /// The buffer is write-only (otherwise read-only). - const WRITE_ONLY = 1 << 1; - /// The buffer contains a list of buffer descriptors. - const INDIRECT = 1 << 2; - } -} - -#[repr(C)] -pub struct Descriptor { - /// Address (guest-physical). - pub address: AtomicU64, - /// Size of the descriptor. - pub size: AtomicU32, - flags: AtomicU16, - /// Index of next desciptor in chain. - pub next: AtomicU16, -} - -const_assert_eq!(core::mem::size_of::(), 16); - -impl Descriptor { - pub fn set_addr(&self, addr: u64) { - self.address.store(addr, Ordering::SeqCst) - } - - pub fn set_size(&self, size: u32) { - self.size.store(size, Ordering::SeqCst) - } - - pub fn set_next(&self, next: Option) { - self.next.store(next.unwrap_or_default(), Ordering::SeqCst) - } - - pub fn set_flags(&self, flags: DescriptorFlags) { - self.flags.store(flags.bits(), Ordering::SeqCst) - } - - pub fn next(&self) -> u16 { - self.next.load(Ordering::SeqCst) - } - - pub fn flags(&self) -> DescriptorFlags { - DescriptorFlags::from_bits_truncate(self.flags.load(Ordering::SeqCst)) - } -} - -/// This indicates compliance with the version 1 VirtIO specification. -/// -/// See `6.1 Driver Requirements: Reserved Feature Bits` section of the VirtIO -/// specification for more information. -pub const VIRTIO_F_VERSION_1: u32 = 32; -pub const VIRTIO_NET_F_MAC: u32 = 5; - -// ======== Available Ring ======== -// -// XXX: The driver uses the available ring to offer buffers to the -// device. Each ring entry refers to the head of a descriptor -// chain. -#[repr(C)] -pub struct AvailableRingElement { - pub table_index: AtomicU16, -} - -impl AvailableRingElement { - pub fn set_table_index(&self, index: u16) { - self.table_index.store(index, Ordering::SeqCst) - } -} - -const_assert_eq!(core::mem::size_of::(), 2); - -#[repr(C)] -pub struct AvailableRing { - pub flags: VolatileCell, - pub head_index: AtomicU16, - pub elements: IncompleteArrayField, -} - -const_assert_eq!(core::mem::size_of::(), 4); - -impl Default for AvailableRing { - fn default() -> Self { - Self { - flags: VolatileCell::new(0), - head_index: AtomicU16::new(0), - elements: IncompleteArrayField::new(), - } - } -} - -#[repr(C)] -pub struct AvailableRingExtra { - pub avail_event: VolatileCell, // Only if `VIRTIO_F_EVENT_IDX` -} - -const_assert_eq!(core::mem::size_of::(), 2); - -// ======== Used Ring ======== -#[repr(C)] -pub struct UsedRingElement { - pub table_index: VolatileCell, - pub written: VolatileCell, -} - -const_assert_eq!(core::mem::size_of::(), 8); - -#[repr(C)] -pub struct UsedRing { - pub flags: VolatileCell, - pub head_index: VolatileCell, - pub elements: IncompleteArrayField, -} - -const_assert_eq!(core::mem::size_of::(), 4); - -impl Default for UsedRing { - fn default() -> Self { - Self { - flags: VolatileCell::new(0), - head_index: VolatileCell::new(0), - elements: IncompleteArrayField::new(), - } - } -} - -#[repr(C)] -pub struct UsedRingExtra { - pub event_index: VolatileCell, -} - -// ======== Utils ======== -pub struct Buffer { - pub(crate) buffer: usize, - pub(crate) size: usize, - pub(crate) flags: DescriptorFlags, -} - -impl Buffer { - pub fn new(val: &common::dma::Dma) -> Self { - Self { - buffer: val.physical(), - size: core::mem::size_of::(), - flags: DescriptorFlags::empty(), - } - } - - pub fn new_unsized(val: &common::dma::Dma<[T]>) -> Self { - Self { - buffer: val.physical(), - size: core::mem::size_of::() * val.len(), - flags: DescriptorFlags::empty(), - } - } - - pub fn new_sized(val: &common::dma::Dma<[T]>, size: usize) -> Self { - Self { - buffer: val.physical(), - size, - flags: DescriptorFlags::empty(), - } - } - - pub fn flags(mut self, flags: DescriptorFlags) -> Self { - self.flags = flags; - self - } -} - -/// XXX: The [`DescriptorFlags::NEXT`] flag is set automatically. -pub struct ChainBuilder { - buffers: Vec, -} - -impl ChainBuilder { - pub fn new() -> Self { - Self { - buffers: Vec::new(), - } - } - - pub fn chain(mut self, mut buffer: Buffer) -> Self { - buffer.flags |= DescriptorFlags::NEXT; - self.buffers.push(buffer); - self - } - - pub fn build(mut self) -> Vec { - let last_buffer = self.buffers.last_mut().expect("virtio-core: empty chain"); - last_buffer.flags.remove(DescriptorFlags::NEXT); - - self.buffers - } -} diff --git a/virtio-core/src/spec/mod.rs b/virtio-core/src/spec/mod.rs new file mode 100644 index 0000000000..b78931d2cb --- /dev/null +++ b/virtio-core/src/spec/mod.rs @@ -0,0 +1,56 @@ +//! https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html +//! +//! This file contains comments copied from the VirtIO specification which are +//! licensed under the following conditions: +//! +//! Copyright © OASIS Open 2022. All Rights Reserved. +//! +//! All capitalized terms in the following text have the meanings assigned to them +//! in the OASIS Intellectual Property Rights Policy (the "OASIS IPR Policy"). The +//! full Policy may be found at the OASIS website. +//! +//! This document and translations of it may be copied and furnished to others, +//! and derivative works that comment on or otherwise explain it or assist in its +//! implementation may be prepared, copied, published, and distributed, in whole +//! or in part, without restriction of any kind, provided that the above copyright +//! notice and this section are included on all such copies and derivative works. +//! However, this document itself may not be modified in any way, including by +//! removing the copyright notice or references to OASIS, except as needed for the +//! purpose of developing any document or deliverable produced by an OASIS Technical +//! Committee (in which case the rules applicable to copyrights, as set forth in the +//! OASIS IPR Policy, must be followed) or as required to translate it into languages +//! other than English. + +bitflags::bitflags! { + /// [2.1 Device Status Field](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-110001) + #[derive(Debug, Copy, Clone, PartialEq)] + #[repr(transparent)] + pub struct DeviceStatusFlags: u8 { + /// Indicates that the guest OS has found the device and recognized it as a + /// valid device. + const ACKNOWLEDGE = 1; + /// Indicates that the guest OS knows how to drive the device. + const DRIVER = 2; + /// Indicates that something went wrong in the guest and it has given up on + /// the device. + const FAILED = 128; + /// Indicates that the driver has acknowledged all the features it understands + /// and feature negotiation is complete. + const FEATURES_OK = 8; + /// Indicates that the driver is set up and ready to drive the device. + const DRIVER_OK = 4; + /// Indicates that the device has experienced an error from which it can’t recover. + const DEVICE_NEEDS_RESET = 64; + } +} + +mod split_virtqueue; +pub use split_virtqueue::*; + +// FIXME add [2.8 Packed Virtqueues](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-720008) + +mod transport_pci; +pub use transport_pci::*; + +mod reserved_features; +pub use reserved_features::*; diff --git a/virtio-core/src/spec/reserved_features.rs b/virtio-core/src/spec/reserved_features.rs new file mode 100644 index 0000000000..9f88676767 --- /dev/null +++ b/virtio-core/src/spec/reserved_features.rs @@ -0,0 +1,100 @@ +//! [6 Reserved Feature Bits](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-6600006) +//! +//! This file contains comments copied from the VirtIO specification which are +//! licensed under the following conditions: +//! +//! Copyright © OASIS Open 2022. All Rights Reserved. +//! +//! All capitalized terms in the following text have the meanings assigned to them +//! in the OASIS Intellectual Property Rights Policy (the "OASIS IPR Policy"). The +//! full Policy may be found at the OASIS website. +//! +//! This document and translations of it may be copied and furnished to others, +//! and derivative works that comment on or otherwise explain it or assist in its +//! implementation may be prepared, copied, published, and distributed, in whole +//! or in part, without restriction of any kind, provided that the above copyright +//! notice and this section are included on all such copies and derivative works. +//! However, this document itself may not be modified in any way, including by +//! removing the copyright notice or references to OASIS, except as needed for the +//! purpose of developing any document or deliverable produced by an OASIS Technical +//! Committee (in which case the rules applicable to copyrights, as set forth in the +//! OASIS IPR Policy, must be followed) or as required to translate it into languages +//! other than English. + +/// Negotiating this feature indicates that the driver can use descriptors +/// with the VIRTQ_DESC_F_INDIRECT flag set as described in 2.7.5.3 Indirect +/// Descriptors and 2.8.7 Indirect Flag: Scatter-Gather Support. +pub const VIRTIO_F_INDIRECT_DESC: u32 = 28; + +/// This feature enables the used_event and the avail_event fields as +/// described in 2.7.7, 2.7.8 and 2.8.10. +pub const VIRTIO_F_EVENT_IDX: u32 = 29; + +/// This indicates compliance with this specification, giving a simple way +/// to detect legacy devices or drivers. +pub const VIRTIO_F_VERSION_1: u32 = 32; + +/// This feature indicates that the device can be used on a platform where device +/// access to data in memory is limited and/or translated. E.g. this is the case +/// if the device can be located behind an IOMMU that translates bus addresses +/// from the device into physical addresses in memory, if the device can be limited +/// to only access certain memory addresses or if special commands such as a cache +/// flush can be needed to synchronise data in memory with the device. Whether +/// accesses are actually limited or translated is described by platform-specific +/// means. If this feature bit is set to 0, then the device has same access to +/// memory addresses supplied to it as the driver has. In particular, the device +/// will always use physical addresses matching addresses used by the driver +/// (typically meaning physical addresses used by the CPU) and not translated +/// further, and can access any address supplied to it by the driver. When clear, +/// this overrides any platform-specific description of whether device access is +/// limited or translated in any way, e.g. whether an IOMMU may be present. +pub const VIRTIO_F_ACCESS_PLATFORM: u32 = 33; + +/// This feature indicates support for the packed virtqueue layout as described +/// in 2.8 Packed Virtqueues. +pub const VIRTIO_F_RING_PACKED: u32 = 34; + +/// This feature indicates that all buffers are used by the device in the same order +/// in which they have been made available. +pub const VIRTIO_F_IN_ORDER: u32 = 35; + +/// This feature indicates that memory accesses by the driver and the device are +/// ordered in a way described by the platform. +/// If this feature bit is negotiated, the ordering in effect for any memory +/// accesses by the driver that need to be ordered in a specific way with respect +/// to accesses by the device is the one suitable for devices described by the +/// platform. This implies that the driver needs to use memory barriers suitable +/// for devices described by the platform; e.g. for the PCI transport in the case +/// of hardware PCI devices. +/// +/// If this feature bit is not negotiated, then the device and driver are assumed +/// to be implemented in software, that is they can be assumed to run on identical +/// CPUs in an SMP configuration. Thus a weaker form of memory barriers is sufficient +/// to yield better performance. +pub const VIRTIO_F_ORDER_PLATFORM: u32 = 36; + +/// This feature indicates that the device supports Single Root I/O Virtualization. +/// Currently only PCI devices support this feature. +pub const VIRTIO_F_SR_IOV: u32 = 37; + +/// This feature indicates that the driver passes extra data (besides identifying +/// the virtqueue) in its device notifications. See 2.9 Driver Notifications. +pub const VIRTIO_F_NOTIFICATION_DATA: u32 = 38; + +/// This feature indicates that the driver uses the data provided by the device as +/// a virtqueue identifier in available buffer notifications. As mentioned in section +/// 2.9, when the driver is required to send an available buffer notification to the +/// device, it sends the virtqueue number to be notified. The method of delivering +/// notifications is transport specific. With the PCI transport, the device can +/// optionally provide a per-virtqueue value for the driver to use in driver +/// notifications, instead of the virtqueue number. Some devices may benefit from this +/// flexibility by providing, for example, an internal virtqueue identifier, or an +/// internal offset related to the virtqueue number. +/// +/// This feature indicates the availability of such value. The definition of the data +/// to be provided in driver notification and the delivery method is transport +/// specific. For more details about driver notifications over PCI see 4.1.5.2. +pub const VIRTIO_F_NOTIF_CONFIG_DATA: u32 = 39; + +/// This feature indicates that the driver can reset a queue individually. See 2.6.1. +pub const VIRTIO_F_RING_RESET: u32 = 40; diff --git a/virtio-core/src/spec/split_virtqueue.rs b/virtio-core/src/spec/split_virtqueue.rs new file mode 100644 index 0000000000..b96367111f --- /dev/null +++ b/virtio-core/src/spec/split_virtqueue.rs @@ -0,0 +1,205 @@ +//! [2.7 Split Virtqueues](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-350007) +//! +//! This file contains comments copied from the VirtIO specification which are +//! licensed under the following conditions: +//! +//! Copyright © OASIS Open 2022. All Rights Reserved. +//! +//! All capitalized terms in the following text have the meanings assigned to them +//! in the OASIS Intellectual Property Rights Policy (the "OASIS IPR Policy"). The +//! full Policy may be found at the OASIS website. +//! +//! This document and translations of it may be copied and furnished to others, +//! and derivative works that comment on or otherwise explain it or assist in its +//! implementation may be prepared, copied, published, and distributed, in whole +//! or in part, without restriction of any kind, provided that the above copyright +//! notice and this section are included on all such copies and derivative works. +//! However, this document itself may not be modified in any way, including by +//! removing the copyright notice or references to OASIS, except as needed for the +//! purpose of developing any document or deliverable produced by an OASIS Technical +//! Committee (in which case the rules applicable to copyrights, as set forth in the +//! OASIS IPR Policy, must be followed) or as required to translate it into languages +//! other than English. + +use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering}; + +use crate::utils::{IncompleteArrayField, VolatileCell}; +use static_assertions::const_assert_eq; + +/// [2.7.5 The Virtqueue Descriptor table](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-430005) +#[repr(C, align(16))] +pub struct Descriptor { + /// Address (guest-physical). + address: AtomicU64, + /// Size of the descriptor. + size: AtomicU32, + flags: AtomicU16, + /// Next field if flags & NEXT + next: AtomicU16, +} + +const_assert_eq!(core::mem::size_of::(), 16); + +bitflags::bitflags! { + #[derive(Debug, Copy, Clone)] + #[repr(transparent)] + pub struct DescriptorFlags: u16 { + /// This marks a buffer as continuing via the next field. + const NEXT = 1 << 0; + /// This marks a buffer as device write-only (otherwise device read-only). + const WRITE_ONLY = 1 << 1; + /// This means the buffer contains a list of buffer descriptors. + const INDIRECT = 1 << 2; + } +} + +impl Descriptor { + pub fn set_addr(&self, addr: u64) { + self.address.store(addr, Ordering::SeqCst) + } + + pub fn set_size(&self, size: u32) { + self.size.store(size, Ordering::SeqCst) + } + + pub fn set_next(&self, next: Option) { + self.next.store(next.unwrap_or_default(), Ordering::SeqCst) + } + + pub fn set_flags(&self, flags: DescriptorFlags) { + self.flags.store(flags.bits(), Ordering::SeqCst) + } + + pub fn next(&self) -> u16 { + self.next.load(Ordering::SeqCst) + } + + pub fn flags(&self) -> DescriptorFlags { + DescriptorFlags::from_bits_truncate(self.flags.load(Ordering::SeqCst)) + } +} + +// ======== Available Ring ======== +// +// XXX: The driver uses the available ring to offer buffers to the +// device. Each ring entry refers to the head of a descriptor +// chain. + +/// [2.7.6 The Virtqueue Available Ring](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-490006) +#[repr(C, align(2))] +pub struct AvailableRing { + pub flags: VolatileCell, + pub head_index: AtomicU16, + pub elements: IncompleteArrayField, +} + +const_assert_eq!(core::mem::size_of::(), 4); + +#[repr(C)] +pub struct AvailableRingElement { + pub table_index: AtomicU16, +} + +impl AvailableRingElement { + pub fn set_table_index(&self, index: u16) { + self.table_index.store(index, Ordering::SeqCst) + } +} + +const_assert_eq!(core::mem::size_of::(), 2); + +#[repr(C)] +pub struct AvailableRingExtra { + pub avail_event: VolatileCell, // Only if `VIRTIO_F_EVENT_IDX` +} + +const_assert_eq!(core::mem::size_of::(), 2); + +// ======== Used Ring ======== + +/// [2.7.8 The Virtqueue Used Ring](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-540008) +#[repr(C, align(4))] +pub struct UsedRing { + pub flags: VolatileCell, + pub head_index: VolatileCell, + pub elements: IncompleteArrayField, +} + +const_assert_eq!(core::mem::size_of::(), 4); + +#[repr(C)] +pub struct UsedRingElement { + pub table_index: VolatileCell, + pub written: VolatileCell, +} + +const_assert_eq!(core::mem::size_of::(), 8); + +#[repr(C)] +pub struct UsedRingExtra { + pub event_index: VolatileCell, +} + +// ======== Utils ======== +pub struct Buffer { + pub(crate) buffer: usize, + pub(crate) size: usize, + pub(crate) flags: DescriptorFlags, +} + +impl Buffer { + pub fn new(val: &common::dma::Dma) -> Self { + Self { + buffer: val.physical(), + size: core::mem::size_of::(), + flags: DescriptorFlags::empty(), + } + } + + pub fn new_unsized(val: &common::dma::Dma<[T]>) -> Self { + Self { + buffer: val.physical(), + size: core::mem::size_of::() * val.len(), + flags: DescriptorFlags::empty(), + } + } + + pub fn new_sized(val: &common::dma::Dma<[T]>, size: usize) -> Self { + Self { + buffer: val.physical(), + size, + flags: DescriptorFlags::empty(), + } + } + + pub fn flags(mut self, flags: DescriptorFlags) -> Self { + self.flags = flags; + self + } +} + +/// XXX: The [`DescriptorFlags::NEXT`] flag is set automatically. +pub struct ChainBuilder { + buffers: Vec, +} + +impl ChainBuilder { + pub fn new() -> Self { + Self { + buffers: Vec::new(), + } + } + + pub fn chain(mut self, mut buffer: Buffer) -> Self { + buffer.flags |= DescriptorFlags::NEXT; + self.buffers.push(buffer); + self + } + + pub fn build(mut self) -> Vec { + let last_buffer = self.buffers.last_mut().expect("virtio-core: empty chain"); + last_buffer.flags.remove(DescriptorFlags::NEXT); + + self.buffers + } +} diff --git a/virtio-core/src/spec/transport_pci.rs b/virtio-core/src/spec/transport_pci.rs new file mode 100644 index 0000000000..cadfaad2de --- /dev/null +++ b/virtio-core/src/spec/transport_pci.rs @@ -0,0 +1,174 @@ +//! [4.1 Virtio Over PCI Bus](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-1150001) +//! +//! This file contains comments copied from the VirtIO specification which are +//! licensed under the following conditions: +//! +//! Copyright © OASIS Open 2022. All Rights Reserved. +//! +//! All capitalized terms in the following text have the meanings assigned to them +//! in the OASIS Intellectual Property Rights Policy (the "OASIS IPR Policy"). The +//! full Policy may be found at the OASIS website. +//! +//! This document and translations of it may be copied and furnished to others, +//! and derivative works that comment on or otherwise explain it or assist in its +//! implementation may be prepared, copied, published, and distributed, in whole +//! or in part, without restriction of any kind, provided that the above copyright +//! notice and this section are included on all such copies and derivative works. +//! However, this document itself may not be modified in any way, including by +//! removing the copyright notice or references to OASIS, except as needed for the +//! purpose of developing any document or deliverable produced by an OASIS Technical +//! Committee (in which case the rules applicable to copyrights, as set forth in the +//! OASIS IPR Policy, must be followed) or as required to translate it into languages +//! other than English. + +use super::DeviceStatusFlags; +use crate::utils::VolatileCell; +use static_assertions::const_assert_eq; + +/// [4.1.4 Virtio Structure PCI Capabilities](https://docs.oasis-open.org/virtio/virtio/v1.2/cs01/virtio-v1.2-cs01.html#x1-1240004) +#[derive(Debug, Copy, Clone)] +#[repr(C, packed)] +pub struct PciCapability { + /// Identifies the structure. + pub cfg_type: CfgType, + /// Where to find it. + pub bar: u8, + /// Multiple capabilities of the same typel + pub id: u8, + /// Pad to a full dword. + pub padding: [u8; 2], + /// Offset within the bar. + pub offset: u32, + /// Length of the structure, in bytes. + pub length: u32, +} + +// The size of `PciCapability` is 13 bytes since the generic +// PCI fields are *not* included. +const_assert_eq!(core::mem::size_of::(), 13); + +#[derive(Debug, Copy, Clone)] +#[repr(u8)] +pub enum CfgType { + /// Common Configuration. + Common = 1, + /// Notifications. + Notify = 2, + /// ISR Status. + Isr = 3, + /// Device specific configuration. + Device = 4, + /// PCI configuration access. + PciConfig = 5, + /// Shared memory region. + SharedMemory = 8, + /// Vendor-specific data. + Vendor = 9, +} + +const_assert_eq!(core::mem::size_of::(), 1); + +#[derive(Debug)] +#[repr(C)] +pub struct CommonCfg { + // About the whole device. + /// The driver uses this to select which feature bits device_feature shows. + /// Value 0x0 selects Feature Bits 0 to 31, 0x1 selects Feature Bits 32 to 63, etc. + /// read-write + pub device_feature_select: VolatileCell, + /// The device uses this to report which feature bits it is offering to the driver: + /// the driver writes to device_feature_select to select which feature bits are presented. + /// read-only for driver + pub device_feature: VolatileCell, + /// The driver uses this to select which feature bits driver_feature shows. + /// Value 0x0 selects Feature Bits 0 to 31, 0x1 selects Feature Bits 32 to 63, etc. + /// read-write + pub driver_feature_select: VolatileCell, + /// The driver writes this to accept feature bits offered by the device. + /// Driver Feature Bits selected by driver_feature_select. + /// read-write + pub driver_feature: VolatileCell, + /// The driver sets the Configuration Vector for MSI-X. + /// read-write + pub config_msix_vector: VolatileCell, + /// The device specifies the maximum number of virtqueues supported here. + /// read-only for driver + pub num_queues: VolatileCell, + /// The driver writes the device status here (see 2.1). + /// Writing 0 into this field resets the device. + /// read-write + pub device_status: VolatileCell, + /// Configuration atomicity value. The device changes this every time the + /// configuration noticeably changes. + /// read-only for driver + pub config_generation: VolatileCell, + + // About a specific virtqueue. + /// Queue Select. The driver selects which virtqueue the following fields refer to. + /// read-write + pub queue_select: VolatileCell, + /// Queue Size. On reset, specifies the maximum queue size supported by the device. + /// This can be modified by the driver to reduce memory requirements. + /// A 0 means the queue is unavailable. + /// read-write + pub queue_size: VolatileCell, + /// The driver uses this to specify the queue vector for MSI-X. + /// read-write + pub queue_msix_vector: VolatileCell, + /// The driver uses this to selectively prevent the device from executing + /// requests from this virtqueue. 1 - enabled; 0 - disabled. + /// read-write + pub queue_enable: VolatileCell, + /// The driver reads this to calculate the offset from start of Notification + /// structure at which this virtqueue is located. Note: this is not an offset + /// in bytes. See 4.1.4.4 below. + /// read-only for driver + pub queue_notify_off: VolatileCell, + /// The driver writes the physical address of Descriptor Area here. + /// See section 2.6. + /// read-write + pub queue_desc: VolatileCell, + /// The driver writes the physical address of Driver Area here. + /// See section 2.6. + /// read-write + pub queue_driver: VolatileCell, + /// The driver writes the physical address of Device Area here. + /// See section 2.6. + /// read-write + pub queue_device: VolatileCell, + /// This field exists only if VIRTIO_F_NOTIF_CONFIG_DATA has been negotiated. + /// The driver will use this value to put it in the ’virtqueue number’ field + /// in the available buffer notification structure. See section 4.1.5.2. Note: + /// This field provides the device with flexibility to determine how virtqueues + /// will be referred to in available buffer notifications. In a trivial case the + /// device can set queue_notify_data=vqn. Some devices may benefit from providing + /// another value, for example an internal virtqueue identifier, or an internal + /// offset related to the virtqueue number. + /// read-only for driver + pub queue_notify_data: VolatileCell, + /// The driver uses this to selectively reset the queue. This field exists + /// only if VIRTIO_F_RING_RESET has been negotiated. (see 2.6.1). + /// read-write + pub queue_reset: VolatileCell, +} + +const_assert_eq!(core::mem::size_of::(), 64); + +#[derive(Debug, Copy, Clone)] +#[repr(C, packed)] +pub struct PciCapabilityNotify { + pub cap: PciCapability, + /// Multiplier for queue_notify_off. + notify_off_multiplier: u32, +} + +impl PciCapabilityNotify { + pub fn notify_off_multiplier(&self) -> u32 { + self.notify_off_multiplier + } +} + +const_assert_eq!(core::mem::size_of::(), 17); + +/// Vector value used to disable MSI for queue +pub const VIRTIO_MSI_NO_VECTOR: u16 = 0xffff; diff --git a/virtio-core/src/transport.rs b/virtio-core/src/transport.rs index a324a288cc..1ff80a205f 100644 --- a/virtio-core/src/transport.rs +++ b/virtio-core/src/transport.rs @@ -68,7 +68,7 @@ pub const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) { ) } -pub fn spawn_irq_thread(irq_handle: &File, queue: &Arc>) -> Result<(), Error> { +pub fn spawn_irq_thread(irq_handle: &File, queue: &Arc>) { let irq_fd = irq_handle.as_raw_fd(); let queue_copy = queue.clone(); @@ -91,8 +91,6 @@ pub fn spawn_irq_thread(irq_handle: &File, queue: &Arc>) -> Resul event_queue.run().unwrap(); } }); - - Ok(()) } pub trait NotifyBell { @@ -645,7 +643,7 @@ impl Transport for StandardTransport<'_> { vector, ); - spawn_irq_thread(irq_handle, &queue)?; + spawn_irq_thread(irq_handle, &queue); Ok(queue) } diff --git a/virtio-netd/src/main.rs b/virtio-netd/src/main.rs index 4e38b1ec02..c99b2c4577 100644 --- a/virtio-netd/src/main.rs +++ b/virtio-netd/src/main.rs @@ -8,11 +8,12 @@ use pcid_interface::PcidServerHandle; use syscall::{Packet, SchemeBlockMut}; -use virtio_core::spec::VIRTIO_NET_F_MAC; use virtio_core::transport::{Error, Transport}; use scheme::NetworkScheme; +pub const VIRTIO_NET_F_MAC: u32 = 5; + #[derive(Debug)] #[repr(C)] pub struct VirtHeader {