virtiod: split into virtio-core and virtiod

Signed-off-by: Anhad Singh <andypython@protonmail.com>

virtio-core/src/lib.rs

@@ -0,0 +1,9 @@
+#![feature(int_roundings)]
+
+pub mod spec;
+pub mod transport;
+pub mod utils;
+
+mod probe;
+
+pub use probe::{probe_device, MSIX_PRIMARY_VECTOR, Device};

virtio-core/src/probe.rs

@@ -0,0 +1,271 @@
+use std::fs::File;
+use std::ptr::NonNull;
+use std::sync::Arc;
+
+use pcid_interface::irq_helpers::{allocate_single_interrupt_vector, read_bsp_apic_id};
+use pcid_interface::msi::x86_64 as x86_64_msix;
+use pcid_interface::msi::x86_64::DeliveryMode;
+use pcid_interface::msi::{MsixCapability, MsixTableEntry};
+use pcid_interface::*;
+
+use syscall::{Io, PHYSMAP_NO_CACHE, PHYSMAP_WRITE};
+
+use crate::spec::*;
+use crate::transport::{Error, StandardTransport};
+use crate::utils::VolatileCell;
+
+pub struct Device<'a> {
+    pub transport: Arc<StandardTransport<'a>>,
+    pub device_space: *const u8,
+    pub irq_handle: File,
+    pub isr: &'a VolatileCell<u32>,
+}
+
+struct MsixInfo {
+    pub virt_table_base: NonNull<MsixTableEntry>,
+    pub virt_pba_base: NonNull<u64>,
+    pub capability: MsixCapability,
+}
+
+impl MsixInfo {
+    pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry {
+        &mut *self.virt_table_base.as_ptr().add(k)
+    }
+
+    pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry {
+        assert!(k < self.capability.table_size() as usize);
+        unsafe { self.table_entry_pointer_unchecked(k) }
+    }
+}
+
+static_assertions::const_assert_eq!(std::mem::size_of::<MsixTableEntry>(), 16);
+
+pub const MSIX_PRIMARY_VECTOR: u16 = 0;
+
+fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result<File, Error> {
+    let pci_config = pcid_handle.fetch_config()?;
+
+    // Extended message signaled interrupts.
+    let capability = match pcid_handle.feature_info(PciFeature::MsiX)? {
+        PciFeatureInfo::MsiX(capability) => capability,
+        _ => unreachable!(),
+    };
+
+    let table_size = capability.table_size();
+    let table_base = capability.table_base_pointer(pci_config.func.bars);
+    let table_min_length = table_size * 16;
+    let pba_min_length = table_size.div_ceil(8);
+
+    let pba_base = capability.pba_base_pointer(pci_config.func.bars);
+
+    let bir = capability.table_bir() as usize;
+    let bar = pci_config.func.bars[bir];
+    let bar_size = pci_config.func.bar_sizes[bir] as u64;
+
+    let bar_ptr = match bar {
+        PciBar::Memory32(ptr) => ptr.into(),
+        PciBar::Memory64(ptr) => ptr,
+        _ => unreachable!(),
+    };
+
+    let address = unsafe {
+        syscall::physmap(
+            bar_ptr as usize,
+            bar_size as usize,
+            PHYSMAP_WRITE | PHYSMAP_NO_CACHE,
+        )?
+    };
+
+    // Ensure that the table and PBA are be within the BAR.
+    {
+        let bar_range = bar_ptr..bar_ptr + bar_size;
+        assert!(bar_range.contains(&(table_base as u64 + table_min_length as u64)));
+        assert!(bar_range.contains(&(pba_base as u64 + pba_min_length as u64)));
+    }
+
+    let virt_table_base = ((table_base - bar_ptr as usize) + address) as *mut MsixTableEntry;
+    let virt_pba_base = ((pba_base - bar_ptr as usize) + address) as *mut u64;
+
+    let mut info = MsixInfo {
+        virt_table_base: NonNull::new(virt_table_base).unwrap(),
+        virt_pba_base: NonNull::new(virt_pba_base).unwrap(),
+        capability,
+    };
+
+    // Allocate the primary MSI vector.
+    let interrupt_handle = {
+        let table_entry_pointer = info.table_entry_pointer(MSIX_PRIMARY_VECTOR as usize);
+
+        let destination_id = read_bsp_apic_id().expect("virtio_core: `read_bsp_apic_id()` failed");
+        let lapic_id = u8::try_from(destination_id).unwrap();
+
+        let rh = false;
+        let dm = false;
+        let addr = x86_64_msix::message_address(lapic_id, rh, dm);
+
+        let (vector, interrupt_handle) = allocate_single_interrupt_vector(destination_id)
+            .unwrap()
+            .expect("virtio_core: interrupt vector exhaustion");
+
+        let msg_data = x86_64_msix::message_data_edge_triggered(DeliveryMode::Fixed, vector);
+
+        table_entry_pointer.addr_lo.write(addr);
+        table_entry_pointer.addr_hi.write(0);
+        table_entry_pointer.msg_data.write(msg_data);
+        table_entry_pointer
+            .vec_ctl
+            .writef(MsixTableEntry::VEC_CTL_MASK_BIT, false);
+
+        interrupt_handle
+    };
+
+    pcid_handle.enable_feature(PciFeature::MsiX)?;
+
+    log::info!("virtio: using MSI-X (interrupt_handle={interrupt_handle:?})");
+    Ok(interrupt_handle)
+}
+
+/// VirtIO Device Probe
+///
+/// ## Device State
+/// After this function, the device has been successfully reseted and is ready for use.
+///
+/// The caller is required to do the following:
+/// * Negotiate the device and driver supported features (finialize via [`StandardTransport::finalize_features`])
+/// * Create the device specific virtio queues (via [`StandardTransport::setup_queue`])
+/// * Finally start the device (via [`StandardTransport::run_device`]). At this point, the device
+///   is alive.
+///
+/// ## Panics
+/// This function panics if the device is not a virtio device.
+pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result<Device<'a>, Error> {
+    let pci_config = pcid_handle.fetch_config()?;
+    let pci_header = pcid_handle.fetch_header()?;
+
+    assert_eq!(
+        pci_config.func.venid, 6900,
+        "virtio_core::probe_device: not a virtio device"
+    );
+
+    let mut common_addr = None;
+    let mut notify_addr = None;
+    let mut isr_addr = None;
+    let mut device_addr = None;
+
+    for capability in pcid_handle
+        .get_capabilities()?
+        .iter()
+        .filter_map(|capability| {
+            if let Capability::Vendor(vendor) = capability {
+                Some(vendor)
+            } else {
+                None
+            }
+        })
+    {
+        // SAFETY: We have verified that the length of the data is correct.
+        let capability = unsafe { &*(capability.data.as_ptr() as *const PciCapability) };
+
+        match capability.cfg_type {
+            CfgType::Common | CfgType::Notify | CfgType::Isr | CfgType::Device => {}
+            _ => continue,
+        }
+
+        let bar = pci_header.get_bar(capability.bar as usize);
+        let addr = match bar {
+            PciBar::Memory32(addr) => addr as usize,
+            PciBar::Memory64(addr) => addr as usize,
+
+            _ => unreachable!("virtio: unsupported bar type: {bar:?}"),
+        };
+
+        let address = unsafe {
+            syscall::physmap(
+                addr + capability.offset as usize,
+                capability.length as usize,
+                PHYSMAP_WRITE | PHYSMAP_NO_CACHE,
+            )?
+        };
+
+        match capability.cfg_type {
+            CfgType::Common => {
+                debug_assert!(common_addr.is_none());
+                common_addr = Some(address);
+            }
+
+            CfgType::Notify => {
+                debug_assert!(notify_addr.is_none());
+
+                // SAFETY: The capability type is `Notify`, so its safe to access
+                //         the `notify_multiplier` field.
+                let multiplier = unsafe { capability.notify_multiplier() };
+                notify_addr = Some((address, multiplier));
+            }
+
+            CfgType::Isr => {
+                debug_assert!(isr_addr.is_none());
+                isr_addr = Some(address);
+            }
+
+            CfgType::Device => {
+                debug_assert!(device_addr.is_none());
+                device_addr = Some(address);
+            }
+
+            _ => unreachable!(),
+        }
+
+        log::info!("virtio: {capability:?}");
+    }
+
+    let common_addr = common_addr.ok_or(Error::InCapable(CfgType::Common))?;
+    let (notify_addr, notify_multiplier) = notify_addr.ok_or(Error::InCapable(CfgType::Notify))?;
+    let isr_addr = isr_addr.ok_or(Error::InCapable(CfgType::Isr))?;
+    let device_addr = device_addr.ok_or(Error::InCapable(CfgType::Device))?;
+
+    assert!(
+        notify_multiplier != 0,
+        "virtio-core::device_probe: device uses the same Queue Notify addresses for all queues"
+    );
+
+    let common = unsafe { &mut *(common_addr as *mut CommonCfg) };
+    let device_space = unsafe { &mut *(device_addr as *mut u8) };
+    let isr = unsafe { &*(isr_addr as *mut VolatileCell<u32>) };
+
+    // Reset the device.
+    common.device_status.set(DeviceStatusFlags::empty());
+    // Upon reset, the device must initialize device status to 0.
+    assert_eq!(common.device_status.get(), DeviceStatusFlags::empty());
+    log::info!("virtio: successfully reseted the device");
+
+    // XXX: According to the virtio specification v1.2, setting the ACKNOWLEDGE and DRIVER bits
+    //      in `device_status` is required to be done in two steps.
+    common
+        .device_status
+        .set(common.device_status.get() | DeviceStatusFlags::ACKNOWLEDGE);
+
+    common
+        .device_status
+        .set(common.device_status.get() | DeviceStatusFlags::DRIVER);
+
+    // Setup interrupts.
+    let all_pci_features = pcid_handle.fetch_all_features()?;
+    let has_msix = all_pci_features
+        .iter()
+        .any(|(feature, _)| feature.is_msix());
+
+    // According to the virtio specification, the device REQUIRED to support MSI-X.
+    assert!(has_msix, "virtio: device does not support MSI-X");
+    let irq_handle = enable_msix(pcid_handle)?;
+
+    log::info!("virtio: using standard PCI transport");
+
+    let transport = StandardTransport::new(common, notify_addr as *const u8, notify_multiplier);
+
+    Ok(Device {
+        transport,
+        device_space,
+        irq_handle,
+        isr,
+    })
+}

virtio-core/src/spec.rs

@@ -0,0 +1,247 @@
+//! https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html
+
+use crate::utils::{IncompleteArrayField, VolatileCell};
+use static_assertions::const_assert_eq;
+
+#[derive(Debug, Copy, Clone)]
+#[repr(u8)]
+pub enum CfgType {
+    /// Common Configuration.
+    Common = 1,
+    /// Notifications.
+    Notify = 2,
+    /// ISR Status.
+    Isr = 3,
+    /// Device specific configuration.
+    Device = 4,
+    /// PCI configuration access.
+    PciConfig = 5,
+    /// Shared memory region.
+    SharedMemory = 8,
+    /// Vendor-specific data.
+    Vendor = 9,
+}
+
+const_assert_eq!(core::mem::size_of::<CfgType>(), 1);
+
+#[derive(Debug, Copy, Clone)]
+#[repr(C, packed)]
+pub struct PciCapability {
+    /// Identifies the structure.
+    pub cfg_type: CfgType,
+    /// Where to find it.
+    pub bar: u8,
+    /// Pad to a full dword.
+    pub padding: [u8; 3],
+    /// Offset within the bar.
+    pub offset: u32,
+    /// Length of the structure, in bytes.
+    pub length: u32,
+    notify_multiplier: u32,
+}
+
+// The size of `PciCapability` is 13 bytes since the generic
+// PCI fields are *not* included.
+const_assert_eq!(core::mem::size_of::<PciCapability>(), 17);
+
+impl PciCapability {
+    /// ## Safety
+    /// Undefined if accessed from a capability type other than [`CfgType::Notify`].
+    pub unsafe fn notify_multiplier(&self) -> u32 {
+        self.notify_multiplier
+    }
+}
+
+bitflags::bitflags! {
+    #[derive(Debug, Copy, Clone, PartialEq)]
+    #[repr(transparent)]
+    pub struct DeviceStatusFlags: u8 {
+        /// Indicates that the guest OS has found the device and recognized it as a
+        /// valid device.
+        const ACKNOWLEDGE = 1;
+        /// Indicates that the guest OS knows how to drive the device.
+        const DRIVER = 2;
+        /// Indicates that something went wrong in the guest and it has given up on
+        /// the device.
+        const FAILED = 128;
+        /// Indicates that the driver has acknowledged all the features it understands
+        /// and feature negotiation is complete.
+        const FEATURES_OK = 8;
+        /// Indicates that the driver is set up and ready to drive the device.
+        const DRIVER_OK = 4;
+        /// Indicates that the device has experienced an error from which it can’t recover.
+        const DEVICE_NEEDS_RESET = 64;
+    }
+}
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct CommonCfg {
+    // About the whole device.
+    pub device_feature_select: VolatileCell<u32>, // read-write
+    pub device_feature: VolatileCell<u32>,        // read-only for driver
+    pub driver_feature_select: VolatileCell<u32>, // read-write
+    pub driver_feature: VolatileCell<u32>,        // read-write
+    pub msix_config: VolatileCell<u16>,           // read-write
+    pub num_queues: VolatileCell<u16>,            // read-only for driver
+    pub device_status: VolatileCell<DeviceStatusFlags>, // read-write
+    pub config_generation: VolatileCell<u8>,      // read-only for driver
+
+    // About a specific virtqueue.
+    pub queue_select: VolatileCell<u16>,      // read-write
+    pub queue_size: VolatileCell<u16>,        // read-write
+    pub queue_msix_vector: VolatileCell<u16>, // read-write
+    pub queue_enable: VolatileCell<u16>,      // read-write
+    pub queue_notify_off: VolatileCell<u16>,  // read-only for driver
+    pub queue_desc: VolatileCell<u64>,        // read-write
+    pub queue_driver: VolatileCell<u64>,      // read-write
+    pub queue_device: VolatileCell<u64>,      // read-write
+}
+
+const_assert_eq!(core::mem::size_of::<CommonCfg>(), 56);
+
+bitflags::bitflags! {
+    #[derive(Debug, Copy, Clone)]
+    #[repr(transparent)]
+    pub struct DescriptorFlags: u16 {
+        /// The next field contains linked buffer index.
+        const NEXT = 1 << 0;
+        /// The buffer is write-only (otherwise read-only).
+        const WRITE_ONLY = 1 << 1;
+        /// The buffer contains a list of buffer descriptors.
+        const INDIRECT = 1 << 2;
+    }
+}
+
+#[repr(C)]
+pub struct Descriptor {
+    /// Address (guest-physical).
+    pub address: u64,
+    /// Size of the descriptor.
+    pub size: u32,
+    pub flags: DescriptorFlags,
+    /// Index of next desciptor in chain.
+    pub next: u16,
+}
+
+const_assert_eq!(core::mem::size_of::<Descriptor>(), 16);
+
+/// This indicates compliance with the version 1 VirtIO specification.
+///
+/// See `6.1 Driver Requirements: Reserved Feature Bits` section of the VirtIO
+/// specification for more information.
+pub const VIRTIO_F_VERSION_1: u32 = 32;
+
+// ======== Available Ring ========
+//
+// XXX: The driver uses the available ring to offer buffers to the
+//      device. Each ring entry refers to the head of a descriptor
+//      chain.
+#[repr(C)]
+pub struct AvailableRingElement {
+    pub table_index: VolatileCell<u16>,
+}
+
+const_assert_eq!(core::mem::size_of::<AvailableRingElement>(), 2);
+
+#[repr(C)]
+pub struct AvailableRing {
+    pub flags: VolatileCell<u16>,
+    pub head_index: VolatileCell<u16>,
+    pub elements: IncompleteArrayField<AvailableRingElement>,
+}
+
+const_assert_eq!(core::mem::size_of::<AvailableRing>(), 4);
+
+impl Default for AvailableRing {
+    fn default() -> Self {
+        Self {
+            flags: VolatileCell::new(0),
+            head_index: VolatileCell::new(0),
+            elements: IncompleteArrayField::new(),
+        }
+    }
+}
+
+#[repr(C)]
+pub struct AvailableRingExtra {
+    pub avail_event: VolatileCell<u16>, // Only if `VIRTIO_F_EVENT_IDX`
+}
+
+const_assert_eq!(core::mem::size_of::<AvailableRingExtra>(), 2);
+
+// ======== Used Ring ========
+#[repr(C)]
+pub struct UsedRingElement {
+    pub table_index: VolatileCell<u32>,
+    pub written: VolatileCell<u32>,
+}
+
+const_assert_eq!(core::mem::size_of::<UsedRingElement>(), 8);
+
+#[repr(C)]
+pub struct UsedRing {
+    pub flags: VolatileCell<u16>,
+    pub head_index: VolatileCell<u16>,
+    pub elements: IncompleteArrayField<UsedRingElement>,
+}
+
+const_assert_eq!(core::mem::size_of::<UsedRing>(), 4);
+
+impl Default for UsedRing {
+    fn default() -> Self {
+        Self {
+            flags: VolatileCell::new(0),
+            head_index: VolatileCell::new(0),
+            elements: IncompleteArrayField::new(),
+        }
+    }
+}
+
+#[repr(C)]
+pub struct UsedRingExtra {
+    pub event_index: VolatileCell<u16>,
+}
+
+// ======== Utils ========
+pub struct Buffer {
+    pub(crate) buffer: usize,
+    pub(crate) size: usize,
+    pub(crate) flags: DescriptorFlags,
+}
+
+impl Buffer {
+    pub fn new<T>(val: &syscall::Dma<T>) -> Self {
+        Self {
+            buffer: val.physical(),
+            size: core::mem::size_of::<T>(),
+            flags: DescriptorFlags::empty(),
+        }
+    }
+
+    pub fn flags(mut self, flags: DescriptorFlags) -> Self {
+        self.flags = flags;
+        self
+    }
+}
+
+pub struct ChainBuilder {
+    buffers: Vec<Buffer>,
+}
+
+impl ChainBuilder {
+    pub fn new() -> Self {
+        Self {
+            buffers: Vec::new(),
+        }
+    }
+
+    pub fn chain(mut self, buffer: Buffer) -> Self {
+        self.buffers.push(buffer);
+        self
+    }
+
+    pub fn build(self) -> Vec<Buffer> {
+        self.buffers
+    }
+}

virtio-core/src/transport.rs

@@ -0,0 +1,424 @@
+use crate::spec::*;
+use crate::utils::{align, VolatileCell};
+
+use syscall::{Dma, PHYSMAP_WRITE};
+
+use core::mem::size_of;
+use core::sync::atomic::{fence, AtomicU16, Ordering};
+
+use std::collections::VecDeque;
+use std::sync::{Arc, Mutex, Weak};
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("syscall failed")]
+    SyscallError(syscall::Error),
+    #[error("pcid client handle error")]
+    PcidClientHandle(pcid_interface::PcidClientHandleError),
+    #[error("the device is incapable of {0:?}")]
+    InCapable(CfgType),
+}
+
+impl From<pcid_interface::PcidClientHandleError> for Error {
+    fn from(value: pcid_interface::PcidClientHandleError) -> Self {
+        Self::PcidClientHandle(value)
+    }
+}
+
+impl From<syscall::Error> for Error {
+    fn from(value: syscall::Error) -> Self {
+        Self::SyscallError(value)
+    }
+}
+
+/// Returns the queue part sizes in bytes.
+///
+/// ## Reference
+/// Section 2.7 Split Virtqueues of the specfication v1.2 describes the alignment
+/// and size of the queue parts.
+///
+/// ## Panics
+/// If `queue_size` is not a power of two or is zero.
+const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) {
+    assert!(queue_size.is_power_of_two() && queue_size != 0);
+
+    const DESCRIPTOR_ALIGN: usize = 16;
+    const AVAILABLE_ALIGN: usize = 2;
+    const USED_ALIGN: usize = 4;
+
+    let queue_size = queue_size as usize;
+    let desc = size_of::<Descriptor>() * queue_size;
+
+    // `avail_header`: Size of the available ring header and the footer.
+    let avail_header = size_of::<AvailableRing>() + size_of::<AvailableRingExtra>();
+    let avail = avail_header + size_of::<AvailableRingElement>() * queue_size;
+
+    // `used_header`: Size of the used ring header and the footer.
+    let used_header = size_of::<UsedRing>() + size_of::<UsedRingExtra>();
+    let used = used_header + size_of::<UsedRingElement>() * queue_size;
+
+    (
+        align(desc, DESCRIPTOR_ALIGN),
+        align(avail, AVAILABLE_ALIGN),
+        align(used, USED_ALIGN),
+    )
+}
+
+pub struct QueueInner<'a> {
+    pub descriptor: Dma<[Descriptor]>,
+    pub available: Available<'a>,
+    pub used: Used<'a>,
+
+    /// Keeps track of unused descriptor indicies.
+    pub descriptor_stack: VecDeque<u16>,
+
+    notification_bell: &'a mut VolatileCell<u16>,
+    head_index: u16,
+}
+
+unsafe impl Sync for QueueInner<'_> {}
+unsafe impl Send for QueueInner<'_> {}
+
+pub struct Queue<'a> {
+    pub inner: Mutex<QueueInner<'a>>,
+    sref: Weak<Self>,
+}
+
+impl<'a> Queue<'a> {
+    pub fn new(
+        descriptor: Dma<[Descriptor]>,
+        available: Available<'a>,
+        used: Used<'a>,
+
+        notification_bell: &'a mut VolatileCell<u16>,
+    ) -> Arc<Self> {
+        Arc::new_cyclic(|sref| Self {
+            inner: Mutex::new(QueueInner {
+                head_index: 0,
+                descriptor_stack: (0..(descriptor.len() - 1) as u16).rev().collect(),
+
+                descriptor,
+                available,
+                used,
+
+                notification_bell,
+            }),
+
+            sref: sref.clone(),
+        })
+    }
+
+    pub fn send(&self, chain: Vec<Buffer>) {
+        let mut first_descriptor: Option<usize> = None;
+        let mut last_descriptor: Option<usize> = None;
+
+        for buffer in chain.iter() {
+            let descriptor = self.alloc_descriptor();
+
+            let mut inner = self.inner.lock().unwrap();
+
+            if first_descriptor.is_none() {
+                first_descriptor = Some(descriptor);
+            }
+
+            inner.descriptor[descriptor].address = buffer.buffer as u64;
+            inner.descriptor[descriptor].flags = buffer.flags;
+            inner.descriptor[descriptor].size = buffer.size as u32;
+
+            if let Some(index) = last_descriptor {
+                inner.descriptor[index].next = descriptor as u16;
+            }
+
+            last_descriptor = Some(descriptor);
+        }
+
+        let mut inner = self.inner.lock().unwrap();
+
+        let last_descriptor = last_descriptor.unwrap();
+        let first_descriptor = first_descriptor.unwrap();
+
+        inner.descriptor[last_descriptor].next = 0;
+
+        fence(Ordering::SeqCst);
+        let index = inner.head_index as usize;
+
+        inner
+            .available
+            .get_element_at(index)
+            .table_index
+            .set(first_descriptor as u16);
+
+        fence(Ordering::SeqCst);
+        inner.available.set_head_idx(index as u16 + 1);
+        inner.head_index += 1;
+
+        assert_eq!(inner.used.flags(), 0);
+        inner.notification_bell.set(0); // FIXME: This corresponds to the queue index.
+    }
+
+    fn alloc_descriptor(&self) -> usize {
+        let mut inner = self.inner.lock().unwrap();
+
+        if let Some(index) = inner.descriptor_stack.pop_front() {
+            index as usize
+        } else {
+            log::warn!("virtiod: descriptors exhausted, waiting on garabage collector");
+            drop(inner);
+
+            // Wait for the garbage collector thread to release some descriptors.
+            //
+            // TODO(andypython): Instead of just yielding, we should have a proper notificiation
+            //                   mechanism. I am not aware whats the standard way redox applications
+            //                   or drivers implement basically a WaitQueue which you can use to wake
+            //                   up a thread. The descripts really should NEVER run out, but if they
+            //                   do, have a proper way to handle them.
+            std::thread::yield_now();
+            self.alloc_descriptor()
+        }
+    }
+}
+
+pub struct Available<'a> {
+    addr: usize,
+    size: usize,
+
+    queue_size: usize,
+
+    ring: &'a mut AvailableRing,
+}
+
+impl<'a> Available<'a> {
+    pub fn new(queue_size: usize) -> Result<Self, Error> {
+        let (_, size, _) = queue_part_sizes(queue_size);
+        let size = size.next_multiple_of(syscall::PAGE_SIZE); // align to page size
+
+        let addr = unsafe { syscall::physalloc(size) }.map_err(Error::SyscallError)?;
+        let virt =
+            unsafe { syscall::physmap(addr, size, PHYSMAP_WRITE) }.map_err(Error::SyscallError)?;
+
+        let ring = unsafe { &mut *(virt as *mut AvailableRing) };
+
+        Ok(Self {
+            addr,
+            size,
+            ring,
+            queue_size,
+        })
+    }
+
+    /// ## Panics
+    /// This function panics if the index is out of bounds.
+    pub fn get_element_at(&mut self, index: usize) -> &mut AvailableRingElement {
+        // SAFETY: We have exclusive access to the elements and the number of elements
+        //         is correct; same as the queue size.
+        unsafe {
+            self.ring
+                .elements
+                .as_mut_slice(self.queue_size)
+                .get_mut(index % 256)
+                .expect("virtio::available: index out of bounds")
+        }
+    }
+
+    pub fn set_head_idx(&mut self, index: u16) {
+        self.ring.head_index.set(index);
+    }
+
+    pub fn phys_addr(&self) -> usize {
+        self.addr
+    }
+}
+
+impl Drop for Available<'_> {
+    fn drop(&mut self) {
+        log::warn!("virtio: dropping 'available' ring at {:#x}", self.addr);
+
+        unsafe {
+            syscall::physunmap(self.addr).unwrap();
+            syscall::physfree(self.addr, self.size).unwrap();
+        }
+    }
+}
+
+pub struct Used<'a> {
+    addr: usize,
+    size: usize,
+
+    queue_size: usize,
+
+    ring: &'a mut UsedRing,
+}
+
+impl<'a> Used<'a> {
+    pub fn new(queue_size: usize) -> Result<Self, Error> {
+        let (_, _, size) = queue_part_sizes(queue_size);
+        let size = size.next_multiple_of(syscall::PAGE_SIZE); // align to page size
+
+        let addr = unsafe { syscall::physalloc(size) }.map_err(Error::SyscallError)?;
+        let virt =
+            unsafe { syscall::physmap(addr, size, PHYSMAP_WRITE) }.map_err(Error::SyscallError)?;
+
+        let ring = unsafe { &mut *(virt as *mut UsedRing) };
+
+        Ok(Self {
+            addr,
+            size,
+            ring,
+            queue_size,
+        })
+    }
+
+    /// ## Panics
+    /// This function panics if the index is out of bounds.
+    pub fn get_element_at(&mut self, index: usize) -> &mut UsedRingElement {
+        // SAFETY: We have exclusive access to the elements and the number of elements
+        //         is correct; same as the queue size.
+        unsafe {
+            self.ring
+                .elements
+                .as_mut_slice(self.queue_size)
+                .get_mut(index % 256)
+                .expect("virtio::used: index out of bounds")
+        }
+    }
+
+    pub fn flags(&self) -> u16 {
+        self.ring.flags.get()
+    }
+
+    pub fn head_index(&self) -> u16 {
+        self.ring.head_index.get()
+    }
+
+    pub fn phys_addr(&self) -> usize {
+        self.addr
+    }
+}
+
+impl Drop for Used<'_> {
+    fn drop(&mut self) {
+        log::warn!("virtio: dropping 'used' ring at {:#x}", self.addr);
+
+        unsafe {
+            syscall::physunmap(self.addr).unwrap();
+            syscall::physfree(self.addr, self.size).unwrap();
+        }
+    }
+}
+
+pub struct StandardTransport<'a> {
+    common: Mutex<&'a mut CommonCfg>,
+    notify: *const u8,
+    notify_mul: u32,
+
+    queue_index: AtomicU16,
+    sref: Weak<Self>,
+}
+
+impl<'a> StandardTransport<'a> {
+    pub fn new(common: &'a mut CommonCfg, notify: *const u8, notify_mul: u32) -> Arc<Self> {
+        Arc::new_cyclic(|sref| Self {
+            common: Mutex::new(common),
+            notify,
+            notify_mul,
+
+            queue_index: AtomicU16::new(0),
+            sref: sref.clone(),
+        })
+    }
+
+    pub fn sref(&self) -> Arc<Self> {
+        // UNWRAP: The constructor ensures that we are wrapped in our own `Arc`. So this
+        //         unwrap is going to be unreachable.
+        self.sref.upgrade().unwrap()
+    }
+
+    pub fn check_device_feature(&self, feature: u32) -> bool {
+        let mut common = self.common.lock().unwrap();
+
+        common.device_feature_select.set(feature >> 5);
+        (common.device_feature.get() & (1 << (feature & 31))) != 0
+    }
+
+    pub fn ack_driver_feature(&self, feature: u32) {
+        let mut common = self.common.lock().unwrap();
+
+        common.driver_feature_select.set(feature >> 5);
+
+        let current = common.driver_feature.get();
+        common.driver_feature.set(current | (1 << (feature & 31)));
+    }
+
+    pub fn finalize_features(&self) {
+        // Check VirtIO version 1 compliance.
+        assert!(self.check_device_feature(VIRTIO_F_VERSION_1));
+        self.ack_driver_feature(VIRTIO_F_VERSION_1);
+
+        let mut common = self.common.lock().unwrap();
+
+        let status = common.device_status.get();
+        common
+            .device_status
+            .set(status | DeviceStatusFlags::FEATURES_OK);
+
+        // Re-read device status to ensure the `FEATURES_OK` bit is still set: otherwise,
+        // the device does not support our subset of features and the device is unusable.
+        let confirm = common.device_status.get();
+        assert!((confirm & DeviceStatusFlags::FEATURES_OK) == DeviceStatusFlags::FEATURES_OK);
+    }
+
+    pub fn run_device(&self) {
+        let mut common = self.common.lock().unwrap();
+
+        let status = common.device_status.get();
+        common
+            .device_status
+            .set(status | DeviceStatusFlags::DRIVER_OK);
+    }
+
+    pub fn setup_queue(&self, vector: u16) -> Result<Arc<Queue<'a>>, Error> {
+        let mut common = self.common.lock().unwrap();
+
+        let queue_index = self.queue_index.fetch_add(1, Ordering::SeqCst);
+        common.queue_select.set(queue_index);
+
+        let queue_size = common.queue_size.get() as usize;
+        let queue_notify_idx = common.queue_notify_off.get();
+
+        log::info!("notify_idx: {}", queue_notify_idx);
+
+        // Allocate memory for the queue structues.
+        let descriptor = unsafe {
+            Dma::<[Descriptor]>::zeroed_unsized(queue_size).map_err(Error::SyscallError)?
+        };
+
+        let mut avail = Available::new(queue_size)?;
+        let mut used = Used::new(queue_size)?;
+
+        for i in 0..queue_size {
+            // XXX: Fill the `table_index` of the elements with `T::MAX` to help with
+            //      debugging since qemu reports them as illegal values.
+            avail.get_element_at(i).table_index.set(u16::MAX);
+            used.get_element_at(i).table_index.set(u32::MAX);
+        }
+
+        common.queue_desc.set(descriptor.physical() as u64);
+        common.queue_driver.set(avail.phys_addr() as u64);
+        common.queue_device.set(used.phys_addr() as u64);
+
+        // Set the MSI-X vector.
+        common.queue_msix_vector.set(vector);
+        assert!(common.queue_msix_vector.get() == vector);
+
+        // Enable the queue.
+        common.queue_enable.set(1);
+
+        let notification_bell = unsafe {
+            let offset = self.notify_mul * queue_notify_idx as u32;
+            &mut *(self.notify.add(offset as usize) as *mut VolatileCell<u16>)
+        };
+
+        log::info!("virtio: enabled queue #{queue_index} (size={queue_size})");
+        Ok(Queue::new(descriptor, avail, used, notification_bell))
+    }
+}

virtio-core/src/utils.rs

@@ -0,0 +1,65 @@
+use core::cell::UnsafeCell;
+use core::marker::PhantomData;
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct VolatileCell<T> {
+    value: UnsafeCell<T>,
+}
+
+impl<T: Copy> VolatileCell<T> {
+    #[inline]
+    pub fn new(value: T) -> Self {
+        Self {
+            value: UnsafeCell::new(value),
+        }
+    }
+
+    /// Returns a copy of the contained value.
+    #[inline]
+    pub fn get(&self) -> T {
+        unsafe { core::ptr::read_volatile(self.value.get()) }
+    }
+
+    /// Sets the contained value.
+    #[inline]
+    pub fn set(&mut self, value: T) {
+        unsafe { core::ptr::write_volatile(self.value.get(), value) }
+    }
+}
+
+unsafe impl<T> Sync for VolatileCell<T> {}
+
+#[repr(C)]
+pub struct IncompleteArrayField<T>(PhantomData<T>, [T; 0]);
+
+impl<T> IncompleteArrayField<T> {
+    #[inline]
+    pub const fn new() -> Self {
+        IncompleteArrayField(PhantomData, [])
+    }
+
+    #[inline]
+    pub unsafe fn as_slice(&self, len: usize) -> &[T] {
+        core::slice::from_raw_parts(self.as_ptr(), len)
+    }
+
+    #[inline]
+    pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] {
+        core::slice::from_raw_parts_mut(self.as_mut_ptr(), len)
+    }
+
+    #[inline]
+    pub unsafe fn as_ptr(&self) -> *const T {
+        self as *const _ as *const T
+    }
+
+    #[inline]
+    pub unsafe fn as_mut_ptr(&mut self) -> *mut T {
+        self as *mut _ as *mut T
+    }
+}
+
+pub const fn align(val: usize, align: usize) -> usize {
+    (val + align) & !align
+}