vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

virtiod: split into virtio-core and virtiod

Browse Source 
Signed-off-by: Anhad Singh <andypython@protonmail.com>
This commit is contained in:
Anhad Singh
2023-06-27 09:51:16 +10:00
parent ea1c855cdc
commit eee780fa0f
11 changed files with 378 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
virtiod/Cargo.toml
+2 -2
View File
@@ -5,11 +5,10 @@ edition = "2021"
authors = ["Anhad Singh <andypython@protonmail.com>"]
[dependencies]
static_assertions = "1.1.0"
bitflags = "2.3.2"
anyhow = "1.0.71"
log = "0.4"
thiserror = "1.0.40"
static_assertions = "1.1.0"
redox-daemon = "0.1"
redox-log = "0.1"
@@ -19,3 +18,4 @@ partitionlib = { git = "https://gitlab.redox-os.org/redox-os/partitionlib.git" }
block-io-wrapper = { path = "../block-io-wrapper" }
pcid = { path = "../pcid" }
virtio-core = { path = "../virtio-core" }
virtiod/VIRTIO.md
-20
View File
@@ -1,20 +0,0 @@
## Generic
- [x] Reset the device.
- [x] Set the ACKNOWLEDGE status bit: the guest OS has noticed the device.
- [x] Set the DRIVER status bit: the guest OS knows how to drive the device.
- [x] Setup Interrupts
## Driver Specific
- [x] Read device feature bits, and write the subset of feature bits understood by the OS and driver to the device. During this step the driver MAY read (but MUST NOT write) the device-specific configuration fields to check that it can support the device before accepting it.
- [x] Set the FEATURES_OK status bit. The driver MUST NOT accept new feature bits after this step.
- [x] Re-read device status to ensure the FEATURES_OK bit is still set: otherwise, the device does not support our subset of features and the device is unusable.
- [x] Perform device-specific setup, including discovery of virtqueues for the device, optional per-bus setup, reading and possibly writing the device’s virtio configuration space, and population of virtqueues.
- [x] Set the DRIVER_OK status bit. At this point the device is “live”.
## XXX
- [ ] Mark the deamon as ready.
## Drivers
- [ ] `virtio-blk` (in-progress)
- [ ] `virtio-net`
- [ ] `virtio-gpu`
virtiod/src/lib.rs
-267
View File
@@ -1,267 +0,0 @@
//! https://docs.oasis-open.org/virtio/virtio/v1.1/virtio-v1.1.html
#![feature(int_roundings)]
use static_assertions::const_assert_eq;
pub mod transport;
pub mod utils;
use utils::{IncompleteArrayField, VolatileCell};
use thiserror::Error;
#[derive(Debug, Error)]
pub enum Error {
#[error("capability {0:?} not found")]
InCapable(CfgType),
#[error("failed to map memory")]
Physmap,
#[error("failed to allocate an interrupt vector")]
ExhaustedInt,
#[error("syscall error")]
SyscallError(syscall::Error),
}
#[derive(Debug, Copy, Clone)]
#[repr(u8)]
pub enum CfgType {
/// Common Configuration.
Common = 1,
/// Notifications.
Notify = 2,
/// ISR Status.
Isr = 3,
/// Device specific configuration.
Device = 4,
/// PCI configuration access.
PciConfig = 5,
/// Shared memory region.
SharedMemory = 8,
/// Vendor-specific data.
Vendor = 9,
}
const_assert_eq!(core::mem::size_of::<CfgType>(), 1);
#[derive(Debug, Copy, Clone)]
#[repr(C, packed)]
pub struct PciCapability {
/// Identifies the structure.
pub cfg_type: CfgType,
/// Where to find it.
pub bar: u8,
/// Pad to a full dword.
pub padding: [u8; 3],
/// Offset within the bar.
pub offset: u32,
/// Length of the structure, in bytes.
pub length: u32,
notify_multiplier: u32,
}
// The size of `PciCapability` is 13 bytes since the generic
// PCI fields are *not* included.
const_assert_eq!(core::mem::size_of::<PciCapability>(), 17);
impl PciCapability {
/// ## Safety
/// Undefined if accessed from a capability type other than [`CfgType::Notify`].
pub unsafe fn notify_multiplier(&self) -> u32 {
self.notify_multiplier
}
}
bitflags::bitflags! {
#[derive(Debug, Copy, Clone, PartialEq)]
#[repr(transparent)]
pub struct DeviceStatusFlags: u8 {
/// Indicates that the guest OS has found the device and recognized it as a
/// valid device.
const ACKNOWLEDGE = 1;
/// Indicates that the guest OS knows how to drive the device.
const DRIVER = 2;
/// Indicates that something went wrong in the guest and it has given up on
/// the device.
const FAILED = 128;
/// Indicates that the driver has acknowledged all the features it understands
/// and feature negotiation is complete.
const FEATURES_OK = 8;
/// Indicates that the driver is set up and ready to drive the device.
const DRIVER_OK = 4;
/// Indicates that the device has experienced an error from which it can’t recover.
const DEVICE_NEEDS_RESET = 64;
}
}
#[derive(Debug)]
#[repr(C)]
pub struct CommonCfg {
// About the whole device.
pub device_feature_select: VolatileCell<u32>, // read-write
pub device_feature: VolatileCell<u32>, // read-only for driver
pub driver_feature_select: VolatileCell<u32>, // read-write
pub driver_feature: VolatileCell<u32>, // read-write
pub msix_config: VolatileCell<u16>, // read-write
pub num_queues: VolatileCell<u16>, // read-only for driver
pub device_status: VolatileCell<DeviceStatusFlags>, // read-write
pub config_generation: VolatileCell<u8>, // read-only for driver
// About a specific virtqueue.
pub queue_select: VolatileCell<u16>, // read-write
pub queue_size: VolatileCell<u16>, // read-write
pub queue_msix_vector: VolatileCell<u16>, // read-write
pub queue_enable: VolatileCell<u16>, // read-write
pub queue_notify_off: VolatileCell<u16>, // read-only for driver
pub queue_desc: VolatileCell<u64>, // read-write
pub queue_driver: VolatileCell<u64>, // read-write
pub queue_device: VolatileCell<u64>, // read-write
}
const_assert_eq!(core::mem::size_of::<CommonCfg>(), 56);
bitflags::bitflags! {
#[derive(Debug, Copy, Clone)]
#[repr(transparent)]
pub struct DescriptorFlags: u16 {
/// The next field contains linked buffer index.
const NEXT = 1 << 0;
/// The buffer is write-only (otherwise read-only).
const WRITE_ONLY = 1 << 1;
/// The buffer contains a list of buffer descriptors.
const INDIRECT = 1 << 2;
}
}
#[repr(C)]
pub struct Descriptor {
/// Address (guest-physical).
pub address: u64,
/// Size of the descriptor.
pub size: u32,
pub flags: DescriptorFlags,
/// Index of next desciptor in chain.
pub next: u16,
}
const_assert_eq!(core::mem::size_of::<Descriptor>(), 16);
/// This indicates compliance with the version 1 VirtIO specification.
///
/// See `6.1 Driver Requirements: Reserved Feature Bits` section of the VirtIO
/// specification for more information.
pub const VIRTIO_F_VERSION_1: u32 = 32;
// ======== Available Ring ========
//
// XXX: The driver uses the available ring to offer buffers to the
// device. Each ring entry refers to the head of a descriptor
// chain.
#[repr(C)]
pub struct AvailableRingElement {
pub table_index: VolatileCell<u16>,
}
const_assert_eq!(core::mem::size_of::<AvailableRingElement>(), 2);
#[repr(C)]
pub struct AvailableRing {
pub flags: VolatileCell<u16>,
pub head_index: VolatileCell<u16>,
pub elements: IncompleteArrayField<AvailableRingElement>,
}
const_assert_eq!(core::mem::size_of::<AvailableRing>(), 4);
impl Default for AvailableRing {
fn default() -> Self {
Self {
flags: VolatileCell::new(0),
head_index: VolatileCell::new(0),
elements: IncompleteArrayField::new(),
}
}
}
#[repr(C)]
pub struct AvailableRingExtra {
pub avail_event: VolatileCell<u16>, // Only if `VIRTIO_F_EVENT_IDX`
}
const_assert_eq!(core::mem::size_of::<AvailableRingExtra>(), 2);
// ======== Used Ring ========
#[repr(C)]
pub struct UsedRingElement {
pub table_index: VolatileCell<u32>,
pub written: VolatileCell<u32>,
}
const_assert_eq!(core::mem::size_of::<UsedRingElement>(), 8);
#[repr(C)]
pub struct UsedRing {
pub flags: VolatileCell<u16>,
pub head_index: VolatileCell<u16>,
pub elements: IncompleteArrayField<UsedRingElement>,
}
const_assert_eq!(core::mem::size_of::<UsedRing>(), 4);
impl Default for UsedRing {
fn default() -> Self {
Self {
flags: VolatileCell::new(0),
head_index: VolatileCell::new(0),
elements: IncompleteArrayField::new(),
}
}
}
#[repr(C)]
pub struct UsedRingExtra {
pub event_index: VolatileCell<u16>,
}
// ======== Utils ========
pub struct Buffer {
buffer: usize,
size: usize,
flags: DescriptorFlags,
}
impl Buffer {
pub fn new<T>(val: &syscall::Dma<T>) -> Self {
Self {
buffer: val.physical(),
size: core::mem::size_of::<T>(),
flags: DescriptorFlags::empty(),
}
}
pub fn flags(mut self, flags: DescriptorFlags) -> Self {
self.flags = flags;
self
}
}
pub struct ChainBuilder {
buffers: Vec<Buffer>,
}
impl ChainBuilder {
pub fn new() -> Self {
Self {
buffers: Vec::new(),
}
}
pub fn chain(mut self, buffer: Buffer) -> Self {
self.buffers.push(buffer);
self
}
pub fn build(self) -> Vec<Buffer> {
self.buffers
}
}
virtiod/src/main.rs
+32 -267
View File
@@ -1,8 +1,6 @@
#![deny(trivial_numeric_casts, unused_allocation)]
#![feature(int_roundings)]
use core::ptr::NonNull;
use std::fs::File;
use std::io;
use std::io::{Read, Write};
@@ -10,22 +8,30 @@ use std::os::fd::{AsRawFd, FromRawFd, RawFd};
use static_assertions::const_assert_eq;
use virtiod::transport::StandardTransport;
use virtiod::*;
use pcid_interface::irq_helpers::{allocate_single_interrupt_vector, read_bsp_apic_id};
use pcid_interface::msi::x86_64 as x86_64_msix;
use pcid_interface::msi::x86_64::DeliveryMode;
use pcid_interface::msi::{MsixCapability, MsixTableEntry};
use pcid_interface::*;
use virtio_core::spec::*;
use event::EventQueue;
use syscall::{Io, Packet, SchemeBlockMut, PHYSMAP_NO_CACHE, PHYSMAP_WRITE};
use syscall::{Packet, SchemeBlockMut};
use virtiod::utils::VolatileCell;
use virtio_core::utils::VolatileCell;
mod scheme;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum Error {
#[error("capability {0:?} not found")]
InCapable(CfgType),
#[error("failed to map memory")]
Physmap,
#[error("failed to allocate an interrupt vector")]
ExhaustedInt,
#[error("syscall error")]
SyscallError(syscall::Error),
}
pub fn main() -> anyhow::Result<()> {
#[cfg(target_os = "redox")]
setup_logging();
@@ -74,110 +80,6 @@ fn setup_logging() {
log::info!("virtiod: enabled logger");
}
struct MsixInfo {
pub virt_table_base: NonNull<MsixTableEntry>,
pub virt_pba_base: NonNull<u64>,
pub capability: MsixCapability,
}
impl MsixInfo {
pub unsafe fn table_entry_pointer_unchecked(&mut self, k: usize) -> &mut MsixTableEntry {
&mut *self.virt_table_base.as_ptr().add(k)
}
pub fn table_entry_pointer(&mut self, k: usize) -> &mut MsixTableEntry {
assert!(k < self.capability.table_size() as usize);
unsafe { self.table_entry_pointer_unchecked(k) }
}
}
const_assert_eq!(std::mem::size_of::<MsixTableEntry>(), 16);
const MSIX_PRIMARY_VECTOR: u16 = 0;
fn enable_msix(pcid_handle: &mut PcidServerHandle) -> anyhow::Result<File> {
let pci_config = pcid_handle.fetch_config()?;
// Extended message signaled interrupts.
let capability = match pcid_handle.feature_info(PciFeature::MsiX)? {
PciFeatureInfo::MsiX(capability) => capability,
_ => unreachable!(),
};
let table_size = capability.table_size();
let table_base = capability.table_base_pointer(pci_config.func.bars);
let table_min_length = table_size * 16;
let pba_min_length = table_size.div_ceil(8);
let pba_base = capability.pba_base_pointer(pci_config.func.bars);
let bir = capability.table_bir() as usize;
let bar = pci_config.func.bars[bir];
let bar_size = pci_config.func.bar_sizes[bir] as u64;
let bar_ptr = match bar {
PciBar::Memory32(ptr) => ptr.into(),
PciBar::Memory64(ptr) => ptr,
_ => unreachable!(),
};
let address = unsafe {
syscall::physmap(
bar_ptr as usize,
bar_size as usize,
PHYSMAP_WRITE | PHYSMAP_NO_CACHE,
)
.map_err(|_| Error::Physmap)?
};
// Ensure that the table and PBA are be within the BAR.
{
let bar_range = bar_ptr..bar_ptr + bar_size;
assert!(bar_range.contains(&(table_base as u64 + table_min_length as u64)));
assert!(bar_range.contains(&(pba_base as u64 + pba_min_length as u64)));
}
let virt_table_base = ((table_base - bar_ptr as usize) + address) as *mut MsixTableEntry;
let virt_pba_base = ((pba_base - bar_ptr as usize) + address) as *mut u64;
let mut info = MsixInfo {
virt_table_base: NonNull::new(virt_table_base).unwrap(),
virt_pba_base: NonNull::new(virt_pba_base).unwrap(),
capability,
};
// Allocate the primary MSI vector.
let interrupt_handle = {
let table_entry_pointer = info.table_entry_pointer(MSIX_PRIMARY_VECTOR as usize);
let destination_id = read_bsp_apic_id()?;
let lapic_id = u8::try_from(destination_id).unwrap();
let rh = false;
let dm = false;
let addr = x86_64_msix::message_address(lapic_id, rh, dm);
let (vector, interrupt_handle) =
allocate_single_interrupt_vector(destination_id)?.ok_or(Error::ExhaustedInt)?;
let msg_data = x86_64_msix::message_data_edge_triggered(DeliveryMode::Fixed, vector);
table_entry_pointer.addr_lo.write(addr);
table_entry_pointer.addr_hi.write(0);
table_entry_pointer.msg_data.write(msg_data);
table_entry_pointer
.vec_ctl
.writef(MsixTableEntry::VEC_CTL_MASK_BIT, false);
interrupt_handle
};
pcid_handle.enable_feature(PciFeature::MsiX)?;
log::info!("virtio: using MSI-X (interrupt_handle={interrupt_handle:?})");
Ok(interrupt_handle)
}
#[repr(C)]
pub struct BlockGeometry {
pub cylinders: VolatileCell<u16>,
@@ -224,148 +126,35 @@ const_assert_eq!(core::mem::size_of::<BlockVirtRequest>(), 16);
fn deamon(deamon: redox_daemon::Daemon) -> anyhow::Result<()> {
let mut pcid_handle = PcidServerHandle::connect_default()?;
let pci_config = pcid_handle.fetch_config()?;
let pci_header = pcid_handle.fetch_header()?;
// Double check that we have the right device.
//
// 0x1001 - virtio-blk
let pci_config = pcid_handle.fetch_config()?;
assert_eq!(pci_config.func.devid, 0x1001);
log::info!("virtiod: found `virtio-blk` device");
let mut common_addr = None;
let mut notify_addr = None;
let mut isr_addr = None;
let mut device_addr = None;
let mut device = virtio_core::probe_device(&mut pcid_handle)?;
device.transport.finalize_features();
for capability in pcid_handle
.get_capabilities()?
.iter()
.filter_map(|capability| {
if let Capability::Vendor(vendor) = capability {
Some(vendor)
} else {
None
}
})
{
// SAFETY: We have verified that the length of the data is correct.
let capability = unsafe { &*(capability.data.as_ptr() as *const PciCapability) };
match capability.cfg_type {
CfgType::Common | CfgType::Notify | CfgType::Isr | CfgType::Device => {}
_ => continue,
}
let bar = pci_header.get_bar(capability.bar as usize);
let addr = match bar {
PciBar::Memory32(addr) => addr as usize,
PciBar::Memory64(addr) => addr as usize,
_ => unreachable!("virtio: unsupported bar type: {bar:?}"),
};
let address = unsafe {
syscall::physmap(
addr + capability.offset as usize,
capability.length as usize,
PHYSMAP_WRITE | PHYSMAP_NO_CACHE,
)
.map_err(|_| Error::Physmap)?
};
match capability.cfg_type {
CfgType::Common => {
debug_assert!(common_addr.is_none());
common_addr = Some(address);
}
CfgType::Notify => {
debug_assert!(notify_addr.is_none());
// SAFETY: The capability type is `Notify`, so its safe to access
// the `notify_multiplier` field.
let multiplier = unsafe { capability.notify_multiplier() };
notify_addr = Some((address, multiplier));
}
CfgType::Isr => {
debug_assert!(isr_addr.is_none());
isr_addr = Some(address);
}
CfgType::Device => {
debug_assert!(device_addr.is_none());
device_addr = Some(address);
}
_ => unreachable!(),
}
log::info!("virtio: {capability:?}");
}
let common_addr = common_addr.ok_or(Error::InCapable(CfgType::Common))?;
let (notify_addr, notify_multiplier) = notify_addr.ok_or(Error::InCapable(CfgType::Notify))?;
let isr_addr = isr_addr.ok_or(Error::InCapable(CfgType::Isr))?;
let device_addr = device_addr.ok_or(Error::InCapable(CfgType::Device))?;
assert!(
notify_multiplier != 0,
"virtio: device uses the same Queue Notify addresses for all queues"
);
let common = unsafe { &mut *(common_addr as *mut CommonCfg) };
let device_space = unsafe { &mut *(device_addr as *mut BlockDeviceConfig) };
let isr = unsafe { &*(isr_addr as *mut VolatileCell<u32>) };
// Reset the device.
common.device_status.set(DeviceStatusFlags::empty());
// Upon reset, the device must initialize device status to 0.
assert_eq!(common.device_status.get(), DeviceStatusFlags::empty());
log::info!("virtio: successfully reseted the device");
// XXX: According to the virtio specification v1.2, setting the ACKNOWLEDGE and DRIVER bits
// in `device_status` is required to be done in two steps.
common
.device_status
.set(common.device_status.get() | DeviceStatusFlags::ACKNOWLEDGE);
common
.device_status
.set(common.device_status.get() | DeviceStatusFlags::DRIVER);
// Setup interrupts.
let all_pci_features = pcid_handle.fetch_all_features()?;
let has_msix = all_pci_features
.iter()
.any(|(feature, _)| feature.is_msix());
// According to the virtio specification, the device REQUIRED to support MSI-X.
assert!(has_msix, "virtio: device does not support MSI-X");
let mut irq_handle = enable_msix(&mut pcid_handle)?;
log::info!("virtio: using standard PCI transport");
let transport = StandardTransport::new(
pci_header,
common,
notify_addr as *const u8,
notify_multiplier,
);
transport.finalize_features();
let queue = transport.setup_queue(MSIX_PRIMARY_VECTOR)?;
let queue = device
.transport
.setup_queue(virtio_core::MSIX_PRIMARY_VECTOR)?;
let queue_copy = queue.clone();
let device_space = unsafe { &mut *(device.device_space as *mut BlockDeviceConfig) };
std::thread::spawn(move || {
let mut event_queue = EventQueue::<usize>::new().unwrap();
let mut progress_head = 0;
event_queue
.add(
irq_handle.as_raw_fd(),
device.irq_handle.as_raw_fd(),
move |_| -> Result<Option<usize>, io::Error> {
// Read from ISR to acknowledge the interrupt.
let _isr = isr.get() as usize;
let _isr = device.isr.get() as usize;
let mut inner = queue_copy.inner.lock().unwrap();
let used_head = inner.used.head_index();
@@ -394,7 +183,7 @@ fn deamon(deamon: redox_daemon::Daemon) -> anyhow::Result<()> {
drop(inner);
let mut buf = [0u8; 8];
irq_handle.read(&mut buf)?;
device.irq_handle.read(&mut buf)?;
// Acknowledge the interrupt.
// irq_handle.write(&buf)?;
Ok(None)
@@ -408,7 +197,7 @@ fn deamon(deamon: redox_daemon::Daemon) -> anyhow::Result<()> {
});
// At this point the device is alive!
transport.run_device();
device.transport.run_device();
log::info!(
"virtio-blk: disk size: {} sectors and block size of {} bytes",
@@ -444,30 +233,6 @@ fn deamon(deamon: redox_daemon::Daemon) -> anyhow::Result<()> {
.write(&mut packet)
.expect("ahcid: failed to read disk scheme");
}
// for _ in 0..3 {
// let req = syscall::Dma::new(BlockVirtRequest {
// ty: BlockRequestTy::In,
// reserved: 0,
// sector: 0,
// })
// .unwrap();
// let result = syscall::Dma::new([0u8; 512]).unwrap();
// let status = syscall::Dma::new(u8::MAX).unwrap();
// let chain = ChainBuilder::new()
// .chain(Buffer::new(&req).flags(DescriptorFlags::NEXT))
// .chain(Buffer::new(&result).flags(DescriptorFlags::WRITE_ONLY | DescriptorFlags::NEXT))
// .chain(Buffer::new(&status).flags(DescriptorFlags::WRITE_ONLY))
// .build();
// queue.send(chain);
// log::info!("{}", event_queue.run()?);
// log::info!("command status: {}", *status);
// log::info!("data: {:?}", result.as_ref());
// }
}
fn daemon_runner(redox_daemon: redox_daemon::Daemon) -> ! {
virtiod/src/scheme.rs
+5 -7
View File
@@ -10,10 +10,9 @@ use std::sync::Arc;
use partitionlib::LogicalBlockSize;
use partitionlib::PartitionTable;
use syscall::*;
use virtiod::transport::Queue;
use virtiod::Buffer;
use virtiod::ChainBuilder;
use virtiod::DescriptorFlags;
use virtio_core::spec::{Buffer, ChainBuilder, DescriptorFlags};
use virtio_core::transport::Queue;
use crate::BlockDeviceConfig;
use crate::BlockRequestTy;
@@ -28,6 +27,8 @@ trait BlkExtension {
/// XXX: Reads only one block despite the size of the output buffer. Use [`BlkExtension::write`] instead.
fn write_block(&self, block: u64, block_bytes: &[u8]) -> usize;
// FIXME(andypython): The following two methods can be done asynchronously (i.e, send all of the commands at once
// and wait for the result in the end).
fn read(&self, block: u64, block_bytes: &mut [u8]) -> usize {
let sectors = block_bytes.len() / BLK_SIZE as usize;
@@ -72,8 +73,6 @@ impl BlkExtension for Queue<'_> {
let size = core::cmp::min(block_bytes.len(), result.len());
block_bytes[..size].copy_from_slice(&result.as_slice()[..size]);
std::thread::yield_now();
size
}
@@ -103,7 +102,6 @@ impl BlkExtension for Queue<'_> {
core::hint::spin_loop();
}
std::thread::yield_now();
block_bytes.len()
}
}
virtiod/src/transport.rs
-410
View File
@@ -1,410 +0,0 @@
use crate::utils::align;
use crate::*;
use pcid_interface::PciHeader;
use syscall::{Dma, PHYSMAP_WRITE};
use core::mem::size_of;
use core::sync::atomic::{fence, AtomicU16, Ordering};
use std::collections::VecDeque;
use std::sync::{Arc, Mutex, Weak};
/// Returns the queue part sizes in bytes.
///
/// ## Reference
/// Section 2.7 Split Virtqueues of the specfication v1.2 describes the alignment
/// and size of the queue parts.
///
/// ## Panics
/// If `queue_size` is not a power of two or is zero.
const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) {
assert!(queue_size.is_power_of_two() && queue_size != 0);
const DESCRIPTOR_ALIGN: usize = 16;
const AVAILABLE_ALIGN: usize = 2;
const USED_ALIGN: usize = 4;
let queue_size = queue_size as usize;
let desc = size_of::<Descriptor>() * queue_size;
// `avail_header`: Size of the available ring header and the footer.
let avail_header = size_of::<AvailableRing>() + size_of::<AvailableRingExtra>();
let avail = avail_header + size_of::<AvailableRingElement>() * queue_size;
// `used_header`: Size of the used ring header and the footer.
let used_header = size_of::<UsedRing>() + size_of::<UsedRingExtra>();
let used = used_header + size_of::<UsedRingElement>() * queue_size;
(
align(desc, DESCRIPTOR_ALIGN),
align(avail, AVAILABLE_ALIGN),
align(used, USED_ALIGN),
)
}
pub struct QueueInner<'a> {
pub descriptor: Dma<[Descriptor]>,
pub available: Available<'a>,
pub used: Used<'a>,
/// Keeps track of unused descriptor indicies.
pub descriptor_stack: VecDeque<u16>,
notification_bell: &'a mut VolatileCell<u16>,
head_index: u16,
}
unsafe impl Sync for QueueInner<'_> {}
unsafe impl Send for QueueInner<'_> {}
pub struct Queue<'a> {
pub inner: Mutex<QueueInner<'a>>,
sref: Weak<Self>,
}
impl<'a> Queue<'a> {
pub fn new(
descriptor: Dma<[Descriptor]>,
available: Available<'a>,
used: Used<'a>,
notification_bell: &'a mut VolatileCell<u16>,
) -> Arc<Self> {
Arc::new_cyclic(|sref| Self {
inner: Mutex::new(QueueInner {
head_index: 0,
descriptor_stack: (0..(descriptor.len() - 1) as u16).rev().collect(),
descriptor,
available,
used,
notification_bell,
}),
sref: sref.clone(),
})
}
pub fn send(&self, chain: Vec<Buffer>) {
let mut first_descriptor: Option<usize> = None;
let mut last_descriptor: Option<usize> = None;
for buffer in chain.iter() {
let descriptor = self.alloc_descriptor();
let mut inner = self.inner.lock().unwrap();
if first_descriptor.is_none() {
first_descriptor = Some(descriptor);
}
inner.descriptor[descriptor].address = buffer.buffer as u64;
inner.descriptor[descriptor].flags = buffer.flags;
inner.descriptor[descriptor].size = buffer.size as u32;
if let Some(index) = last_descriptor {
inner.descriptor[index].next = descriptor as u16;
}
last_descriptor = Some(descriptor);
}
let mut inner = self.inner.lock().unwrap();
let last_descriptor = last_descriptor.unwrap();
let first_descriptor = first_descriptor.unwrap();
inner.descriptor[last_descriptor].next = 0;
fence(Ordering::SeqCst);
let index = inner.head_index as usize;
inner
.available
.get_element_at(index)
.table_index
.set(first_descriptor as u16);
fence(Ordering::SeqCst);
inner.available.set_head_idx(index as u16 + 1);
inner.head_index += 1;
assert_eq!(inner.used.flags(), 0);
inner.notification_bell.set(0); // FIXME: This corresponds to the queue index.
}
fn alloc_descriptor(&self) -> usize {
let mut inner = self.inner.lock().unwrap();
if let Some(index) = inner.descriptor_stack.pop_front() {
index as usize
} else {
log::warn!("virtiod: descriptors exhausted, waiting on garabage collector");
drop(inner);
// Wait for the garbage collector thread to release some descriptors.
//
// TODO(andypython): Instead of just yielding, we should have a proper notificiation
// mechanism. I am not aware whats the standard way redox applications
// or drivers implement basically a WaitQueue which you can use to wake
// up a thread. The descripts really should NEVER run out, but if they
// do, have a proper way to handle them.
std::thread::yield_now();
self.alloc_descriptor()
}
}
}
pub struct Available<'a> {
addr: usize,
size: usize,
queue_size: usize,
ring: &'a mut AvailableRing,
}
impl<'a> Available<'a> {
pub fn new(queue_size: usize) -> anyhow::Result<Self> {
let (_, size, _) = queue_part_sizes(queue_size);
let size = size.next_multiple_of(syscall::PAGE_SIZE); // align to page size
let addr = unsafe { syscall::physalloc(size) }.map_err(Error::SyscallError)?;
let virt =
unsafe { syscall::physmap(addr, size, PHYSMAP_WRITE) }.map_err(Error::SyscallError)?;
let ring = unsafe { &mut *(virt as *mut AvailableRing) };
Ok(Self {
addr,
size,
ring,
queue_size,
})
}
/// ## Panics
/// This function panics if the index is out of bounds.
pub fn get_element_at(&mut self, index: usize) -> &mut AvailableRingElement {
// SAFETY: We have exclusive access to the elements and the number of elements
// is correct; same as the queue size.
unsafe {
self.ring
.elements
.as_mut_slice(self.queue_size)
.get_mut(index % 256)
.expect("virtio::available: index out of bounds")
}
}
pub fn set_head_idx(&mut self, index: u16) {
self.ring.head_index.set(index);
}
pub fn phys_addr(&self) -> usize {
self.addr
}
}
impl Drop for Available<'_> {
fn drop(&mut self) {
log::warn!("virtio: dropping 'available' ring at {:#x}", self.addr);
unsafe {
syscall::physunmap(self.addr).unwrap();
syscall::physfree(self.addr, self.size).unwrap();
}
}
}
pub struct Used<'a> {
addr: usize,
size: usize,
queue_size: usize,
ring: &'a mut UsedRing,
}
impl<'a> Used<'a> {
pub fn new(queue_size: usize) -> anyhow::Result<Self> {
let (_, _, size) = queue_part_sizes(queue_size);
let size = size.next_multiple_of(syscall::PAGE_SIZE); // align to page size
let addr = unsafe { syscall::physalloc(size) }.map_err(Error::SyscallError)?;
let virt =
unsafe { syscall::physmap(addr, size, PHYSMAP_WRITE) }.map_err(Error::SyscallError)?;
let ring = unsafe { &mut *(virt as *mut UsedRing) };
Ok(Self {
addr,
size,
ring,
queue_size,
})
}
/// ## Panics
/// This function panics if the index is out of bounds.
pub fn get_element_at(&mut self, index: usize) -> &mut UsedRingElement {
// SAFETY: We have exclusive access to the elements and the number of elements
// is correct; same as the queue size.
unsafe {
self.ring
.elements
.as_mut_slice(self.queue_size)
.get_mut(index % 256)
.expect("virtio::used: index out of bounds")
}
}
pub fn flags(&self) -> u16 {
self.ring.flags.get()
}
pub fn head_index(&self) -> u16 {
self.ring.head_index.get()
}
pub fn phys_addr(&self) -> usize {
self.addr
}
}
impl Drop for Used<'_> {
fn drop(&mut self) {
log::warn!("virtio: dropping 'used' ring at {:#x}", self.addr);
unsafe {
syscall::physunmap(self.addr).unwrap();
syscall::physfree(self.addr, self.size).unwrap();
}
}
}
pub struct StandardTransport<'a> {
header: PciHeader,
common: Mutex<&'a mut CommonCfg>,
notify: *const u8,
notify_mul: u32,
queue_index: AtomicU16,
sref: Weak<Self>,
}
impl<'a> StandardTransport<'a> {
pub fn new(
header: PciHeader,
common: &'a mut CommonCfg,
notify: *const u8,
notify_mul: u32,
) -> Arc<Self> {
Arc::new_cyclic(|sref| Self {
header,
common: Mutex::new(common),
notify,
notify_mul,
queue_index: AtomicU16::new(0),
sref: sref.clone(),
})
}
pub fn sref(&self) -> Arc<Self> {
// UNWRAP: The constructor ensures that we are wrapped in our own `Arc`. So this
// unwrap is going to be unreachable.
self.sref.upgrade().unwrap()
}
pub fn check_device_feature(&self, feature: u32) -> bool {
let mut common = self.common.lock().unwrap();
common.device_feature_select.set(feature >> 5);
(common.device_feature.get() & (1 << (feature & 31))) != 0
}
pub fn ack_driver_feature(&self, feature: u32) {
let mut common = self.common.lock().unwrap();
common.driver_feature_select.set(feature >> 5);
let current = common.driver_feature.get();
common.driver_feature.set(current | (1 << (feature & 31)));
}
pub fn finalize_features(&self) {
// Check VirtIO version 1 compliance.
assert!(self.check_device_feature(VIRTIO_F_VERSION_1));
self.ack_driver_feature(VIRTIO_F_VERSION_1);
let mut common = self.common.lock().unwrap();
let status = common.device_status.get();
common
.device_status
.set(status | DeviceStatusFlags::FEATURES_OK);
// Re-read device status to ensure the `FEATURES_OK` bit is still set: otherwise,
// the device does not support our subset of features and the device is unusable.
let confirm = common.device_status.get();
assert!((confirm & DeviceStatusFlags::FEATURES_OK) == DeviceStatusFlags::FEATURES_OK);
}
pub fn run_device(&self) {
let mut common = self.common.lock().unwrap();
let status = common.device_status.get();
common
.device_status
.set(status | DeviceStatusFlags::DRIVER_OK);
}
pub fn setup_queue(&self, vector: u16) -> anyhow::Result<Arc<Queue<'a>>> {
let mut common = self.common.lock().unwrap();
let queue_index = self.queue_index.fetch_add(1, Ordering::SeqCst);
common.queue_select.set(queue_index);
let queue_size = common.queue_size.get() as usize;
let queue_notify_idx = common.queue_notify_off.get();
log::info!("notify_idx: {}", queue_notify_idx);
// Allocate memory for the queue structues.
let descriptor = unsafe {
Dma::<[Descriptor]>::zeroed_unsized(queue_size).map_err(Error::SyscallError)?
};
let mut avail = Available::new(queue_size)?;
let mut used = Used::new(queue_size)?;
for i in 0..queue_size {
// XXX: Fill the `table_index` of the elements with `T::MAX` to help with
// debugging since qemu reports them as illegal values.
avail.get_element_at(i).table_index.set(u16::MAX);
used.get_element_at(i).table_index.set(u32::MAX);
}
common.queue_desc.set(descriptor.physical() as u64);
common.queue_driver.set(avail.phys_addr() as u64);
common.queue_device.set(used.phys_addr() as u64);
// Set the MSI-X vector.
common.queue_msix_vector.set(vector);
assert!(common.queue_msix_vector.get() == vector);
// Enable the queue.
common.queue_enable.set(1);
let notification_bell = unsafe {
let offset = self.notify_mul * queue_notify_idx as u32;
&mut *(self.notify.add(offset as usize) as *mut VolatileCell<u16>)
};
log::info!("virtio: enabled queue #{queue_index} (size={queue_size})");
Ok(Queue::new(descriptor, avail, used, notification_bell))
}
}
virtiod/src/utils.rs
-65
View File
@@ -1,65 +0,0 @@
use core::cell::UnsafeCell;
use core::marker::PhantomData;
#[derive(Debug)]
#[repr(C)]
pub struct VolatileCell<T> {
value: UnsafeCell<T>,
}
impl<T: Copy> VolatileCell<T> {
#[inline]
pub fn new(value: T) -> Self {
Self {
value: UnsafeCell::new(value),
}
}
/// Returns a copy of the contained value.
#[inline]
pub fn get(&self) -> T {
unsafe { core::ptr::read_volatile(self.value.get()) }
}
/// Sets the contained value.
#[inline]
pub fn set(&mut self, value: T) {
unsafe { core::ptr::write_volatile(self.value.get(), value) }
}
}
unsafe impl<T> Sync for VolatileCell<T> {}
#[repr(C)]
pub struct IncompleteArrayField<T>(PhantomData<T>, [T; 0]);
impl<T> IncompleteArrayField<T> {
#[inline]
pub const fn new() -> Self {
IncompleteArrayField(PhantomData, [])
}
#[inline]
pub unsafe fn as_slice(&self, len: usize) -> &[T] {
core::slice::from_raw_parts(self.as_ptr(), len)
}
#[inline]
pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] {
core::slice::from_raw_parts_mut(self.as_mut_ptr(), len)
}
#[inline]
pub unsafe fn as_ptr(&self) -> *const T {
self as *const _ as *const T
}
#[inline]
pub unsafe fn as_mut_ptr(&mut self) -> *mut T {
self as *mut _ as *mut T
}
}
pub const fn align(val: usize, align: usize) -> usize {
(val + align) & !align
}