From 6a5e9d2613fbb92abaafa1969ae58a2039cd1358 Mon Sep 17 00:00:00 2001 From: Anhad Singh Date: Thu, 10 Aug 2023 17:22:38 +1000 Subject: [PATCH] virtio-core: add support for legacy transport Signed-off-by: Anhad Singh --- Cargo.lock | 1 + inputd/src/main.rs | 37 ++- virtio-blkd/Cargo.toml | 1 + virtio-blkd/src/main.rs | 57 +++-- virtio-blkd/src/scheme.rs | 6 +- virtio-core/src/probe.rs | 118 +++++---- virtio-core/src/transport.rs | 449 +++++++++++++++++++++++++++++------ virtio-core/src/utils.rs | 2 +- virtio-gpud/src/scheme.rs | 35 ++- virtio-netd/src/main.rs | 2 +- 10 files changed, 544 insertions(+), 164 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67b819364a..82abf7976e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1864,6 +1864,7 @@ dependencies = [ "redox-daemon", "redox-log", "redox_syscall 0.3.5", + "spin", "static_assertions", "thiserror", "virtio-core", diff --git a/inputd/src/main.rs b/inputd/src/main.rs index fa37f58b2c..f1d06198fb 100644 --- a/inputd/src/main.rs +++ b/inputd/src/main.rs @@ -483,20 +483,20 @@ pub fn main() { // provided by the firmware. The GPU devices are latter started by `pcid` (such as `virtio-gpu`). let mut devices = vec![]; let schemes = std::fs::read_dir(":").unwrap(); - + for entry in schemes { let path = entry.unwrap().path(); let path_str = path .into_os_string() .into_string() .expect("inputd: failed to convert path to string"); - + if path_str.contains("display") { println!("inputd: found display scheme {}", path_str); devices.push(path_str); } } - + let device = devices .iter() .filter(|d| !d.contains("vesa")) @@ -507,10 +507,10 @@ pub fn main() { // TODO: What should we do when there are multiple display devices? device[0].split('/').nth(2).unwrap() }; - + let mut handle = inputd::Handle::new(device).expect("inputd: failed to open display handle"); - + handle .activate(vt, VtMode::Graphic) .expect("inputd: failed to activate VT in graphic mode"); @@ -520,20 +520,33 @@ pub fn main() { "-A" => { let vt = args.next().unwrap().parse::().unwrap(); - let handle = File::open(format!("input:consumer/{vt}")).expect("inputd: failed to open consumer handle"); + let handle = File::open(format!("input:consumer/{vt}")) + .expect("inputd: failed to open consumer handle"); let mut display_path = [0; 4096]; - let written = syscall::fpath(handle.as_raw_fd() as usize, &mut display_path).expect("inputd: fpath() failed"); + let written = syscall::fpath(handle.as_raw_fd() as usize, &mut display_path) + .expect("inputd: fpath() failed"); assert!(written <= display_path.len()); drop(handle); - let display_path = std::str::from_utf8(&display_path[..written]).expect("inputd: display path UTF-8 validation failed"); - let display_name = display_path.split('/').skip(1).next().expect("inputd: invalid display path"); - let display_scheme = display_name.split(':').next().expect("inputd: invalid display path"); + let display_path = std::str::from_utf8(&display_path[..written]) + .expect("inputd: display path UTF-8 validation failed"); + let display_name = display_path + .split('/') + .skip(1) + .next() + .expect("inputd: invalid display path"); + let display_scheme = display_name + .split(':') + .next() + .expect("inputd: invalid display path"); - let mut handle = inputd::Handle::new(display_scheme).expect("inputd: failed to open display handle"); - handle.activate(vt, VtMode::Default).expect("inputd: failed to activate VT"); + let mut handle = inputd::Handle::new(display_scheme) + .expect("inputd: failed to open display handle"); + handle + .activate(vt, VtMode::Default) + .expect("inputd: failed to activate VT"); } _ => panic!("inputd: invalid argument: {}", val), diff --git a/virtio-blkd/Cargo.toml b/virtio-blkd/Cargo.toml index 2068f8923a..3f0f969ea6 100644 --- a/virtio-blkd/Cargo.toml +++ b/virtio-blkd/Cargo.toml @@ -10,6 +10,7 @@ log = "0.4" thiserror = "1.0.40" static_assertions = "1.1.0" futures = { version = "0.3.28", features = ["executor"] } +spin = "*" redox-daemon = "0.1" redox-log = "0.1" diff --git a/virtio-blkd/src/main.rs b/virtio-blkd/src/main.rs index 963156a37b..959361406e 100644 --- a/virtio-blkd/src/main.rs +++ b/virtio-blkd/src/main.rs @@ -1,9 +1,12 @@ #![deny(trivial_numeric_casts, unused_allocation)] #![feature(int_roundings, async_fn_in_trait)] +// TODO(andypython): driver panic with an empty disk. + use std::fs::File; use std::io::{Read, Write}; use std::os::fd::{FromRawFd, RawFd}; +use std::sync::{Arc, Weak}; use static_assertions::const_assert_eq; @@ -12,6 +15,7 @@ use virtio_core::spec::*; use syscall::{Packet, SchemeBlockMut}; +use virtio_core::transport::Transport; use virtio_core::utils::VolatileCell; mod scheme; @@ -43,23 +47,48 @@ pub struct BlockGeometry { pub sectors: VolatileCell, } -#[repr(C)] -pub struct BlockDeviceConfig { - capacity: VolatileCell, - pub size_max: VolatileCell, - pub seq_max: VolatileCell, - pub geometry: BlockGeometry, - blk_size: VolatileCell, +#[repr(u8)] +pub enum DeviceConfigTy { + Capacity = 0, + SizeMax = 0x8, + SeqMax = 0xc, + Geometry = 0x10, + BlkSize = 0x14, } +#[repr(C)] +pub struct BlockDeviceConfig(Weak); + impl BlockDeviceConfig { - /// Returns the capacity of the block device in bytes. - pub fn capacity(&self) -> u64 { - self.capacity.get() + #[inline] + fn new(tranport: &Arc) -> Self { + Self(Arc::downgrade(&tranport)) } + pub fn load_config(&self, ty: DeviceConfigTy) -> T + where + T: Sized + TryFrom, + >::Error: std::fmt::Debug, + { + let transport = self.0.upgrade().unwrap(); + + let size = core::mem::size_of::() + .try_into() + .expect("load_config: invalid size"); + + let value = transport.load_config(ty as u8, size); + T::try_from(value).unwrap() + } + + /// Returns the capacity of the block device in bytes. + #[inline] + pub fn capacity(&self) -> u64 { + self.load_config(DeviceConfigTy::Capacity) + } + + #[inline] pub fn block_size(&self) -> u32 { - self.blk_size.get() + self.load_config(DeviceConfigTy::BlkSize) } } @@ -98,15 +127,15 @@ fn deamon(deamon: redox_daemon::Daemon) -> anyhow::Result<()> { .transport .setup_queue(virtio_core::MSIX_PRIMARY_VECTOR, &device.irq_handle)?; - let device_space = unsafe { &mut *(device.device_space as *mut BlockDeviceConfig) }; + let device_space = BlockDeviceConfig::new(&device.transport); // At this point the device is alive! device.transport.run_device(); log::info!( "virtio-blk: disk size: {} sectors and block size of {} bytes", - device_space.capacity.get(), - device_space.blk_size.get() + device_space.capacity(), + device_space.block_size() ); let mut name = pci_config.func.name(); diff --git a/virtio-blkd/src/scheme.rs b/virtio-blkd/src/scheme.rs index d94f9a76b4..e4ec4c7b1d 100644 --- a/virtio-blkd/src/scheme.rs +++ b/virtio-blkd/src/scheme.rs @@ -99,13 +99,13 @@ pub enum Handle { pub struct DiskScheme<'a> { queue: Arc>, next_id: usize, - cfg: &'a mut BlockDeviceConfig, + cfg: BlockDeviceConfig, handles: BTreeMap, part_table: Option, } impl<'a> DiskScheme<'a> { - pub fn new(queue: Arc>, cfg: &'a mut BlockDeviceConfig) -> Self { + pub fn new(queue: Arc>, cfg: BlockDeviceConfig) -> Self { let mut this = Self { queue, next_id: 0, @@ -157,7 +157,7 @@ impl<'a> DiskScheme<'a> { impl<'a, 'b> Seek for VirtioShim<'a, 'b> { fn seek(&mut self, from: std::io::SeekFrom) -> IoResult { - let size_u = self.scheme.cfg.capacity.get() * self.scheme.cfg.blk_size.get() as u64; + let size_u = self.scheme.cfg.capacity() * self.scheme.cfg.block_size() as u64; let size = i64::try_from(size_u).or(Err(std::io::Error::new( std::io::ErrorKind::Other, "Disk larger than 2^63 - 1 bytes", diff --git a/virtio-core/src/probe.rs b/virtio-core/src/probe.rs index 3e4ca9e1e8..9dc1347cfa 100644 --- a/virtio-core/src/probe.rs +++ b/virtio-core/src/probe.rs @@ -11,11 +11,11 @@ use pcid_interface::*; use syscall::Io; use crate::spec::*; -use crate::transport::{Error, StandardTransport}; +use crate::transport::{Error, LegacyTransport, StandardTransport, Transport}; use crate::utils::{align_down, VolatileCell}; pub struct Device<'a> { - pub transport: Arc>, + pub transport: Arc, pub device_space: *const u8, pub irq_handle: File, pub isr: &'a VolatileCell, @@ -233,59 +233,95 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result log::trace!("virtio-core::device-probe: {capability:?}"); } - let common_addr = common_addr.ok_or(Error::InCapable(CfgType::Common))?; - let (notify_addr, notify_multiplier) = notify_addr.ok_or(Error::InCapable(CfgType::Notify))?; - let isr_addr = isr_addr.ok_or(Error::InCapable(CfgType::Isr))?; - let device_addr = device_addr.ok_or(Error::InCapable(CfgType::Device))?; + if let ( + Some(common_addr), + Some(isr_addr), + Some(device_addr), + Some((notify_addr, notify_multiplier)), + ) = (common_addr, isr_addr, device_addr, notify_addr) + { + assert!( + notify_multiplier != 0, + "virtio-core::device_probe: device uses the same Queue Notify addresses for all queues" + ); - assert!( - notify_multiplier != 0, - "virtio-core::device_probe: device uses the same Queue Notify addresses for all queues" - ); + let common = unsafe { &mut *(common_addr as *mut CommonCfg) }; + let device_space = unsafe { &mut *(device_addr as *mut u8) }; + let isr = unsafe { &*(isr_addr as *mut VolatileCell) }; - let common = unsafe { &mut *(common_addr as *mut CommonCfg) }; - let device_space = unsafe { &mut *(device_addr as *mut u8) }; - let isr = unsafe { &*(isr_addr as *mut VolatileCell) }; + let transport = StandardTransport::new( + common, + notify_addr as *const u8, + notify_multiplier, + device_space, + ); - let transport = StandardTransport::new(common, notify_addr as *const u8, notify_multiplier); + // Setup interrupts. + let all_pci_features = pcid_handle.fetch_all_features()?; + let has_msix = all_pci_features + .iter() + .any(|(feature, _)| feature.is_msix()); - // Setup interrupts. - let all_pci_features = pcid_handle.fetch_all_features()?; - let has_msix = all_pci_features - .iter() - .any(|(feature, _)| feature.is_msix()); + // According to the virtio specification, the device REQUIRED to support MSI-X. + assert!(has_msix, "virtio: device does not support MSI-X"); + let irq_handle = enable_msix(pcid_handle)?; - // According to the virtio specification, the device REQUIRED to support MSI-X. - assert!(has_msix, "virtio: device does not support MSI-X"); - let irq_handle = enable_msix(pcid_handle)?; + log::info!("virtio: using standard PCI transport"); - log::info!("virtio: using standard PCI transport"); + let device = Device { + transport, + device_space, + irq_handle, + isr, + }; - let device = Device { - transport, - device_space, - irq_handle, - isr, - }; + device.transport.reset(); + reinit(&device)?; - device.transport.reset(); - reinit(&device)?; + Ok(device) + } else { + if let PciBar::Port(port) = pci_header.get_bar(0) { + unsafe { syscall::iopl(3).expect("virtio: failed to set I/O privilege level") }; + log::warn!("virtio: using legacy transport"); - Ok(device) + static SHIM: VolatileCell = VolatileCell::new(0); + + let transport = LegacyTransport::new(port); + + // Setup interrupts. + let all_pci_features = pcid_handle.fetch_all_features()?; + let has_msix = all_pci_features + .iter() + .any(|(feature, _)| feature.is_msix()); + + // According to the virtio specification, the device REQUIRED to support MSI-X. + assert!(has_msix, "virtio: device does not support MSI-X"); + let irq_handle = enable_msix(pcid_handle)?; + + let device = Device { + transport, + irq_handle, + isr: &SHIM, + device_space: core::ptr::null_mut(), + }; + + device.transport.reset(); + reinit(&device)?; + + Ok(device) + } else { + unreachable!("virtio: legacy transport with non-port IO?") + } + } } pub fn reinit<'a>(device: &Device<'a>) -> Result<(), Error> { // XXX: According to the virtio specification v1.2, setting the ACKNOWLEDGE and DRIVER bits // in `device_status` is required to be done in two steps. - let mut common = device.transport.common.lock().unwrap(); - let old = common.device_status.get(); - - common - .device_status - .set(old | DeviceStatusFlags::ACKNOWLEDGE); - - let old = common.device_status.get(); - common.device_status.set(old | DeviceStatusFlags::DRIVER); + device + .transport + .insert_status(DeviceStatusFlags::ACKNOWLEDGE); + device.transport.insert_status(DeviceStatusFlags::DRIVER); Ok(()) } diff --git a/virtio-core/src/transport.rs b/virtio-core/src/transport.rs index 9d72b4fab0..53934b6f56 100644 --- a/virtio-core/src/transport.rs +++ b/virtio-core/src/transport.rs @@ -1,8 +1,9 @@ use crate::spec::*; use crate::utils::align; -use common::dma::Dma; +use common::dma::{Dma, PhysBox}; use event::EventQueue; +use syscall::{Io, Pio}; use core::mem::size_of; use core::sync::atomic::{AtomicU16, Ordering}; @@ -62,12 +63,43 @@ const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) { let used = used_header + size_of::() * queue_size; ( - align(desc, DESCRIPTOR_ALIGN), - align(avail, AVAILABLE_ALIGN), - align(used, USED_ALIGN), + align(desc, DESCRIPTOR_ALIGN).next_multiple_of(syscall::PAGE_SIZE), + align(avail, AVAILABLE_ALIGN).next_multiple_of(syscall::PAGE_SIZE), + align(used, USED_ALIGN).next_multiple_of(syscall::PAGE_SIZE), ) } +fn spawn_irq_thread(irq_handle: &File, queue: &Arc>) -> Result<(), Error> { + let irq_fd = irq_handle.as_raw_fd(); + let queue_copy = queue.clone(); + + std::thread::spawn(move || { + let mut event_queue = EventQueue::::new().unwrap(); + + event_queue + .add(irq_fd, move |_| -> Result, std::io::Error> { + // Wake up the tasks waiting on the queue. + for (_, task) in queue_copy.waker.lock().unwrap().iter() { + task.wake_by_ref(); + } + + // Wake up the tasks waiting on the queue. + Ok(None) + }) + .unwrap(); + + loop { + event_queue.run().unwrap(); + } + }); + + Ok(()) +} + +pub trait NotifyBell { + fn ring(&self, queue_index: u16); +} + pub struct PendingRequest<'a> { queue: Arc>, first_descriptor: u32, @@ -133,26 +165,29 @@ pub struct Queue<'a> { pub used_head: AtomicU16, vector: u16, - notification_bell: &'a mut AtomicU16, + notification_bell: Box, descriptor_stack: crossbeam_queue::SegQueue, sref: Weak, } impl<'a> Queue<'a> { - pub fn new( + pub fn new( descriptor: Dma<[Descriptor]>, available: Available<'a>, used: Used<'a>, - notification_bell: &'a mut AtomicU16, + notification_bell: N, queue_index: u16, vector: u16, - ) -> Arc { + ) -> Arc + where + N: NotifyBell + 'static, + { let descriptor_stack = crossbeam_queue::SegQueue::new(); (0..descriptor.len() as u16).for_each(|i| descriptor_stack.push(i)); Arc::new_cyclic(|sref| Self { - notification_bell, + notification_bell: Box::new(notification_bell), available, descriptor, used, @@ -211,8 +246,7 @@ impl<'a> Queue<'a> { .set_table_index(first_descriptor as u16); self.available.set_head_idx(index as u16 + 1); - self.notification_bell - .store(self.queue_index, Ordering::SeqCst); + self.notification_bell.ring(self.queue_index); PendingRequest { queue: self.sref.upgrade().unwrap(), @@ -240,22 +274,33 @@ pub struct Available<'a> { impl<'a> Available<'a> { pub fn new(queue_size: usize) -> Result { - let (_, size, _) = queue_part_sizes(queue_size); - let size = size.next_multiple_of(syscall::PAGE_SIZE); // align to page size - + let (_, _, size) = queue_part_sizes(queue_size); let addr = unsafe { syscall::physalloc(size) }.map_err(Error::SyscallError)?; - let virt = - unsafe { common::physmap(addr, size, common::Prot::RW, common::MemoryType::default()) } - .map_err(Error::SyscallError)?; + + unsafe { Self::from_raw(addr, size, queue_size) } + } + + /// `addr` is the physical address of the ring. + pub unsafe fn from_raw(addr: usize, size: usize, queue_size: usize) -> Result { + let virt = unsafe { + common::physmap(addr, size, common::Prot::RW, common::MemoryType::default()) + }?; let ring = unsafe { &mut *(virt as *mut AvailableRing) }; - - Ok(Self { + let ring = Self { addr, size, ring, queue_size, - }) + }; + + for i in 0..queue_size { + // Setting them to `u16::MAX` helps with debugging since qemu reports them + // as illegal values. + ring.get_element_at(i).table_index.store(u16::MAX, Ordering::SeqCst); + } + + Ok(ring) } /// ## Panics @@ -308,21 +353,32 @@ pub struct Used<'a> { impl<'a> Used<'a> { pub fn new(queue_size: usize) -> Result { let (_, _, size) = queue_part_sizes(queue_size); - let size = size.next_multiple_of(syscall::PAGE_SIZE); // align to page size - let addr = unsafe { syscall::physalloc(size) }.map_err(Error::SyscallError)?; - let virt = - unsafe { common::physmap(addr, size, common::Prot::RW, common::MemoryType::default()) } - .map_err(Error::SyscallError)?; + + unsafe { Self::from_raw(addr, size, queue_size) } + } + + /// `addr` is the physical address of the ring. + pub unsafe fn from_raw(addr: usize, size: usize, queue_size: usize) -> Result { + let virt = unsafe { + common::physmap(addr, size, common::Prot::RW, common::MemoryType::default()) + }?; let ring = unsafe { &mut *(virt as *mut UsedRing) }; - - Ok(Self { + let mut ring = Self { addr, size, ring, queue_size, - }) + }; + + for i in 0..queue_size { + // Setting them to `u32::MAX` helps with debugging since qemu reports them + // as illegal values. + ring.get_mut_element_at(i).table_index.set(u32::MAX); + } + + Ok(ring) } /// ## Panics @@ -377,43 +433,307 @@ impl Drop for Used<'_> { } } +pub trait Transport: Sync + Send { + /// `size` specifies the size of the read in bytes. + /// + /// ## Panics + /// This function panics if the provided `size` is more then `size_of::()`. + fn load_config(&self, offset: u8, size: u8) -> u64; + + /// Resets the device. + fn reset(&self); + + /// Returns whether the device supports the specified feature. + fn check_device_feature(&self, feature: u32) -> bool; + + /// Acknowledges the specified feature. + /// + /// **Note**: [`Transport::check_device_feature`] must be used to check whether + /// the device supports the feature before acknowledging it. + fn ack_driver_feature(&self, feature: u32); + + /// Finalizes the acknowledged features by setting the `FEATURES_OK` bit in the + /// device status flags. No-op on a legacy device. + fn finalize_features(&self); + + /// Runs the device. + /// + /// At this point, all of the queues must be created and the features must be + /// finalized. + /// + /// ## Panics + /// This function panics if the device is already running. + fn run_device(&self) { + self.insert_status(DeviceStatusFlags::DRIVER_OK); + } + + /// Creates a new queue. + /// + /// ## Panics + /// This function panics if the device is running. + fn setup_queue(&self, vector: u16, irq_handle: &File) -> Result, Error>; + + // TODO(andypython): Should this function be unsafe? + fn reinit_queue(&self, queue: Arc); + fn insert_status(&self, status: DeviceStatusFlags); +} + +pub enum LegacyRegister { + DeviceFeatures = 0, // u32 + + QueueAddress = 8, // u32 + QueueSize = 12, // u16 + QueueSelect = 14, // u16 + QueueNotify = 16, // u16 + + DeviceStatus = 18, // u8 + + ConfigMsixVector = 20, // u16 + QueueMsixVector = 22, // u16 +} + +struct LegacyBell(Weak); + +impl NotifyBell for LegacyBell { + #[inline] + fn ring(&self, queue_index: u16) { + let transport = self.0.upgrade().expect("bell: transport dropped"); + transport.write::(LegacyRegister::QueueNotify, queue_index) + } +} + +pub struct LegacyTransport(u16, AtomicU16, Weak); + +impl LegacyTransport { + pub(super) fn new(port: u16) -> Arc { + Arc::new_cyclic(|sref| Self(port, AtomicU16::new(0), sref.clone())) + } + + unsafe fn read_raw(&self, offset: usize) -> V + where + V: Sized + TryFrom, + >::Error: std::fmt::Debug, + { + let port = self.0 + offset as u16; + + if size_of::() == size_of::() { + V::try_from(Pio::::new(port).read() as u64).unwrap() + } else if size_of::() == size_of::() { + V::try_from(Pio::::new(port).read() as u64).unwrap() + } else if size_of::() == size_of::() { + V::try_from(Pio::::new(port).read() as u64).unwrap() + } else if size_of::() == size_of::() { + let lower = Pio::::new(port).read() as u64; + let upper = Pio::::new(port + size_of::() as u16).read() as u64; + + V::try_from(lower | (upper << 32)).unwrap() + } else { + unreachable!() + } + } + + fn read(&self, register: LegacyRegister) -> V + where + V: Sized + TryFrom, + >::Error: std::fmt::Debug, + { + unsafe { self.read_raw(register as usize) } + } + + fn write(&self, register: LegacyRegister, value: V) + where + V: Sized + TryInto, + >::Error: std::fmt::Debug, + { + if size_of::() == size_of::() { + Pio::::new(self.0 + register as u16).write(value.try_into().unwrap() as u8); + } else if size_of::() == size_of::() { + Pio::::new(self.0 + register as u16).write(value.try_into().unwrap() as u16); + } else if size_of::() == size_of::() { + Pio::::new(self.0 + register as u16).write(value.try_into().unwrap() as u32); + } else { + unreachable!() + } + } +} + +impl Transport for LegacyTransport { + fn reset(&self) { + self.write(LegacyRegister::DeviceStatus, 0u8); + + let status = self.read::(LegacyRegister::DeviceStatus); + assert_eq!(status, 0); + } + + fn check_device_feature(&self, feature: u32) -> bool { + assert!( + feature < 32, + "virtio: cannot query feature {feature} on a legacy device" + ); + self.read::(LegacyRegister::DeviceFeatures) & (1 << feature) == (1 << feature) + } + + fn ack_driver_feature(&self, feature: u32) { + assert!( + feature < 32, + "virtio: cannot ack feature {feature} on a legacy device" + ); + + let current = self.read::(LegacyRegister::DeviceFeatures); + self.write::(LegacyRegister::DeviceFeatures, current | (1 << feature)); + } + + fn setup_queue(&self, vector: u16, irq_handle: &File) -> Result, Error> { + let queue_index = self.1.fetch_add(1, Ordering::SeqCst); + self.write(LegacyRegister::QueueSelect, queue_index); + + let queue_size = self.read::(LegacyRegister::QueueSize) as usize; + let (desc_size, avail_size, used_size) = queue_part_sizes(queue_size); + + let size_bytes = desc_size + avail_size + used_size; + let addr = unsafe { syscall::physalloc(size_bytes).map_err(Error::SyscallError)? }; + + let descriptor = unsafe { + let physbox = PhysBox::from_raw_parts(addr, desc_size); + let table = Dma::<[Descriptor]>::from_physbox_uninit_unsized(physbox, queue_size)?; + + table.assume_init() + }; + + let avail_addr = addr + desc_size; + let avail = unsafe { Available::from_raw(avail_addr, avail_size, queue_size)? }; + + let used_addr = avail_addr + avail_size; + let used = unsafe { Used::from_raw(used_addr, used_size, queue_size)? }; + + self.write::(LegacyRegister::QueueMsixVector, vector); + self.write::(LegacyRegister::QueueAddress, (addr as u32) >> 12); + + log::info!("virtio-core: enabled queue #{queue_index} (size={queue_size})"); + + let queue = Queue::new( + descriptor, + avail, + used, + LegacyBell(self.2.clone()), + queue_index, + vector, + ); + + spawn_irq_thread(irq_handle, &queue)?; + Ok(queue) + } + + fn load_config(&self, offset: u8, size: u8) -> u64 { + // We always enable MSI-X. So, the device configuration space offset will + // always be 0x18. + // + // Checkout 4.1.4.8 Legacy Interfaces: A Note on PCI Device Layout + const DEVICE_SPACE_OFFSET: usize = 0x18; + + let size = size as usize; + let offset = DEVICE_SPACE_OFFSET + offset as usize; + + unsafe { + if size == size_of::() { + self.read_raw::(offset) as u64 + } else if size == size_of::() { + self.read_raw::(offset) as u64 + } else if size == size_of::() { + self.read_raw::(offset) as u64 + } else if size == size_of::() { + self.read_raw::(offset) as u64 + } else { + unreachable!() + } + } + } + + fn insert_status(&self, status: DeviceStatusFlags) { + let old = self.read::(LegacyRegister::DeviceStatus); + self.write(LegacyRegister::DeviceStatus, old | status.bits()); + } + + fn reinit_queue(&self, _queue: Arc) { + todo!() + } + + // Legacy devices do not have the `FEATURES_OK` bit. + fn finalize_features(&self) {} +} + +struct StandardBell<'a>(&'a mut AtomicU16); + +impl NotifyBell for StandardBell<'_> { + #[inline] + fn ring(&self, queue_index: u16) { + self.0.store(queue_index, Ordering::SeqCst); + } +} + pub struct StandardTransport<'a> { pub(crate) common: Mutex<&'a mut CommonCfg>, notify: *const u8, notify_mul: u32, + device_space: *const u8, queue_index: AtomicU16, } impl<'a> StandardTransport<'a> { - pub fn new(common: &'a mut CommonCfg, notify: *const u8, notify_mul: u32) -> Arc { + pub fn new( + common: &'a mut CommonCfg, + notify: *const u8, + notify_mul: u32, + device_space: *const u8, + ) -> Arc { Arc::new(Self { common: Mutex::new(common), notify, notify_mul, queue_index: AtomicU16::new(0), + device_space, }) } +} - pub fn reset(&self) { +impl Transport for StandardTransport<'_> { + fn load_config(&self, offset: u8, size: u8) -> u64 { + unsafe { + let ptr = self.device_space.add(offset as usize); + let size = size as usize; + + if size == size_of::() { + ptr.cast::().read() as u64 + } else if size == size_of::() { + ptr.cast::().read() as u64 + } else if size == size_of::() { + ptr.cast::().read() as u64 + } else if size == size_of::() { + ptr.cast::().read() as u64 + } else { + unreachable!() + } + } + } + + fn reset(&self) { let mut common = self.common.lock().unwrap(); common.device_status.set(DeviceStatusFlags::empty()); // Upon reset, the device must initialize device status to 0. assert_eq!(common.device_status.get(), DeviceStatusFlags::empty()); - - log::debug!("virtio: successfully reseted device"); } - pub fn check_device_feature(&self, feature: u32) -> bool { + fn check_device_feature(&self, feature: u32) -> bool { let mut common = self.common.lock().unwrap(); common.device_feature_select.set(feature >> 5); (common.device_feature.get() & (1 << (feature & 31))) != 0 } - pub fn ack_driver_feature(&self, feature: u32) { + fn ack_driver_feature(&self, feature: u32) { let mut common = self.common.lock().unwrap(); common.driver_feature_select.set(feature >> 5); @@ -422,7 +742,7 @@ impl<'a> StandardTransport<'a> { common.driver_feature.set(current | (1 << (feature & 31))); } - pub fn finalize_features(&self) { + fn finalize_features(&self) { // Check VirtIO version 1 compliance. assert!(self.check_device_feature(VIRTIO_F_VERSION_1)); self.ack_driver_feature(VIRTIO_F_VERSION_1); @@ -440,16 +760,7 @@ impl<'a> StandardTransport<'a> { assert!((confirm & DeviceStatusFlags::FEATURES_OK) == DeviceStatusFlags::FEATURES_OK); } - pub fn run_device(&self) { - let mut common = self.common.lock().unwrap(); - - let status = common.device_status.get(); - common - .device_status - .set(status | DeviceStatusFlags::DRIVER_OK); - } - - pub fn setup_queue(&self, vector: u16, irq_handle: &File) -> Result>, Error> { + fn setup_queue(&self, vector: u16, irq_handle: &File) -> Result, Error> { let mut common = self.common.lock().unwrap(); let queue_index = self.queue_index.fetch_add(1, Ordering::SeqCst); @@ -464,17 +775,7 @@ impl<'a> StandardTransport<'a> { }; let avail = Available::new(queue_size)?; - let mut used = Used::new(queue_size)?; - - for i in 0..queue_size { - // XXX: Fill the `table_index` of the elements with `T::MAX` to help with - // debugging since qemu reports them as illegal values. - avail - .get_element_at(i) - .table_index - .store(u16::MAX, Ordering::Relaxed); - used.get_mut_element_at(i).table_index.set(u32::MAX); - } + let used = Used::new(queue_size)?; common.queue_desc.set(descriptor.physical() as u64); common.queue_driver.set(avail.phys_addr() as u64); @@ -498,37 +799,24 @@ impl<'a> StandardTransport<'a> { descriptor, avail, used, - notification_bell, + StandardBell(notification_bell), queue_index, vector, ); - let queue_copy = queue.clone(); - let irq_fd = irq_handle.as_raw_fd(); - - std::thread::spawn(move || { - let mut event_queue = EventQueue::::new().unwrap(); - - event_queue - .add(irq_fd, move |_| -> Result, std::io::Error> { - // Wake up the tasks waiting on the queue. - for (_, task) in queue_copy.waker.lock().unwrap().iter() { - task.wake_by_ref(); - } - Ok(None) - }) - .unwrap(); - - loop { - event_queue.run().unwrap(); - } - }); - + spawn_irq_thread(irq_handle, &queue)?; Ok(queue) } + fn insert_status(&self, status: DeviceStatusFlags) { + let mut common = self.common.lock().unwrap(); + let old = common.device_status.get(); + + common.device_status.set(old | status); + } + /// Re-initializes a queue; usually done after a device reset. - pub fn reinit_queue(&self, queue: Arc) { + fn reinit_queue(&self, queue: Arc) { let mut common = self.common.lock().unwrap(); queue.reinit(); @@ -546,3 +834,6 @@ impl<'a> StandardTransport<'a> { common.queue_enable.set(1); } } + +unsafe impl Send for StandardTransport<'_> {} +unsafe impl Sync for StandardTransport<'_> {} diff --git a/virtio-core/src/utils.rs b/virtio-core/src/utils.rs index 6f71dac4ea..07214248e5 100644 --- a/virtio-core/src/utils.rs +++ b/virtio-core/src/utils.rs @@ -9,7 +9,7 @@ pub struct VolatileCell { impl VolatileCell { #[inline] - pub fn new(value: T) -> Self { + pub const fn new(value: T) -> Self { Self { value: UnsafeCell::new(value), } diff --git a/virtio-gpud/src/scheme.rs b/virtio-gpud/src/scheme.rs index 7fe379a24c..7342eeea6c 100644 --- a/virtio-gpud/src/scheme.rs +++ b/virtio-gpud/src/scheme.rs @@ -9,7 +9,7 @@ use common::dma::Dma; use syscall::{Error as SysError, SchemeMut, EAGAIN, EINVAL}; use virtio_core::spec::{Buffer, ChainBuilder, DescriptorFlags}; -use virtio_core::transport::{Error, Queue, StandardTransport}; +use virtio_core::transport::{Error, Queue, Transport}; use virtio_core::utils::VolatileCell; use crate::*; @@ -30,7 +30,7 @@ impl Into for &Damage { pub struct Display<'a> { control_queue: Arc>, cursor_queue: Arc>, - transport: Arc>, + transport: Arc, // TODO(andypython): Remove the need for the spin crate after the `once_cell` // API is stabilized. @@ -49,7 +49,7 @@ impl<'a> Display<'a> { pub fn new( control_queue: Arc>, cursor_queue: Arc>, - transport: Arc>, + transport: Arc, id: usize, ) -> Self { Self { @@ -250,7 +250,10 @@ impl<'a> Display<'a> { } enum Handle<'a> { - Vt { display: Arc>, vt: usize }, + Vt { + display: Arc>, + vt: usize, + }, Input, } @@ -266,7 +269,7 @@ impl<'a> Scheme<'a> { config: &'a mut GpuConfig, control_queue: Arc>, cursor_queue: Arc>, - transport: Arc>, + transport: Arc, ) -> Result, Error> { let displays = Self::probe( control_queue.clone(), @@ -286,7 +289,7 @@ impl<'a> Scheme<'a> { async fn probe( control_queue: Arc>, cursor_queue: Arc>, - transport: Arc>, + transport: Arc, config: &GpuConfig, ) -> Result>>, Error> { let mut display_info = Self::get_display_info(control_queue.clone()).await?; @@ -353,7 +356,13 @@ impl<'a> SchemeMut for Scheme<'a> { let display = self.displays.get(id).ok_or(SysError::new(EINVAL))?; let fd = self.next_id.fetch_add(1, Ordering::SeqCst); - self.handles.insert(fd, Handle::Vt {display: display.clone(), vt }); + self.handles.insert( + fd, + Handle::Vt { + display: display.clone(), + vt, + }, + ); Ok(fd) } @@ -440,9 +449,9 @@ impl<'a> SchemeMut for Scheme<'a> { let target_vt = vt; for handle in self.handles.values() { - if let Handle::Vt { display , vt } = handle { - if *vt != target_vt { - continue; + if let Handle::Vt { display, vt } = handle { + if *vt != target_vt { + continue; } futures::executor::block_on(display.init()).unwrap(); @@ -452,9 +461,9 @@ impl<'a> SchemeMut for Scheme<'a> { DisplayCommand::Deactivate(target_vt) => { for handle in self.handles.values() { - if let Handle::Vt { display , vt } = handle { - if *vt != target_vt { - continue; + if let Handle::Vt { display, vt } = handle { + if *vt != target_vt { + continue; } futures::executor::block_on(display.detach()).unwrap(); diff --git a/virtio-netd/src/main.rs b/virtio-netd/src/main.rs index e6fb3ee842..4e38b1ec02 100644 --- a/virtio-netd/src/main.rs +++ b/virtio-netd/src/main.rs @@ -9,7 +9,7 @@ use pcid_interface::PcidServerHandle; use syscall::{Packet, SchemeBlockMut}; use virtio_core::spec::VIRTIO_NET_F_MAC; -use virtio_core::transport::Error; +use virtio_core::transport::{Error, Transport}; use scheme::NetworkScheme;