virtio-core: move things around so aarch64 builds
This commit is contained in:
Enver Balalic
committed by
Jeremy Soller
Jeremy Soller
parent
f228483f1f
commit
ad9295715f
@@ -0,0 +1,16 @@
|
||||
use std::fs::File;
|
||||
|
||||
use pcid_interface::*;
|
||||
|
||||
use crate::{transport::Error, Device};
|
||||
|
||||
pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result<File, Error> {
|
||||
unimplemented!("virtio_core: aarch64 enable_msix")
|
||||
}
|
||||
|
||||
pub fn probe_legacy_port_transport<'a>(
|
||||
pci_header: &PciHeader,
|
||||
pcid_handle: &mut PcidServerHandle,
|
||||
) -> Result<Device<'a>, Error> {
|
||||
panic!("virtio-core: aarch64 doesn't support legacy port I/O")
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result<File, Error> {
|
||||
panic!("virtio-core: x86 doesn't support enable_msix")
|
||||
}
|
||||
|
||||
pub fn probe_legacy_port_transport<'a>(
|
||||
pci_header: &PciHeader,
|
||||
pcid_handle: &mut PcidServerHandle,
|
||||
) -> Result<Device<'a>, Error> {
|
||||
crate::x86_64::probe_legacy_port_transport(pci_header, pcid_handle)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
use crate::{
|
||||
reinit,
|
||||
transport::{Error},
|
||||
utils::VolatileCell,
|
||||
Device, legacy_transport::LegacyTransport,
|
||||
};
|
||||
|
||||
use pcid_interface::msi::{self, MsixTableEntry};
|
||||
use pcid_interface::irq_helpers::{allocate_single_interrupt_vector, read_bsp_apic_id};
|
||||
use std::{ptr::NonNull, fs::File};
|
||||
|
||||
use syscall::Io;
|
||||
|
||||
use crate::{probe::MsixInfo, MSIX_PRIMARY_VECTOR};
|
||||
|
||||
use pcid_interface::*;
|
||||
|
||||
pub fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result<File, Error> {
|
||||
|
||||
let pci_config = pcid_handle.fetch_config()?;
|
||||
|
||||
// Extended message signaled interrupts.
|
||||
let capability = match pcid_handle.feature_info(PciFeature::MsiX)? {
|
||||
PciFeatureInfo::MsiX(capability) => capability,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let table_size = capability.table_size();
|
||||
let table_base = capability.table_base_pointer(pci_config.func.bars);
|
||||
let table_min_length = table_size * 16;
|
||||
let pba_min_length = table_size.div_ceil(8);
|
||||
|
||||
let pba_base = capability.pba_base_pointer(pci_config.func.bars);
|
||||
|
||||
let bir = capability.table_bir() as usize;
|
||||
let bar = pci_config.func.bars[bir];
|
||||
let bar_size = pci_config.func.bar_sizes[bir] as u64;
|
||||
|
||||
let bar_ptr = match bar {
|
||||
PciBar::Memory32(ptr) => ptr.into(),
|
||||
PciBar::Memory64(ptr) => ptr,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let address = unsafe {
|
||||
common::physmap(
|
||||
bar_ptr as usize,
|
||||
bar_size as usize,
|
||||
common::Prot::RW,
|
||||
common::MemoryType::Uncacheable,
|
||||
)? as usize
|
||||
};
|
||||
|
||||
// Ensure that the table and PBA are be within the BAR.
|
||||
{
|
||||
let bar_range = bar_ptr..bar_ptr + bar_size;
|
||||
assert!(bar_range.contains(&(table_base as u64 + table_min_length as u64)));
|
||||
assert!(bar_range.contains(&(pba_base as u64 + pba_min_length as u64)));
|
||||
}
|
||||
|
||||
let virt_table_base = ((table_base - bar_ptr as usize) + address) as *mut MsixTableEntry;
|
||||
|
||||
let mut info = MsixInfo {
|
||||
virt_table_base: NonNull::new(virt_table_base).unwrap(),
|
||||
capability,
|
||||
};
|
||||
|
||||
// Allocate the primary MSI vector.
|
||||
let interrupt_handle = {
|
||||
let table_entry_pointer = info.table_entry_pointer(MSIX_PRIMARY_VECTOR as usize);
|
||||
|
||||
let destination_id = read_bsp_apic_id().expect("virtio_core: `read_bsp_apic_id()` failed");
|
||||
let lapic_id = u8::try_from(destination_id).unwrap();
|
||||
|
||||
let rh = false;
|
||||
let dm = false;
|
||||
let addr = msi::x86_64::message_address(lapic_id, rh, dm);
|
||||
|
||||
let (vector, interrupt_handle) = allocate_single_interrupt_vector(destination_id)
|
||||
.unwrap()
|
||||
.expect("virtio_core: interrupt vector exhaustion");
|
||||
|
||||
let msg_data = msi::x86_64::message_data_edge_triggered(msi::x86_64::DeliveryMode::Fixed, vector);
|
||||
|
||||
table_entry_pointer.addr_lo.write(addr);
|
||||
table_entry_pointer.addr_hi.write(0);
|
||||
table_entry_pointer.msg_data.write(msg_data);
|
||||
table_entry_pointer
|
||||
.vec_ctl
|
||||
.writef(MsixTableEntry::VEC_CTL_MASK_BIT, false);
|
||||
|
||||
interrupt_handle
|
||||
};
|
||||
|
||||
pcid_handle.enable_feature(PciFeature::MsiX)?;
|
||||
|
||||
log::info!("virtio: using MSI-X (interrupt_handle={interrupt_handle:?})");
|
||||
Ok(interrupt_handle)
|
||||
}
|
||||
|
||||
pub fn probe_legacy_port_transport<'a>(
|
||||
pci_header: &PciHeader,
|
||||
pcid_handle: &mut PcidServerHandle,
|
||||
) -> Result<Device<'a>, Error> {
|
||||
if let PciBar::Port(port) = pci_header.get_bar(0) {
|
||||
unsafe { syscall::iopl(3).expect("virtio: failed to set I/O privilege level") };
|
||||
log::warn!("virtio: using legacy transport");
|
||||
|
||||
static SHIM: VolatileCell<u32> = VolatileCell::new(0);
|
||||
|
||||
let transport = LegacyTransport::new(port);
|
||||
|
||||
// Setup interrupts.
|
||||
let all_pci_features = pcid_handle.fetch_all_features()?;
|
||||
let has_msix = all_pci_features
|
||||
.iter()
|
||||
.any(|(feature, _)| feature.is_msix());
|
||||
|
||||
// According to the virtio specification, the device REQUIRED to support MSI-X.
|
||||
assert!(has_msix, "virtio: device does not support MSI-X");
|
||||
let irq_handle = enable_msix(pcid_handle)?;
|
||||
|
||||
let device = Device {
|
||||
transport,
|
||||
irq_handle,
|
||||
isr: &SHIM,
|
||||
device_space: core::ptr::null_mut(),
|
||||
};
|
||||
|
||||
device.transport.reset();
|
||||
reinit(&device)?;
|
||||
|
||||
Ok(device)
|
||||
} else {
|
||||
unreachable!("virtio: legacy transport with non-port IO?")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
use std::{sync::{Weak, atomic::{AtomicU16, Ordering}, Arc}, mem::size_of, fs::File};
|
||||
|
||||
use common::dma::{PhysBox, Dma};
|
||||
use syscall::{Pio, Io};
|
||||
|
||||
use crate::{transport::{NotifyBell, Transport, Queue, Error, Available, Used, queue_part_sizes, spawn_irq_thread}, spec::{Descriptor, DeviceStatusFlags}};
|
||||
|
||||
|
||||
pub enum LegacyRegister {
|
||||
DeviceFeatures = 0, // u32
|
||||
|
||||
QueueAddress = 8, // u32
|
||||
QueueSize = 12, // u16
|
||||
QueueSelect = 14, // u16
|
||||
QueueNotify = 16, // u16
|
||||
|
||||
DeviceStatus = 18, // u8
|
||||
|
||||
ConfigMsixVector = 20, // u16
|
||||
QueueMsixVector = 22, // u16
|
||||
}
|
||||
|
||||
struct LegacyBell(Weak<LegacyTransport>);
|
||||
|
||||
impl NotifyBell for LegacyBell {
|
||||
#[inline]
|
||||
fn ring(&self, queue_index: u16) {
|
||||
let transport = self.0.upgrade().expect("bell: transport dropped");
|
||||
transport.write::<u16>(LegacyRegister::QueueNotify, queue_index)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LegacyTransport(u16, AtomicU16, Weak<Self>);
|
||||
|
||||
impl LegacyTransport {
|
||||
pub(super) fn new(port: u16) -> Arc<Self> {
|
||||
Arc::new_cyclic(|sref| Self(port, AtomicU16::new(0), sref.clone()))
|
||||
}
|
||||
|
||||
unsafe fn read_raw<V>(&self, offset: usize) -> V
|
||||
where
|
||||
V: Sized + TryFrom<u64>,
|
||||
<V as TryFrom<u64>>::Error: std::fmt::Debug,
|
||||
{
|
||||
let port = self.0 + offset as u16;
|
||||
|
||||
if size_of::<V>() == size_of::<u8>() {
|
||||
V::try_from(Pio::<u8>::new(port).read() as u64).unwrap()
|
||||
} else if size_of::<V>() == size_of::<u16>() {
|
||||
V::try_from(Pio::<u16>::new(port).read() as u64).unwrap()
|
||||
} else if size_of::<V>() == size_of::<u32>() {
|
||||
V::try_from(Pio::<u32>::new(port).read() as u64).unwrap()
|
||||
} else if size_of::<V>() == size_of::<u64>() {
|
||||
let lower = Pio::<u32>::new(port).read() as u64;
|
||||
let upper = Pio::<u32>::new(port + size_of::<u32>() as u16).read() as u64;
|
||||
|
||||
V::try_from(lower | (upper << 32)).unwrap()
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
fn read<V>(&self, register: LegacyRegister) -> V
|
||||
where
|
||||
V: Sized + TryFrom<u64>,
|
||||
<V as TryFrom<u64>>::Error: std::fmt::Debug,
|
||||
{
|
||||
unsafe { self.read_raw(register as usize) }
|
||||
}
|
||||
|
||||
fn write<V>(&self, register: LegacyRegister, value: V)
|
||||
where
|
||||
V: Sized + TryInto<usize>,
|
||||
<V as TryInto<usize>>::Error: std::fmt::Debug,
|
||||
{
|
||||
if size_of::<V>() == size_of::<u8>() {
|
||||
Pio::<u8>::new(self.0 + register as u16).write(value.try_into().unwrap() as u8);
|
||||
} else if size_of::<V>() == size_of::<u16>() {
|
||||
Pio::<u16>::new(self.0 + register as u16).write(value.try_into().unwrap() as u16);
|
||||
} else if size_of::<V>() == size_of::<u32>() {
|
||||
Pio::<u32>::new(self.0 + register as u16).write(value.try_into().unwrap() as u32);
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Transport for LegacyTransport {
|
||||
fn reset(&self) {
|
||||
self.write(LegacyRegister::DeviceStatus, 0u8);
|
||||
|
||||
let status = self.read::<u8>(LegacyRegister::DeviceStatus);
|
||||
assert_eq!(status, 0);
|
||||
}
|
||||
|
||||
fn check_device_feature(&self, feature: u32) -> bool {
|
||||
assert!(
|
||||
feature < 32,
|
||||
"virtio: cannot query feature {feature} on a legacy device"
|
||||
);
|
||||
self.read::<u32>(LegacyRegister::DeviceFeatures) & (1 << feature) == (1 << feature)
|
||||
}
|
||||
|
||||
fn ack_driver_feature(&self, feature: u32) {
|
||||
assert!(
|
||||
feature < 32,
|
||||
"virtio: cannot ack feature {feature} on a legacy device"
|
||||
);
|
||||
|
||||
let current = self.read::<u32>(LegacyRegister::DeviceFeatures);
|
||||
self.write::<u32>(LegacyRegister::DeviceFeatures, current | (1 << feature));
|
||||
}
|
||||
|
||||
fn setup_queue(&self, vector: u16, irq_handle: &File) -> Result<Arc<Queue>, Error> {
|
||||
let queue_index = self.1.fetch_add(1, Ordering::SeqCst);
|
||||
self.write(LegacyRegister::QueueSelect, queue_index);
|
||||
|
||||
let queue_size = self.read::<u16>(LegacyRegister::QueueSize) as usize;
|
||||
let (desc_size, avail_size, used_size) = queue_part_sizes(queue_size);
|
||||
|
||||
let size_bytes = desc_size + avail_size + used_size;
|
||||
let addr = unsafe { syscall::physalloc(size_bytes).map_err(Error::SyscallError)? };
|
||||
|
||||
let descriptor = unsafe {
|
||||
let physbox = PhysBox::from_raw_parts(addr, desc_size);
|
||||
let table = Dma::<[Descriptor]>::from_physbox_uninit_unsized(physbox, queue_size)?;
|
||||
|
||||
table.assume_init()
|
||||
};
|
||||
|
||||
let avail_addr = addr + desc_size;
|
||||
let avail = unsafe { Available::from_raw(avail_addr, avail_size, queue_size)? };
|
||||
|
||||
let used_addr = avail_addr + avail_size;
|
||||
let used = unsafe { Used::from_raw(used_addr, used_size, queue_size)? };
|
||||
|
||||
self.write::<u16>(LegacyRegister::QueueMsixVector, vector);
|
||||
self.write::<u32>(LegacyRegister::QueueAddress, (addr as u32) >> 12);
|
||||
|
||||
log::info!("virtio-core: enabled queue #{queue_index} (size={queue_size})");
|
||||
|
||||
let queue = Queue::new(
|
||||
descriptor,
|
||||
avail,
|
||||
used,
|
||||
LegacyBell(self.2.clone()),
|
||||
queue_index,
|
||||
vector,
|
||||
);
|
||||
|
||||
spawn_irq_thread(irq_handle, &queue)?;
|
||||
Ok(queue)
|
||||
}
|
||||
|
||||
fn load_config(&self, offset: u8, size: u8) -> u64 {
|
||||
// We always enable MSI-X. So, the device configuration space offset will
|
||||
// always be 0x18.
|
||||
//
|
||||
// Checkout 4.1.4.8 Legacy Interfaces: A Note on PCI Device Layout
|
||||
const DEVICE_SPACE_OFFSET: usize = 0x18;
|
||||
|
||||
let size = size as usize;
|
||||
let offset = DEVICE_SPACE_OFFSET + offset as usize;
|
||||
|
||||
unsafe {
|
||||
if size == size_of::<u8>() {
|
||||
self.read_raw::<u8>(offset) as u64
|
||||
} else if size == size_of::<u16>() {
|
||||
self.read_raw::<u16>(offset) as u64
|
||||
} else if size == size_of::<u32>() {
|
||||
self.read_raw::<u32>(offset) as u64
|
||||
} else if size == size_of::<u64>() {
|
||||
self.read_raw::<u64>(offset) as u64
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_status(&self, status: DeviceStatusFlags) {
|
||||
let old = self.read::<u8>(LegacyRegister::DeviceStatus);
|
||||
self.write(LegacyRegister::DeviceStatus, old | status.bits());
|
||||
}
|
||||
|
||||
fn reinit_queue(&self, _queue: Arc<Queue>) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
// Legacy devices do not have the `FEATURES_OK` bit.
|
||||
fn finalize_features(&self) {}
|
||||
}
|
||||
@@ -6,4 +6,20 @@ pub mod utils;
|
||||
|
||||
mod probe;
|
||||
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
#[path="arch/aarch64.rs"]
|
||||
mod arch;
|
||||
|
||||
#[cfg(target_arch = "x86")]
|
||||
#[path="arch/x86.rs"]
|
||||
mod arch;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
#[path="arch/x86_64.rs"]
|
||||
mod arch;
|
||||
|
||||
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
|
||||
mod legacy_transport;
|
||||
|
||||
|
||||
pub use probe::{probe_device, reinit, Device, MSIX_PRIMARY_VECTOR};
|
||||
|
||||
+7
-124
@@ -1,15 +1,12 @@
|
||||
use std::fs::File;
|
||||
|
||||
use std::ptr::NonNull;
|
||||
use std::fs::File;
|
||||
use std::sync::Arc;
|
||||
|
||||
use pcid_interface::irq_helpers::{allocate_single_interrupt_vector, read_bsp_apic_id};
|
||||
use pcid_interface::msi::{self, MsixCapability, MsixTableEntry};
|
||||
use pcid_interface::*;
|
||||
|
||||
use syscall::Io;
|
||||
use pcid_interface::msi::{self, MsixTableEntry, MsixCapability};
|
||||
|
||||
use crate::spec::*;
|
||||
use crate::transport::{Error, LegacyTransport, StandardTransport, Transport};
|
||||
use crate::transport::{Error, StandardTransport, Transport};
|
||||
use crate::utils::{align_down, VolatileCell};
|
||||
|
||||
pub struct Device<'a> {
|
||||
@@ -24,7 +21,7 @@ pub struct Device<'a> {
|
||||
unsafe impl Send for Device<'_> {}
|
||||
unsafe impl Sync for Device<'_> {}
|
||||
|
||||
struct MsixInfo {
|
||||
pub struct MsixInfo {
|
||||
pub virt_table_base: NonNull<MsixTableEntry>,
|
||||
pub capability: MsixCapability,
|
||||
}
|
||||
@@ -44,89 +41,6 @@ static_assertions::const_assert_eq!(std::mem::size_of::<MsixTableEntry>(), 16);
|
||||
|
||||
pub const MSIX_PRIMARY_VECTOR: u16 = 0;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result<File, Error> {
|
||||
let pci_config = pcid_handle.fetch_config()?;
|
||||
|
||||
// Extended message signaled interrupts.
|
||||
let capability = match pcid_handle.feature_info(PciFeature::MsiX)? {
|
||||
PciFeatureInfo::MsiX(capability) => capability,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let table_size = capability.table_size();
|
||||
let table_base = capability.table_base_pointer(pci_config.func.bars);
|
||||
let table_min_length = table_size * 16;
|
||||
let pba_min_length = table_size.div_ceil(8);
|
||||
|
||||
let pba_base = capability.pba_base_pointer(pci_config.func.bars);
|
||||
|
||||
let bir = capability.table_bir() as usize;
|
||||
let bar = pci_config.func.bars[bir];
|
||||
let bar_size = pci_config.func.bar_sizes[bir] as u64;
|
||||
|
||||
let bar_ptr = match bar {
|
||||
PciBar::Memory32(ptr) => ptr.into(),
|
||||
PciBar::Memory64(ptr) => ptr,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let address = unsafe {
|
||||
common::physmap(
|
||||
bar_ptr as usize,
|
||||
bar_size as usize,
|
||||
common::Prot::RW,
|
||||
common::MemoryType::Uncacheable,
|
||||
)? as usize
|
||||
};
|
||||
|
||||
// Ensure that the table and PBA are be within the BAR.
|
||||
{
|
||||
let bar_range = bar_ptr..bar_ptr + bar_size;
|
||||
assert!(bar_range.contains(&(table_base as u64 + table_min_length as u64)));
|
||||
assert!(bar_range.contains(&(pba_base as u64 + pba_min_length as u64)));
|
||||
}
|
||||
|
||||
let virt_table_base = ((table_base - bar_ptr as usize) + address) as *mut MsixTableEntry;
|
||||
|
||||
let mut info = MsixInfo {
|
||||
virt_table_base: NonNull::new(virt_table_base).unwrap(),
|
||||
capability,
|
||||
};
|
||||
|
||||
// Allocate the primary MSI vector.
|
||||
let interrupt_handle = {
|
||||
let table_entry_pointer = info.table_entry_pointer(MSIX_PRIMARY_VECTOR as usize);
|
||||
|
||||
let destination_id = read_bsp_apic_id().expect("virtio_core: `read_bsp_apic_id()` failed");
|
||||
let lapic_id = u8::try_from(destination_id).unwrap();
|
||||
|
||||
let rh = false;
|
||||
let dm = false;
|
||||
let addr = msi::x86_64::message_address(lapic_id, rh, dm);
|
||||
|
||||
let (vector, interrupt_handle) = allocate_single_interrupt_vector(destination_id)
|
||||
.unwrap()
|
||||
.expect("virtio_core: interrupt vector exhaustion");
|
||||
|
||||
let msg_data = msi::x86_64::message_data_edge_triggered(msi::x86_64::DeliveryMode::Fixed, vector);
|
||||
|
||||
table_entry_pointer.addr_lo.write(addr);
|
||||
table_entry_pointer.addr_hi.write(0);
|
||||
table_entry_pointer.msg_data.write(msg_data);
|
||||
table_entry_pointer
|
||||
.vec_ctl
|
||||
.writef(MsixTableEntry::VEC_CTL_MASK_BIT, false);
|
||||
|
||||
interrupt_handle
|
||||
};
|
||||
|
||||
pcid_handle.enable_feature(PciFeature::MsiX)?;
|
||||
|
||||
log::info!("virtio: using MSI-X (interrupt_handle={interrupt_handle:?})");
|
||||
Ok(interrupt_handle)
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
fn enable_msix(pcid_handle: &mut PcidServerHandle) -> Result<File, Error> {
|
||||
panic!("Msi-X only supported on x86_64");
|
||||
@@ -268,7 +182,7 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result<Device<'a>
|
||||
|
||||
// According to the virtio specification, the device REQUIRED to support MSI-X.
|
||||
assert!(has_msix, "virtio: device does not support MSI-X");
|
||||
let irq_handle = enable_msix(pcid_handle)?;
|
||||
let irq_handle = crate::arch::enable_msix(pcid_handle)?;
|
||||
|
||||
log::info!("virtio: using standard PCI transport");
|
||||
|
||||
@@ -284,38 +198,7 @@ pub fn probe_device<'a>(pcid_handle: &mut PcidServerHandle) -> Result<Device<'a>
|
||||
|
||||
Ok(device)
|
||||
} else {
|
||||
if let PciBar::Port(port) = pci_header.get_bar(0) {
|
||||
unsafe { syscall::iopl(3).expect("virtio: failed to set I/O privilege level") };
|
||||
log::warn!("virtio: using legacy transport");
|
||||
|
||||
static SHIM: VolatileCell<u32> = VolatileCell::new(0);
|
||||
|
||||
let transport = LegacyTransport::new(port);
|
||||
|
||||
// Setup interrupts.
|
||||
let all_pci_features = pcid_handle.fetch_all_features()?;
|
||||
let has_msix = all_pci_features
|
||||
.iter()
|
||||
.any(|(feature, _)| feature.is_msix());
|
||||
|
||||
// According to the virtio specification, the device REQUIRED to support MSI-X.
|
||||
assert!(has_msix, "virtio: device does not support MSI-X");
|
||||
let irq_handle = enable_msix(pcid_handle)?;
|
||||
|
||||
let device = Device {
|
||||
transport,
|
||||
irq_handle,
|
||||
isr: &SHIM,
|
||||
device_space: core::ptr::null_mut(),
|
||||
};
|
||||
|
||||
device.transport.reset();
|
||||
reinit(&device)?;
|
||||
|
||||
Ok(device)
|
||||
} else {
|
||||
unreachable!("virtio: legacy transport with non-port IO?")
|
||||
}
|
||||
crate::arch::probe_legacy_port_transport(&pci_header, pcid_handle)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
use crate::spec::*;
|
||||
use crate::utils::align;
|
||||
|
||||
use common::dma::{Dma, PhysBox};
|
||||
use common::dma::Dma;
|
||||
use event::EventQueue;
|
||||
use syscall::{Io, Pio};
|
||||
|
||||
use core::mem::size_of;
|
||||
use core::sync::atomic::{AtomicU16, Ordering};
|
||||
@@ -44,7 +43,7 @@ impl From<syscall::Error> for Error {
|
||||
///
|
||||
/// ## Panics
|
||||
/// If `queue_size` is not a power of two or is zero.
|
||||
const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) {
|
||||
pub const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) {
|
||||
assert!(queue_size.is_power_of_two() && queue_size != 0);
|
||||
|
||||
const DESCRIPTOR_ALIGN: usize = 16;
|
||||
@@ -69,7 +68,7 @@ const fn queue_part_sizes(queue_size: usize) -> (usize, usize, usize) {
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_irq_thread(irq_handle: &File, queue: &Arc<Queue<'static>>) -> Result<(), Error> {
|
||||
pub fn spawn_irq_thread(irq_handle: &File, queue: &Arc<Queue<'static>>) -> Result<(), Error> {
|
||||
let irq_fd = irq_handle.as_raw_fd();
|
||||
let queue_copy = queue.clone();
|
||||
|
||||
@@ -297,7 +296,9 @@ impl<'a> Available<'a> {
|
||||
for i in 0..queue_size {
|
||||
// Setting them to `u16::MAX` helps with debugging since qemu reports them
|
||||
// as illegal values.
|
||||
ring.get_element_at(i).table_index.store(u16::MAX, Ordering::SeqCst);
|
||||
ring.get_element_at(i)
|
||||
.table_index
|
||||
.store(u16::MAX, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
Ok(ring)
|
||||
@@ -478,190 +479,6 @@ pub trait Transport: Sync + Send {
|
||||
fn insert_status(&self, status: DeviceStatusFlags);
|
||||
}
|
||||
|
||||
pub enum LegacyRegister {
|
||||
DeviceFeatures = 0, // u32
|
||||
|
||||
QueueAddress = 8, // u32
|
||||
QueueSize = 12, // u16
|
||||
QueueSelect = 14, // u16
|
||||
QueueNotify = 16, // u16
|
||||
|
||||
DeviceStatus = 18, // u8
|
||||
|
||||
ConfigMsixVector = 20, // u16
|
||||
QueueMsixVector = 22, // u16
|
||||
}
|
||||
|
||||
struct LegacyBell(Weak<LegacyTransport>);
|
||||
|
||||
impl NotifyBell for LegacyBell {
|
||||
#[inline]
|
||||
fn ring(&self, queue_index: u16) {
|
||||
let transport = self.0.upgrade().expect("bell: transport dropped");
|
||||
transport.write::<u16>(LegacyRegister::QueueNotify, queue_index)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LegacyTransport(u16, AtomicU16, Weak<Self>);
|
||||
|
||||
impl LegacyTransport {
|
||||
pub(super) fn new(port: u16) -> Arc<Self> {
|
||||
Arc::new_cyclic(|sref| Self(port, AtomicU16::new(0), sref.clone()))
|
||||
}
|
||||
|
||||
unsafe fn read_raw<V>(&self, offset: usize) -> V
|
||||
where
|
||||
V: Sized + TryFrom<u64>,
|
||||
<V as TryFrom<u64>>::Error: std::fmt::Debug,
|
||||
{
|
||||
let port = self.0 + offset as u16;
|
||||
|
||||
if size_of::<V>() == size_of::<u8>() {
|
||||
V::try_from(Pio::<u8>::new(port).read() as u64).unwrap()
|
||||
} else if size_of::<V>() == size_of::<u16>() {
|
||||
V::try_from(Pio::<u16>::new(port).read() as u64).unwrap()
|
||||
} else if size_of::<V>() == size_of::<u32>() {
|
||||
V::try_from(Pio::<u32>::new(port).read() as u64).unwrap()
|
||||
} else if size_of::<V>() == size_of::<u64>() {
|
||||
let lower = Pio::<u32>::new(port).read() as u64;
|
||||
let upper = Pio::<u32>::new(port + size_of::<u32>() as u16).read() as u64;
|
||||
|
||||
V::try_from(lower | (upper << 32)).unwrap()
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
fn read<V>(&self, register: LegacyRegister) -> V
|
||||
where
|
||||
V: Sized + TryFrom<u64>,
|
||||
<V as TryFrom<u64>>::Error: std::fmt::Debug,
|
||||
{
|
||||
unsafe { self.read_raw(register as usize) }
|
||||
}
|
||||
|
||||
fn write<V>(&self, register: LegacyRegister, value: V)
|
||||
where
|
||||
V: Sized + TryInto<usize>,
|
||||
<V as TryInto<usize>>::Error: std::fmt::Debug,
|
||||
{
|
||||
if size_of::<V>() == size_of::<u8>() {
|
||||
Pio::<u8>::new(self.0 + register as u16).write(value.try_into().unwrap() as u8);
|
||||
} else if size_of::<V>() == size_of::<u16>() {
|
||||
Pio::<u16>::new(self.0 + register as u16).write(value.try_into().unwrap() as u16);
|
||||
} else if size_of::<V>() == size_of::<u32>() {
|
||||
Pio::<u32>::new(self.0 + register as u16).write(value.try_into().unwrap() as u32);
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Transport for LegacyTransport {
|
||||
fn reset(&self) {
|
||||
self.write(LegacyRegister::DeviceStatus, 0u8);
|
||||
|
||||
let status = self.read::<u8>(LegacyRegister::DeviceStatus);
|
||||
assert_eq!(status, 0);
|
||||
}
|
||||
|
||||
fn check_device_feature(&self, feature: u32) -> bool {
|
||||
assert!(
|
||||
feature < 32,
|
||||
"virtio: cannot query feature {feature} on a legacy device"
|
||||
);
|
||||
self.read::<u32>(LegacyRegister::DeviceFeatures) & (1 << feature) == (1 << feature)
|
||||
}
|
||||
|
||||
fn ack_driver_feature(&self, feature: u32) {
|
||||
assert!(
|
||||
feature < 32,
|
||||
"virtio: cannot ack feature {feature} on a legacy device"
|
||||
);
|
||||
|
||||
let current = self.read::<u32>(LegacyRegister::DeviceFeatures);
|
||||
self.write::<u32>(LegacyRegister::DeviceFeatures, current | (1 << feature));
|
||||
}
|
||||
|
||||
fn setup_queue(&self, vector: u16, irq_handle: &File) -> Result<Arc<Queue>, Error> {
|
||||
let queue_index = self.1.fetch_add(1, Ordering::SeqCst);
|
||||
self.write(LegacyRegister::QueueSelect, queue_index);
|
||||
|
||||
let queue_size = self.read::<u16>(LegacyRegister::QueueSize) as usize;
|
||||
let (desc_size, avail_size, used_size) = queue_part_sizes(queue_size);
|
||||
|
||||
let size_bytes = desc_size + avail_size + used_size;
|
||||
let addr = unsafe { syscall::physalloc(size_bytes).map_err(Error::SyscallError)? };
|
||||
|
||||
let descriptor = unsafe {
|
||||
let physbox = PhysBox::from_raw_parts(addr, desc_size);
|
||||
let table = Dma::<[Descriptor]>::from_physbox_uninit_unsized(physbox, queue_size)?;
|
||||
|
||||
table.assume_init()
|
||||
};
|
||||
|
||||
let avail_addr = addr + desc_size;
|
||||
let avail = unsafe { Available::from_raw(avail_addr, avail_size, queue_size)? };
|
||||
|
||||
let used_addr = avail_addr + avail_size;
|
||||
let used = unsafe { Used::from_raw(used_addr, used_size, queue_size)? };
|
||||
|
||||
self.write::<u16>(LegacyRegister::QueueMsixVector, vector);
|
||||
self.write::<u32>(LegacyRegister::QueueAddress, (addr as u32) >> 12);
|
||||
|
||||
log::info!("virtio-core: enabled queue #{queue_index} (size={queue_size})");
|
||||
|
||||
let queue = Queue::new(
|
||||
descriptor,
|
||||
avail,
|
||||
used,
|
||||
LegacyBell(self.2.clone()),
|
||||
queue_index,
|
||||
vector,
|
||||
);
|
||||
|
||||
spawn_irq_thread(irq_handle, &queue)?;
|
||||
Ok(queue)
|
||||
}
|
||||
|
||||
fn load_config(&self, offset: u8, size: u8) -> u64 {
|
||||
// We always enable MSI-X. So, the device configuration space offset will
|
||||
// always be 0x18.
|
||||
//
|
||||
// Checkout 4.1.4.8 Legacy Interfaces: A Note on PCI Device Layout
|
||||
const DEVICE_SPACE_OFFSET: usize = 0x18;
|
||||
|
||||
let size = size as usize;
|
||||
let offset = DEVICE_SPACE_OFFSET + offset as usize;
|
||||
|
||||
unsafe {
|
||||
if size == size_of::<u8>() {
|
||||
self.read_raw::<u8>(offset) as u64
|
||||
} else if size == size_of::<u16>() {
|
||||
self.read_raw::<u16>(offset) as u64
|
||||
} else if size == size_of::<u32>() {
|
||||
self.read_raw::<u32>(offset) as u64
|
||||
} else if size == size_of::<u64>() {
|
||||
self.read_raw::<u64>(offset) as u64
|
||||
} else {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_status(&self, status: DeviceStatusFlags) {
|
||||
let old = self.read::<u8>(LegacyRegister::DeviceStatus);
|
||||
self.write(LegacyRegister::DeviceStatus, old | status.bits());
|
||||
}
|
||||
|
||||
fn reinit_queue(&self, _queue: Arc<Queue>) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
// Legacy devices do not have the `FEATURES_OK` bit.
|
||||
fn finalize_features(&self) {}
|
||||
}
|
||||
|
||||
struct StandardBell<'a>(&'a mut AtomicU16);
|
||||
|
||||
impl NotifyBell for StandardBell<'_> {
|
||||
|
||||
Reference in New Issue
Block a user