From e6a46bb6a679d4bd332f6ec6d679c34842e93d13 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 25 Apr 2020 23:02:16 +0200 Subject: [PATCH] Begin with NVME MSI support. --- Cargo.lock | 8 ++ nvmed/Cargo.toml | 2 + nvmed/src/main.rs | 127 ++++++++++++++++++++++-- nvmed/src/nvme/cq_reactor.rs | 153 +++++++++++++++++++++++++++++ nvmed/src/{nvme.rs => nvme/mod.rs} | 124 +++++++++++++++-------- nvmed/src/scheme.rs | 21 ++-- 6 files changed, 379 insertions(+), 56 deletions(-) create mode 100644 nvmed/src/nvme/cq_reactor.rs rename nvmed/src/{nvme.rs => nvme/mod.rs} (81%) diff --git a/Cargo.lock b/Cargo.lock index f27f4203eb..0a006c6d13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,6 +42,11 @@ dependencies = [ "nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "atty" version = "0.2.14" @@ -727,8 +732,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "nvmed" version = "0.1.0" dependencies = [ + "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "block-io-wrapper 0.1.0", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "partitionlib 0.1.0 (git+https://gitlab.redox-os.org/redox-os/partitionlib.git)", "pcid 0.1.0", @@ -1716,6 +1723,7 @@ dependencies = [ "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum arg_parser 0.1.0 (git+https://gitlab.redox-os.org/redox-os/arg-parser.git)" = "" "checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" diff --git a/nvmed/Cargo.toml b/nvmed/Cargo.toml index 92a180f585..ec8152f9e9 100644 --- a/nvmed/Cargo.toml +++ b/nvmed/Cargo.toml @@ -4,7 +4,9 @@ version = "0.1.0" edition = "2018" [dependencies] +arrayvec = "0.5" bitflags = "0.7" +crossbeam-channel = "0.4" log = "0.4" redox-log = { git = "https://gitlab.redox-os.org/redox-os/redox-log.git", tag = "v0.1.0" } redox_syscall = "0.1" diff --git a/nvmed/src/main.rs b/nvmed/src/main.rs index 7e6178840a..5bec4fca96 100644 --- a/nvmed/src/main.rs +++ b/nvmed/src/main.rs @@ -1,20 +1,127 @@ -use std::{env, usize}; +use std::{env, slice, usize}; use std::fs::File; use std::io::{ErrorKind, Read, Write}; +use std::ptr::NonNull; use std::os::unix::io::{RawFd, FromRawFd}; +use std::sync::{Arc, Mutex}; -use pcid_interface::{PcidServerHandle, PciBar}; - +use pcid_interface::{PciFeature, PciFeatureInfo, PciFunction, PcidServerHandle, PciBar}; use syscall::{EVENT_READ, PHYSMAP_NO_CACHE, PHYSMAP_WRITE, Event, Packet, Result, SchemeBlockMut}; +use syscall::io::Mmio; +use arrayvec::ArrayVec; use log::{debug, error, info, warn, trace}; -use self::nvme::Nvme; +use self::nvme::{InterruptMethod, Nvme}; use self::scheme::DiskScheme; mod nvme; mod scheme; +#[derive(Default)] +pub struct Bar { + ptr: NonNull, + physical: usize, + bar_size: usize, +} +impl Bar { + pub fn allocate(bar: usize, bar_size: usize) -> Result { + Ok(Self { + ptr: NonNull::new(syscall::physmap(bar, bar_size, PHYSMAP_NO_CACHE | PHYSMAP_WRITE)? as *mut u8).expect("Mapping a BAR resulted in a nullptr"), + physical: bar, + bar_size, + }) + } +} + +impl Drop for Bar { + fn drop(&mut self) { + let _ = syscall::physunmap(self.physical); + } +} + +#[derive(Default)] +pub struct AllocatedBars(pub [Mutex>; 6]); + +/// Get the most optimal yet functional interrupt +fn get_int_method(pcid_handle: &mut PcidServerHandle, function: &PciFunction, nvme: &mut Nvme, allocated_bars: &AllocatedBars) -> Result { + + let features = pcid_handle.fetch_all_features().unwrap(); + + let has_msi = features.iter().any(|(feature, _)| feature.is_msi()); + let has_msix = features.iter().any(|(feature, _)| feature.is_msix()); + + // TODO: Allocate more than one vector when possible and useful. + if has_msix { + // Extended message signaled interrupts. + use pcid_interface::msi::MsixTableEntry; + use self::nvme::MsixCfg; + + let mut capability_struct = match pcid_handle.feature_info(PciFeature::MsiX).unwrap() { + PciFeatureInfo::MsiX(msix) => msix, + _ => unreachable!(), + }; + fn bar_base(allocated_bars: &AllocatedBars, function: &PciFunction, bir: u8) -> Result> { + let bir = usize::from(bir); + let bar_guard = allocated_bars.0[bir].lock().unwrap(); + match &mut *bar_guard { + &mut Some(ref bar) => Ok(bar.ptr), + bar_to_set @ &mut None => { + let bar = match function.bars[bir] { + PciBar::Memory(addr) => addr, + other => panic!("Expected memory BAR, found {:?}", other), + }; + let bar_size = function.bar_sizes[bir]; + + let bar = Bar::allocate(bar as usize, bar_size as usize)?; + *bar_to_set = Some(bar); + Ok(bar_to_set.as_ref().unwrap().ptr) + } + } + } + let table_bar_base: *mut u8 = bar_base(allocated_bars, function, capability_struct.table_bir())?.as_ptr(); + let pba_bar_base: *mut u8 = bar_base(allocated_bars, function, capability_struct.pba_bir())?.as_ptr(); + let table_base = unsafe { table_bar_base.offset(capability_struct.table_offset() as isize) }; + let pba_base = unsafe { pba_bar_base.offset(capability_struct.pba_offset() as isize) }; + + let vector_count = capability_struct.table_size(); + let table_entries: &'static mut [MsixTableEntry] = unsafe { slice::from_raw_parts_mut(table_base as *mut MsixTableEntry, vector_count as usize) }; + let pba_entries: &'static mut [Mmio] = unsafe { slice::from_raw_parts_mut(table_base as *mut Mmio, (vector_count as usize + 63) / 64) }; + + // Mask all interrupts in case some earlier driver/os already unmasked them (according to + // the PCI Local Bus spec 3.0, they are masked after system reset). + for table_entry in table_entries { + table_entry.mask(); + } + + pcid_handle.enable_feature(PciFeature::MsiX).unwrap(); + capability_struct.set_msix_enabled(true); // only affects our local mirror of the cap + + // We don't allocate any vectors yet; that's later done when we get into + // submission/completion queues. + + Ok(InterruptMethod::MsiX(MsixCfg { + cap: capability_struct, + table: table_entries, + pba: pba_entries, + })) + } else if has_msi { + // Message signaled interrupts. + let capability_struct = match pcid_handle.feature_info(PciFeature::Msi).unwrap() { + PciFeatureInfo::Msi(msi) => msi, + _ => unreachable!(), + }; + // We don't enable MSI until needed. + Ok(InterruptMethod::Msi(capability_struct)) + } else if function.legacy_interrupt_pin.is_some() { + // INTx# pin based interrupts. + Ok(InterruptMethod::Intx) + } else { + // No interrupts at all + todo!("handling of no interrupts") + } +} + fn main() { // Daemonize if unsafe { syscall::clone(0).unwrap() } != 0 { @@ -36,10 +143,13 @@ fn main() { info!("NVME PCI CONFIG: {:?}", pci_config); + let allocated_bars = AllocatedBars::default(); + let address = unsafe { syscall::physmap(bar as usize, bar_size as usize, PHYSMAP_WRITE | PHYSMAP_NO_CACHE) .expect("nvmed: failed to map address") }; + *allocated_bars.0[0].lock().unwrap() = Some(Bar { physical: bar as usize, bar_size: bar_size as usize, ptr: NonNull::new(address as *mut u8).expect("Physmapping BAR gave nullptr") }); { let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC) .expect("nvmed: failed to open event queue"); @@ -70,8 +180,12 @@ fn main() { syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace"); - let mut nvme = Nvme::new(address).expect("nvmed: failed to allocate driver data"); - let namespaces = unsafe { nvme.init() }; + let (reactor_sender, reactor_receiver) = crossbeam_channel::unbounded(); + let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender).expect("nvmed: failed to allocate driver data"); + let nvme = Arc::new(nvme); + unsafe { nvme.init() } + nvme::cq_reactor::start_cq_reactor_thread(nvme); + let namespaces = unsafe { nvme.init_with_queues() }; let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces); let mut todo = Vec::new(); 'events: loop { @@ -120,5 +234,4 @@ fn main() { //TODO: destroy NVMe stuff } - unsafe { let _ = syscall::physunmap(address); } } diff --git a/nvmed/src/nvme/cq_reactor.rs b/nvmed/src/nvme/cq_reactor.rs new file mode 100644 index 0000000000..f0ea1336de --- /dev/null +++ b/nvmed/src/nvme/cq_reactor.rs @@ -0,0 +1,153 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::future::Future; +use std::io::prelude::*; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::{io, task, thread}; +use std::os::unix::io::{FromRawFd, RawFd}; + +use syscall::Result; + +use crossbeam_channel::{Sender, Receiver}; + +use crate::nvme::{InterruptMethod, Nvme, NvmeComp}; + +/// A source of interrupts. The NVME spec splits the definition of MSI into "single msi" and "multi +/// msi". +#[derive(Debug)] +pub enum IntSources { + Intx(File), + SingleMsi(File), + MultiMsi(BTreeMap), + MsiX(BTreeMap), +} + +/// A notification request, sent by the future in order to tell the completion thread that the +/// current task wants a notification when a matching completion queue entry has been seen. +pub enum NotifReq { + RequestCompletion { + queue_id: usize, + waker: task::Waker, + // TODO: Get rid of this allocation + message: Arc>>, + }, +} + +struct PendingReq { + waker: task::Waker, + message: Arc>>, + queue_id: usize, +} +struct CqReactor { + int_sources: Option, + nvme: Arc, + pending_reqs: Vec, + receiver: Receiver, + event_queue: File, +} +impl CqReactor { + fn create_event_queue() -> Result { + use syscall::flag::*; + let fd = syscall::open("event:", O_CLOEXEC | O_RDWR)?; + let mut file = unsafe { File::from_raw_fd(fd as RawFd) }; + todo!() + } + fn new(nvme: Arc, receiver: Receiver) -> Result { + Ok(Self { + int_sources: None, // TODO + nvme, + pending_reqs: Vec::new(), + receiver, + event_queue: Self::create_event_queue()?, + }) + } + fn handle_notif_reqs(&mut self) { + for req in self.receiver.try_iter() { + match req { + NotifReq::RequestCompletion { queue_id, waker, message } => self.pending_reqs.push(PendingReq { + queue_id, + message, + waker, + }), + } + } + } + fn run(mut self) -> ! { + loop { + self.handle_notif_reqs(); + } + } +} + +pub fn start_cq_reactor_thread(nvme: Arc, receiver: Receiver) -> thread::JoinHandle<()> { + // Actually, nothing prevents us from spawning additional threads. the channel is MPMC and + // everything is properly synchronized. I'm not saying this is strictly required, but with + // multiple completion queues it might actually be worth considering. + thread::spawn(move || { + CqReactor::new(nvme, receiver) + .expect("nvmed: failed to setup CQ reactor") + .run() + }) +} + +pub struct CompletionMessage { + cq_entry: NvmeComp, +} + +enum CompletionFuture { + // not really required, but makes futures inert + Init { + sender: Sender, + queue_id: usize, + }, + Pending { + message: Arc>>, + }, + Finished, +} + +// enum not self-referential +impl Unpin for CompletionFuture {} + +impl Future for CompletionFuture { + type Output = NvmeComp; + + fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll { + let this = self.get_mut(); + + match this { + &mut Self::Init { sender, queue_id } => { + let message = Arc::new(Mutex::new(None)); + sender.send(NotifReq::RequestCompletion { + queue_id, + waker: context.waker().clone(), + message: Arc::clone(&message), + }); + *this = CompletionFuture::Pending { + message, + }; + task::Poll::Pending + } + &mut Self::Pending { message } => if let Some(value) = message.lock().unwrap().take() { + *this = Self::Finished; + task::Poll::Ready(value.cq_entry) + } else { + // woken up but the reactor hadn't sent the message. + // this is ideally unreachable + task::Poll::Pending + } + &mut Self::Finished => panic!("calling poll() on an already finished CompletionFuture"), + } + } +} + + +impl Nvme { + pub fn completion(&self, cq_id: usize) -> impl Future + '_ { + CompletionFuture::Init { + sender: self.reactor_sender.clone(), + queue_id: cq_id, + } + } +} diff --git a/nvmed/src/nvme.rs b/nvmed/src/nvme/mod.rs similarity index 81% rename from nvmed/src/nvme.rs rename to nvmed/src/nvme/mod.rs index 9b74892f65..aeef71471b 100644 --- a/nvmed/src/nvme.rs +++ b/nvmed/src/nvme/mod.rs @@ -1,8 +1,33 @@ -use std::{ptr, thread}; +use std::ptr; use std::collections::BTreeMap; +use std::sync::{Mutex, RwLock}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use crossbeam_channel::Sender; + use syscall::io::{Dma, Io, Mmio}; use syscall::error::{Error, Result, EINVAL}; +pub mod cq_reactor; +use self::cq_reactor::{self, NotifReq}; + +use pcid_interface::msi::{MsiCapability, MsixCapability, MsixTableEntry}; +use pcid_interface::PcidServerHandle; + +#[derive(Debug)] +pub enum InterruptMethod { + Intx, + Msi(MsiCapability), + MsiX(MsixCfg), +} + +#[derive(Debug)] +pub struct MsixCfg { + pub cap: MsixCapability, + pub table: &'static mut [MsixTableEntry], + pub pba: &'static mut [Mmio], +} + #[derive(Clone, Copy)] #[repr(packed)] pub struct NvmeCmd { @@ -39,7 +64,7 @@ impl NvmeCmd { Self { opcode: 5, flags: 0, - cid: cid, + cid, nsid: 0, _rsvd: 0, mptr: 0, @@ -272,43 +297,55 @@ pub struct NvmeNamespace { } pub struct Nvme { - regs: &'static mut NvmeRegs, - submission_queues: [NvmeCmdQueue; 2], - pub (crate) completion_queues: [NvmeCompQueue; 2], - buffer: Dma<[u8; 512 * 4096]>, // 2MB of buffer - buffer_prp: Dma<[u64; 512]>, // 4KB of PRP for the buffer + interrupt_method: Mutex, + pcid_interface: Mutex, + regs: Mutex<&'static mut NvmeRegs>, + submission_queues: RwLock>>, + pub (crate) completion_queues: RwLock>>, + buffer: Mutex>, // 2MB of buffer + buffer_prp: Mutex>, // 4KB of PRP for the buffer + reactor_sender: Sender, + next_cid: AtomicUsize, } impl Nvme { - pub fn new(address: usize) -> Result { + pub fn new(address: usize, interrupt_method: InterruptMethod, pcid_interface: PcidServerHandle, reactor_sender: Sender) -> Result { Ok(Nvme { - regs: unsafe { &mut *(address as *mut NvmeRegs) }, - submission_queues: [NvmeCmdQueue::new()?, NvmeCmdQueue::new()?], - completion_queues: [NvmeCompQueue::new()?, NvmeCompQueue::new()?], - buffer: Dma::zeroed()?, - buffer_prp: Dma::zeroed()?, + regs: Mutex::new(unsafe { &mut *(address as *mut NvmeRegs) }), + submission_queues: RwLock::new(vec! [Mutex::new(NvmeCmdQueue::new()?), Mutex::new(NvmeCmdQueue::new()?)]), + completion_queues: RwLock::new(vec! [Mutex::new(NvmeCompQueue::new()?), Mutex::new(NvmeCompQueue::new()?)]), + buffer: Mutex::new(Dma::zeroed()?), + buffer_prp: Mutex::new(Dma::zeroed()?), + next_cid: AtomicUsize::new(0), + interrupt_method: Mutex::new(interrupt_method), + pcid_interface: Mutex::new(pcid_interface), + reactor_sender, }) } + unsafe fn doorbell_write(&self, index: usize, value: u32) { + let mut regs_guard = self.regs.lock().unwrap(); - unsafe fn doorbell(&mut self, index: usize) -> &'static mut Mmio { - let dstrd = ((self.regs.cap.read() >> 32) & 0b1111) as usize; - let addr = (self.regs as *mut _ as usize) + let dstrd = ((regs_guard.cap.read() >> 32) & 0b1111) as usize; + let addr = (regs_guard as *mut u8 as usize) + 0x1000 + index * (4 << dstrd); - &mut *(addr as *mut Mmio) + (&mut *(addr as *mut Mmio)).write(value); } - pub unsafe fn submission_queue_tail(&mut self, qid: u16, tail: u16) { - self.doorbell(2 * (qid as usize)).write(tail as u32); + pub unsafe fn submission_queue_tail(&self, qid: u16, tail: u16) { + self.doorbell_write(2 * (qid as usize), u32::from(tail)); } - pub unsafe fn completion_queue_head(&mut self, qid: u16, head: u16) { - self.doorbell(2 * (qid as usize) + 1).write(head as u32) + pub unsafe fn completion_queue_head(&self, qid: u16, head: u16) { + self.doorbell_write(2 * (qid as usize) + 1, u32::from(head)); } - pub unsafe fn init(&mut self) -> BTreeMap { - for i in 0..self.buffer_prp.len() { - self.buffer_prp[i] = (self.buffer.physical() + i * 4096) as u64; + pub unsafe fn init(&mut self) { + let mut buffer = self.buffer.get_mut().unwrap(); + let mut buffer_prp = self.buffer_prp.get_mut().unwrap(); + + for i in 0..buffer_prp.len() { + buffer_prp[i] = (buffer.physical() + i * 4096) as u64; } // println!(" - CAPS: {:X}", self.regs.cap.read()); @@ -317,11 +354,11 @@ impl Nvme { // println!(" - CSTS: {:X}", self.regs.csts.read()); // println!(" - Disable"); - self.regs.cc.writef(1, false); + self.regs.get_mut().unwrap().cc.writef(1, false); // println!(" - Waiting for not ready"); loop { - let csts = self.regs.csts.read(); + let csts = self.regs.get_mut().unwrap().csts.read(); // println!("CSTS: {:X}", csts); if csts & 1 == 1 { unsafe { std::arch::x86_64::_mm_pause() } @@ -331,38 +368,42 @@ impl Nvme { } // println!(" - Mask all interrupts"); - self.regs.intms.write(0xFFFFFFFF); + self.regs.get_mut().unwrap().intms.write(0xFFFFFFFF); // TODO: Don't mask - for (qid, queue) in self.completion_queues.iter().enumerate() { - let data = &queue.data; + for (qid, queue) in self.completion_queues.get_mut().unwrap().iter().enumerate() { + let data = &queue.get_mut().unwrap().data; // println!(" - completion queue {}: {:X}, {}", qid, data.physical(), data.len()); } - for (qid, queue) in self.submission_queues.iter().enumerate() { - let data = &queue.data; + for (qid, queue) in self.submission_queues.get_mut().unwrap().iter().enumerate() { + let data = &queue.get_mut().unwrap().data; // println!(" - submission queue {}: {:X}, {}", qid, data.physical(), data.len()); } { - let asq = &self.submission_queues[0]; - let acq = &self.completion_queues[0]; - self.regs.aqa.write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1)); - self.regs.asq.write(asq.data.physical() as u64); - self.regs.acq.write(acq.data.physical() as u64); + let regs = self.regs.get_mut().unwrap(); + let submission_queues = self.submission_queues.get_mut().unwrap(); + let completion_queues = self.submission_queues.get_mut().unwrap(); + + let asq = &submission_queues[0].get_mut().unwrap(); + let acq = &completion_queues[0].get_mut().unwrap(); + regs.aqa.write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1)); + regs.asq.write(asq.data.physical() as u64); + regs.acq.write(acq.data.physical() as u64); // Set IOCQES, IOSQES, AMS, MPS, and CSS - let mut cc = self.regs.cc.read(); + let mut cc = regs.cc.read(); cc &= 0xFF00000F; cc |= (4 << 20) | (6 << 16); - self.regs.cc.write(cc); + regs.cc.write(cc); } // println!(" - Enable"); - self.regs.cc.writef(1, true); + self.regs.get_mut().unwrap().cc.writef(1, true); // println!(" - Waiting for ready"); loop { - let csts = self.regs.csts.read(); + let csts = self.regs.get_mut().unwrap().csts.read(); // println!("CSTS: {:X}", csts); if csts & 1 == 0 { unsafe { std::arch::x86_64::_mm_pause() } @@ -370,7 +411,8 @@ impl Nvme { break; } } - + } + pub fn init_with_queues(&self) -> BTreeMap { { //TODO: Use buffer let data: Dma<[u8; 4096]> = Dma::zeroed().unwrap(); diff --git a/nvmed/src/scheme.rs b/nvmed/src/scheme.rs index 108518a3e6..40f6f1b68c 100644 --- a/nvmed/src/scheme.rs +++ b/nvmed/src/scheme.rs @@ -4,6 +4,8 @@ use std::convert::{TryFrom, TryInto}; use std::fmt::Write; use std::io::prelude::*; use std::io; +use std::sync::Arc; + use syscall::{ Error, EACCES, EBADF, EINVAL, EISDIR, ENOENT, EOVERFLOW, Result, Io, SchemeBlockMut, Stat, MODE_DIR, MODE_FILE, O_DIRECTORY, @@ -32,13 +34,13 @@ impl AsRef for DiskWrapper { } impl DiskWrapper { - fn pt(disk: &mut NvmeNamespace, nvme: &mut Nvme) -> Option { + fn pt(disk: &mut NvmeNamespace, nvme: &Nvme) -> Option { let bs = match disk.block_size { 512 => LogicalBlockSize::Lb512, 4096 => LogicalBlockSize::Lb4096, _ => return None, }; - struct Device<'a, 'b> { disk: &'a mut NvmeNamespace, nvme: &'a mut Nvme, offset: u64, block_bytes: &'b mut [u8] } + struct Device<'a, 'b> { disk: &'a mut NvmeNamespace, nvme: &'a Nvme, offset: u64, block_bytes: &'b mut [u8] } impl<'a, 'b> Seek for Device<'a, 'b> { fn seek(&mut self, from: io::SeekFrom) -> io::Result { @@ -91,7 +93,7 @@ impl DiskWrapper { partitionlib::get_partitions(&mut Device { disk, nvme, offset: 0, block_bytes: &mut block_bytes[..bs.into()] }, bs).ok().flatten() } - fn new(mut inner: NvmeNamespace, nvme: &mut Nvme) -> Self { + fn new(mut inner: NvmeNamespace, nvme: &Nvme) -> Self { Self { pt: Self::pt(&mut inner, nvme), inner, @@ -101,17 +103,17 @@ impl DiskWrapper { pub struct DiskScheme { scheme_name: String, - nvme: Nvme, + nvme: Arc, disks: BTreeMap, handles: BTreeMap, next_id: usize } impl DiskScheme { - pub fn new(scheme_name: String, mut nvme: Nvme, disks: BTreeMap) -> DiskScheme { + pub fn new(scheme_name: String, nvme: Arc, disks: BTreeMap) -> DiskScheme { DiskScheme { scheme_name, - disks: disks.into_iter().map(|(k, v)| (k, DiskWrapper::new(v, &mut nvme))).collect(), + disks: disks.into_iter().map(|(k, v)| (k, DiskWrapper::new(v, &nvme))).collect(), nvme, handles: BTreeMap::new(), next_id: 0 @@ -124,8 +126,11 @@ impl DiskScheme { let mut found_completion = false; let nvme = &mut self.nvme; - for qid in 0..nvme.completion_queues.len() { - while let Some((head, entry)) = nvme.completion_queues[qid].complete() { + let completion_queues = nvme.completion_queues.read().unwrap(); + + for qid in 0..completion_queues.len() { + let queue = completion_queues[qid].lock().unwrap(); + while let Some((head, entry)) = queue.complete() { found_completion = true; println!("nvmed: Unhandled completion {:?}", entry); //TODO: Handle errors