diff --git a/nvmed/src/nvme/mod.rs b/nvmed/src/nvme/mod.rs index 6dc5b47274..57b39a41cf 100644 --- a/nvmed/src/nvme/mod.rs +++ b/nvmed/src/nvme/mod.rs @@ -1,3 +1,4 @@ +use std::convert::TryFrom; use std::collections::BTreeMap; use std::fs::File; use std::sync::{Mutex, RwLock}; @@ -93,7 +94,7 @@ pub struct MsixCfg { pub pba: &'static mut [Mmio], } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, Default)] #[repr(packed)] pub struct NvmeCmd { /// Opcode @@ -124,11 +125,12 @@ pub struct NvmeCmd { cdw15: u32, } -pub struct IoCompletionQueueCreateInfo { -} - impl NvmeCmd { - pub fn create_io_completion_queue(cid: u16, qid: u16, ptr: usize, size: u16) -> Self { + pub fn create_io_completion_queue(cid: u16, qid: u16, ptr: usize, size: u16, iv: Option) -> Self { + const DW11_PHYSICALLY_CONTIGUOUS_BIT: u32 = 0x0000_0001; + const DW11_ENABLE_INTERRUPTS_BIT: u32 = 0x0000_0002; + const DW11_INTERRUPT_VECTOR_SHIFT: u8 = 16; + Self { opcode: 5, flags: 0, @@ -138,7 +140,13 @@ impl NvmeCmd { mptr: 0, dptr: [ptr as u64, 0], cdw10: ((size as u32) << 16) | (qid as u32), - cdw11: 1 /* Physically Contiguous */, //TODO: IV, IEN + + cdw11: DW11_PHYSICALLY_CONTIGUOUS_BIT | if let Some(iv) = iv { + // enable interrupts if a vector is present + DW11_ENABLE_INTERRUPTS_BIT + | (u32::from(iv) << DW11_INTERRUPT_VECTOR_SHIFT) + } else { 0 }, + cdw12: 0, cdw13: 0, cdw14: 0, @@ -217,13 +225,21 @@ impl NvmeCmd { cdw15: 0, } } + pub fn get_features(cid: u16, ptr: usize, fid: u8) -> Self { + Self { + opcode: 0xA, + dptr: [ptr as u64, 0], + cdw10: u32::from(fid), // TODO: SEL + .. Default::default() + } + } pub fn io_read(cid: u16, nsid: u32, lba: u64, blocks_1: u16, ptr0: u64, ptr1: u64) -> Self { Self { opcode: 2, flags: 1 << 6, - cid: cid, - nsid: nsid, + cid, + nsid, _rsvd: 0, mptr: 0, dptr: [ptr0, ptr1], @@ -240,8 +256,8 @@ impl NvmeCmd { Self { opcode: 1, flags: 1 << 6, - cid: cid, - nsid: nsid, + cid, + nsid, _rsvd: 0, mptr: 0, dptr: [ptr0, ptr1], @@ -617,7 +633,32 @@ impl Nvme { block_size: 512, // TODO } } + pub async fn create_io_completion_queue(&self, io_cq_id: CqId, vector: Option) { + let (ptr, len) = { + let mut completion_queues_guard = self.completion_queues.write().unwrap(); + let queue_guard = completion_queues_guard.entry(io_cq_id).or_insert_with(|| { + let queue = NvmeCompQueue::new().expect("nvmed: failed to allocate I/O completion queue"); + let sqs = SmallVec::new(); + Mutex::new((queue, sqs)) + }).get_mut().unwrap(); + + let &(ref queue, _) = &*queue_guard; + (queue.data.physical(), queue.data.len()) + }; + + let len = u16::try_from(len).expect("nvmed: internal error: I/O CQ longer than 2^16 entries"); + let raw_len = len.checked_sub(1).expect("nvmed: internal error: CQID 0 for I/O CQ"); + + let cmd_id = self.submit_admin_command(|cid| NvmeCmd::create_io_completion_queue( + cid, io_cq_id, ptr, raw_len, vector, + )); + let comp = self.admin_queue_completion(cmd_id).await; + + if let Some(vector) = vector { + self.cqs_for_ivs.write().unwrap().entry(vector).or_insert_with(SmallVec::new).push(io_cq_id); + } + } pub async fn init_with_queues(&self) -> BTreeMap { let ((), nsids) = futures::join!(self.identify_controller(), self.identify_namespace_list(0)); @@ -627,32 +668,11 @@ impl Nvme { namespaces.insert(nsid, self.identify_namespace(nsid).await); } - for io_qid in 1..self.completion_queues.len() { - let (ptr, len) = { - let queue = &self.completion_queues[io_qid]; - (queue.data.physical(), queue.data.len()) - }; - - // println!(" - Attempting to create I/O completion queue {}", io_qid); - { - let qid = 0; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = NvmeCmd::create_io_completion_queue( - cid, io_qid as u16, ptr, (len - 1) as u16 - ); - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - // println!(" - Waiting to create I/O completion queue {}", io_qid); - { - let qid = 0; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - self.completion_queue_head(qid as u16, head as u16); - } + // TODO: Multiple queues + for io_qid in 1u16..=1 { + self.create_io_completion_queue(io_qid, Some(0)).await; } + /* for io_qid in 1..self.submission_queues.len() { let (ptr, len) = { @@ -681,6 +701,7 @@ impl Nvme { self.completion_queue_head(qid as u16, head as u16); } } + */ // println!(" - Complete");