diff --git a/nvmed/src/main.rs b/nvmed/src/main.rs index 539c70b69d..483c9aafe5 100644 --- a/nvmed/src/main.rs +++ b/nvmed/src/main.rs @@ -32,7 +32,7 @@ impl Bar { pub fn allocate(bar: usize, bar_size: usize) -> Result { Ok(Self { ptr: NonNull::new( - syscall::physmap(bar, bar_size, PHYSMAP_NO_CACHE | PHYSMAP_WRITE)? as *mut u8, + unsafe { syscall::physmap(bar, bar_size, PHYSMAP_NO_CACHE | PHYSMAP_WRITE)? as *mut u8 }, ) .expect("Mapping a BAR resulted in a nullptr"), physical: bar, @@ -43,7 +43,7 @@ impl Bar { impl Drop for Bar { fn drop(&mut self) { - let _ = syscall::physunmap(self.physical); + let _ = unsafe { syscall::physunmap(self.physical) }; } } @@ -82,7 +82,7 @@ fn get_int_method( bir: u8, ) -> Result> { let bir = usize::from(bir); - let bar_guard = allocated_bars.0[bir].lock().unwrap(); + let mut bar_guard = allocated_bars.0[bir].lock().unwrap(); match &mut *bar_guard { &mut Some(ref bar) => Ok(bar.ptr), bar_to_set @ &mut None => { @@ -119,7 +119,7 @@ fn get_int_method( // Mask all interrupts in case some earlier driver/os already unmasked them (according to // the PCI Local Bus spec 3.0, they are masked after system reset). - for table_entry in table_entries { + for table_entry in table_entries.iter_mut() { table_entry.mask(); } @@ -284,10 +284,10 @@ fn main() { .expect("nvmed: failed to find a suitable interrupt method"); let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender) .expect("nvmed: failed to allocate driver data"); - let nvme = Arc::new(nvme); unsafe { nvme.init() } - nvme::cq_reactor::start_cq_reactor_thread(nvme, interrupt_sources, reactor_receiver); - let namespaces = unsafe { futures::executor::block_on(nvme.init_with_queues()) }; + let nvme = Arc::new(nvme); + nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver); + let namespaces = futures::executor::block_on(nvme.init_with_queues()); let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces); let mut todo = Vec::new(); diff --git a/nvmed/src/nvme/cq_reactor.rs b/nvmed/src/nvme/cq_reactor.rs index c2e80fa5a5..e4dbfefe87 100644 --- a/nvmed/src/nvme/cq_reactor.rs +++ b/nvmed/src/nvme/cq_reactor.rs @@ -62,7 +62,7 @@ struct CqReactor { event_queue: File, } impl CqReactor { - fn create_event_queue(int_sources: &InterruptSources) -> Result { + fn create_event_queue(int_sources: &mut InterruptSources) -> Result { use syscall::flag::*; let fd = syscall::open("event:", O_CLOEXEC | O_RDWR)?; let mut file = unsafe { File::from_raw_fd(fd as RawFd) }; @@ -84,11 +84,11 @@ impl CqReactor { } fn new( nvme: Arc, - int_sources: InterruptSources, + mut int_sources: InterruptSources, receiver: Receiver, ) -> Result { Ok(Self { - event_queue: Self::create_event_queue(&int_sources)?, + event_queue: Self::create_event_queue(&mut int_sources)?, int_sources, nvme, pending_reqs: Vec::new(), @@ -121,8 +121,10 @@ impl CqReactor { let mut entry_count = 0; - for cq_id in ivs_read_guard.get(&iv)?.iter().copied() { - let completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap(); + let cq_ids = ivs_read_guard.get(&iv)?; + + for cq_id in cq_ids.iter().copied() { + let mut completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap(); let &mut (ref mut completion_queue, _) = &mut *completion_queue_guard; let (head, entry) = match completion_queue.complete() { @@ -130,11 +132,11 @@ impl CqReactor { None => continue, }; - self.nvme.completion_queue_head(cq_id, head); + unsafe { self.nvme.completion_queue_head(cq_id, head) }; - self.nvme.submission_queues.read().unwrap().get(&entry.sq_id).expect("nvmed: internal error: queue returned from controller doesn't exist").lock().unwrap().head = entry.sq_head; + self.nvme.submission_queues.read().unwrap().get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist").lock().unwrap().head = entry.sq_head; - self.try_notify_futures(cq_id, &entry); + Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry); entry_count += 1; } @@ -142,12 +144,12 @@ impl CqReactor { Some(()) } - fn finish_pending_completion(&mut self, req_cq_id: CqId, cq_id: CqId, sq_id: SqId, cmd_id: CmdId, entry: &NvmeComp, i: usize) -> bool { + fn finish_pending_completion(pending_reqs: &mut Vec, req_cq_id: CqId, cq_id: CqId, sq_id: SqId, cmd_id: CmdId, entry: &NvmeComp, i: usize) -> bool { if req_cq_id == cq_id && sq_id == entry.sq_id && cmd_id == entry.cid { - let (waker, message) = match self.pending_reqs.remove(i) { + let (waker, message) = match pending_reqs.remove(i) { PendingReq::PendingCompletion { waker, message, .. } => (waker, message), _ => unreachable!(), }; @@ -160,9 +162,9 @@ impl CqReactor { false } } - fn finish_pending_avail_submission(&mut self, sq_id: SqId, entry: &NvmeComp, i: usize) -> bool { + fn finish_pending_avail_submission(pending_reqs: &mut Vec, sq_id: SqId, entry: &NvmeComp, i: usize) -> bool { if sq_id == entry.sq_id { - let waker = match self.pending_reqs.remove(i) { + let waker = match pending_reqs.remove(i) { PendingReq::PendingAvailSubmission { waker, .. } => waker, _ => unreachable!(), }; @@ -173,19 +175,19 @@ impl CqReactor { false } } - fn try_notify_futures(&mut self, cq_id: CqId, entry: &NvmeComp) -> Option<()> { + fn try_notify_futures(pending_reqs: &mut Vec, cq_id: CqId, entry: &NvmeComp) -> Option<()> { let mut i = 0usize; let mut futures_notified = 0; - while i < self.pending_reqs.len() { - match &self.pending_reqs[i] { - &PendingReq::PendingCompletion { cq_id: req_cq_id, sq_id, cmd_id, .. } => if self.finish_pending_completion(req_cq_id, cq_id, sq_id, cmd_id, entry, i) { + while i < pending_reqs.len() { + match &pending_reqs[i] { + &PendingReq::PendingCompletion { cq_id: req_cq_id, sq_id, cmd_id, .. } => if Self::finish_pending_completion(pending_reqs, req_cq_id, cq_id, sq_id, cmd_id, entry, i) { futures_notified += 1; } else { i += 1; } - &PendingReq::PendingAvailSubmission { sq_id, .. } => if self.finish_pending_avail_submission(sq_id, entry, i) { + &PendingReq::PendingAvailSubmission { sq_id, .. } => if Self::finish_pending_avail_submission(pending_reqs, sq_id, entry, i) { futures_notified += 1; } else { i += 1; @@ -250,7 +252,7 @@ pub fn start_cq_reactor_thread( }) } -struct CompletionMessage { +pub struct CompletionMessage { cq_entry: NvmeComp, } @@ -264,6 +266,7 @@ enum CompletionFutureState { message: Arc>>, }, Finished, + Placeholder, } pub struct CompletionFuture { state: CompletionFutureState, @@ -278,8 +281,8 @@ impl Future for CompletionFuture { fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll { let this = &mut self.get_mut().state; - match this { - &mut CompletionFutureState::Pending { + match mem::replace(this, CompletionFutureState::Placeholder) { + CompletionFutureState::Pending { message, cq_id, cmd_id, @@ -288,21 +291,22 @@ impl Future for CompletionFuture { } => { if let Some(value) = message.lock().unwrap().take() { *this = CompletionFutureState::Finished; - task::Poll::Ready(value.cq_entry) - } else { - sender.send(NotifReq::RequestCompletion { - cq_id, - sq_id, - cmd_id, - waker: context.waker().clone(), - message: Arc::clone(&message), - }); - task::Poll::Pending + return task::Poll::Ready(value.cq_entry); } + sender.send(NotifReq::RequestCompletion { + cq_id, + sq_id, + cmd_id, + waker: context.waker().clone(), + message: Arc::clone(&message), + }).expect("reactor dead"); + *this = CompletionFutureState::Pending { message, cq_id, cmd_id, sq_id, sender }; + task::Poll::Pending } - &mut CompletionFutureState::Finished => { + CompletionFutureState::Finished => { panic!("calling poll() on an already finished CompletionFuture") } + CompletionFutureState::Placeholder => unreachable!(), } } } @@ -345,6 +349,7 @@ pub(crate) enum SubmissionFutureState<'a, F> { // returned when there was an available submission entry from the beginning Ready(CmdId), Finished, + Placeholder, } /// A future representing a submission queue eventually becoming non-full. In most cases this @@ -361,8 +366,8 @@ impl NvmeCmd> Future for SubmissionFuture<'_, F> { fn poll(self: Pin<&mut Self>, context: &mut task::Context<'_>) -> task::Poll { let state = &mut self.get_mut().state; - match state { - &mut SubmissionFutureState::Pending { sq_id, cmd_init, nvme } => match nvme.try_submit_command(sq_id, cmd_init) { + match mem::replace(state, SubmissionFutureState::Placeholder) { + SubmissionFutureState::Pending { sq_id, cmd_init, nvme } => match nvme.try_submit_command(sq_id, cmd_init) { Ok(cmd_id) => { *state = SubmissionFutureState::Finished; task::Poll::Ready(cmd_id) @@ -373,11 +378,12 @@ impl NvmeCmd> Future for SubmissionFuture<'_, F> { task::Poll::Pending } } - &mut SubmissionFutureState::Ready(value) => { + SubmissionFutureState::Ready(value) => { *state = SubmissionFutureState::Finished; task::Poll::Ready(value) } - &mut SubmissionFutureState::Finished => panic!("calling poll() on an already finished SubmissionFuture"), + SubmissionFutureState::Finished => panic!("calling poll() on an already finished SubmissionFuture"), + SubmissionFutureState::Placeholder => unreachable!(), } } } diff --git a/nvmed/src/nvme/identify.rs b/nvmed/src/nvme/identify.rs index 5bae5d76c2..4bcb830d22 100644 --- a/nvmed/src/nvme/identify.rs +++ b/nvmed/src/nvme/identify.rs @@ -78,8 +78,9 @@ impl Nvme { // println!(" - Dumping identify namespace"); - let size = *(data.as_ptr().offset(0) as *const u64); - let capacity = *(data.as_ptr().offset(8) as *const u64); + // TODO: Use struct + let size = unsafe { *(data.as_ptr().offset(0) as *const u64) }; + let capacity = unsafe { *(data.as_ptr().offset(8) as *const u64) }; println!(" - ID: {} Size: {} Capacity: {}", nsid, size, capacity); //TODO: Read block size diff --git a/nvmed/src/nvme/mod.rs b/nvmed/src/nvme/mod.rs index 362bedf789..e2c194db96 100644 --- a/nvmed/src/nvme/mod.rs +++ b/nvmed/src/nvme/mod.rs @@ -55,9 +55,9 @@ impl InterruptSources { } fn size_hint(&self) -> (usize, Option) { match self { - &Self::Msi(mut iter) => iter.size_hint(), - &Self::MsiX(mut iter) => iter.size_hint(), - &Self::Intx(mut iter) => iter.size_hint(), + &Self::Msi(ref iter) => iter.size_hint(), + &Self::MsiX(ref iter) => iter.size_hint(), + &Self::Intx(ref iter) => iter.size_hint(), } } } @@ -224,10 +224,13 @@ impl Nvme { /// # Locking /// Locks `regs`. unsafe fn doorbell_write(&self, index: usize, value: u32) { - let mut regs_guard = self.regs.write().unwrap(); + use std::ops::DerefMut; - let dstrd = ((regs_guard.cap.read() >> 32) & 0b1111) as usize; - let addr = ((*regs_guard) as *mut NvmeRegs as usize) + 0x1000 + index * (4 << dstrd); + let mut regs_guard = self.regs.write().unwrap(); + let mut regs: &mut NvmeRegs = regs_guard.deref_mut(); + + let dstrd = ((regs.cap.read() >> 32) & 0b1111) as usize; + let addr = (regs as *mut NvmeRegs as usize) + 0x1000 + index * (4 << dstrd); (&mut *(addr as *mut Mmio)).write(value); } @@ -260,7 +263,7 @@ impl Nvme { let csts = self.regs.get_mut().unwrap().csts.read(); // println!("CSTS: {:X}", csts); if csts & 1 == 1 { - unsafe { std::arch::x86_64::_mm_pause() } + std::arch::x86_64::_mm_pause(); } else { break; } @@ -276,13 +279,13 @@ impl Nvme { } } - for (qid, queue) in self.completion_queues.get_mut().unwrap().iter() { + for (qid, queue) in self.completion_queues.get_mut().unwrap().iter_mut() { let &(ref cq, ref sq_ids) = &*queue.get_mut().unwrap(); let data = &cq.data; // println!(" - completion queue {}: {:X}, {}", qid, data.physical(), data.len()); } - for (qid, queue) in self.submission_queues.get_mut().unwrap().iter() { + for (qid, queue) in self.submission_queues.get_mut().unwrap().iter_mut() { let data = &queue.get_mut().unwrap().data; // println!(" - submission queue {}: {:X}, {}", qid, data.physical(), data.len()); } @@ -290,10 +293,10 @@ impl Nvme { { let regs = self.regs.get_mut().unwrap(); let submission_queues = self.submission_queues.get_mut().unwrap(); - let completion_queues = self.submission_queues.get_mut().unwrap(); + let completion_queues = self.completion_queues.get_mut().unwrap(); - let asq = submission_queues.get(&0).unwrap().get_mut().unwrap(); - let acq = completion_queues.get(&0).unwrap().get_mut().unwrap(); + let asq = submission_queues.get_mut(&0).unwrap().get_mut().unwrap(); + let (acq, _) = completion_queues.get_mut(&0).unwrap().get_mut().unwrap(); regs.aqa .write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1)); regs.asq.write(asq.data.physical() as u64); @@ -314,7 +317,7 @@ impl Nvme { let csts = self.regs.get_mut().unwrap().csts.read(); // println!("CSTS: {:X}", csts); if csts & 1 == 0 { - unsafe { std::arch::x86_64::_mm_pause() } + std::arch::x86_64::_mm_pause(); } else { break; } @@ -326,7 +329,7 @@ impl Nvme { /// # Panics /// Will panic if the same vector is called twice with different mask flags. pub fn set_vectors_masked(&self, vectors: impl IntoIterator) { - let interrupt_method_guard = self.interrupt_method.lock().unwrap(); + let mut interrupt_method_guard = self.interrupt_method.lock().unwrap(); match &mut *interrupt_method_guard { &mut InterruptMethod::Intx => { @@ -398,7 +401,7 @@ impl Nvme { pub fn submit_command_generic<'a, F: FnOnce(CmdId) -> NvmeCmd>(&'a self, sq_id: SqId, full_sq_handling: FullSqHandling, cmd_init: F) -> SubmissionBehavior<'a, F> { let sqs_read_guard = self.submission_queues.read().unwrap(); - let sq_lock = sqs_read_guard + let mut sq_lock = sqs_read_guard .get(&sq_id) .expect("nvmed: internal error: given SQ for SQ ID not there") .lock() @@ -413,7 +416,8 @@ impl Nvme { u16::try_from(sq_lock.tail).expect("nvmed: internal error: CQ has more than 2^16 entries"); let tail = sq_lock.submit_unchecked(cmd_init(cmd_id)); let tail = u16::try_from(tail).unwrap(); - self.submission_queue_tail(sq_id, tail); + + unsafe { self.submission_queue_tail(sq_id, tail) }; match full_sq_handling { FullSqHandling::ErrorDirectly => SubmissionBehavior::Nonblocking(Ok(cmd_id)), @@ -535,8 +539,8 @@ impl Nvme { namespaces } - unsafe fn namespace_rw( - &mut self, + async fn namespace_rw( + &self, nsid: u32, lba: u64, blocks_1: u16, @@ -545,59 +549,51 @@ impl Nvme { //TODO: Get real block size let block_size = 512; + let buffer_prp_guard = self.buffer_prp.lock().unwrap(); + let bytes = ((blocks_1 as u64) + 1) * block_size; let (ptr0, ptr1) = if bytes <= 4096 { - (self.buffer_prp[0], 0) + (buffer_prp_guard[0], 0) } else if bytes <= 8192 { - (self.buffer_prp[0], self.buffer_prp[1]) + (buffer_prp_guard[0], buffer_prp_guard[1]) } else { - (self.buffer_prp[0], (self.buffer_prp.physical() + 8) as u64) + (buffer_prp_guard[0], (buffer_prp_guard.physical() + 8) as u64) }; - { - let qid = 1; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = if write { + let cmd_id = self.submit_command_async(1, |cid| { + if write { NvmeCmd::io_write(cid, nsid, lba, blocks_1, ptr0, ptr1) } else { NvmeCmd::io_read(cid, nsid, lba, blocks_1, ptr0, ptr1) - }; - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } + } + }).await; - { - let qid = 1; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - //TODO: Handle errors - self.completion_queue_head(qid as u16, head as u16); - } + let comp = self.completion(1, cmd_id, 1).await; + // TODO: Handle errors Ok(()) } - pub unsafe fn namespace_read( - &mut self, + pub async fn namespace_read( + &self, nsid: u32, mut lba: u64, buf: &mut [u8], ) -> Result> { - //TODO: Use interrupts - //TODO: Get real block size let block_size = 512; - for chunk in buf.chunks_mut(self.buffer.len()) { + let mut buffer_guard = self.buffer.lock().unwrap(); + + for chunk in buf.chunks_mut(buffer_guard.len()) { let blocks = (chunk.len() + block_size - 1) / block_size; assert!(blocks > 0); assert!(blocks <= 0x1_0000); - self.namespace_rw(nsid, lba, (blocks - 1) as u16, false)?; + self.namespace_rw(nsid, lba, (blocks - 1) as u16, false).await?; - chunk.copy_from_slice(&self.buffer[..chunk.len()]); + chunk.copy_from_slice(&buffer_guard[..chunk.len()]); lba += blocks as u64; } @@ -605,8 +601,8 @@ impl Nvme { Ok(Some(buf.len())) } - pub unsafe fn namespace_write( - &mut self, + pub async fn namespace_write( + &self, nsid: u32, mut lba: u64, buf: &[u8], @@ -616,15 +612,17 @@ impl Nvme { //TODO: Get real block size let block_size = 512; - for chunk in buf.chunks(self.buffer.len()) { + let mut buffer_guard = self.buffer.lock().unwrap(); + + for chunk in buf.chunks(buffer_guard.len()) { let blocks = (chunk.len() + block_size - 1) / block_size; assert!(blocks > 0); assert!(blocks <= 0x1_0000); - self.buffer[..chunk.len()].copy_from_slice(chunk); + buffer_guard[..chunk.len()].copy_from_slice(chunk); - self.namespace_rw(nsid, lba, (blocks - 1) as u16, true)?; + self.namespace_rw(nsid, lba, (blocks - 1) as u16, true).await?; lba += blocks as u64; } diff --git a/nvmed/src/nvme/queues.rs b/nvmed/src/nvme/queues.rs index aca1b49efc..cac0001b7f 100644 --- a/nvmed/src/nvme/queues.rs +++ b/nvmed/src/nvme/queues.rs @@ -55,7 +55,7 @@ pub struct NvmeCompQueue { impl NvmeCompQueue { pub fn new() -> Result { Ok(Self { - data: Dma::zeroed_unsized(256)?, + data: unsafe { Dma::zeroed_unsized(256)? }, head: 0, phase: true, }) @@ -98,7 +98,7 @@ pub struct NvmeCmdQueue { impl NvmeCmdQueue { pub fn new() -> Result { Ok(Self { - data: Dma::zeroed_unsized(64)?, + data: unsafe { Dma::zeroed_unsized(64)? }, tail: 0, head: 0, }) diff --git a/nvmed/src/scheme.rs b/nvmed/src/scheme.rs index 1badbcd282..090fdb67ba 100644 --- a/nvmed/src/scheme.rs +++ b/nvmed/src/scheme.rs @@ -82,10 +82,8 @@ impl DiskWrapper { return Err(io::Error::from_raw_os_error(syscall::EOVERFLOW)); } loop { - match unsafe { - nvme.namespace_read(disk.id, block, block_bytes) - .map_err(|err| io::Error::from_raw_os_error(err.errno))? - } { + match futures::executor::block_on(nvme.namespace_read(disk.id, block, block_bytes)) + .map_err(|err| io::Error::from_raw_os_error(err.errno))? { Some(bytes) => { assert_eq!(bytes, block_bytes.len()); assert_eq!(bytes, blksize as usize); @@ -161,29 +159,6 @@ impl DiskScheme { } } -impl DiskScheme { - pub fn irq(&mut self) -> bool { - let mut found_completion = false; - - let nvme = &mut self.nvme; - let completion_queues = nvme.completion_queues.read().unwrap(); - - for qid in 0..completion_queues.len() { - let queue = completion_queues[qid].lock().unwrap(); - while let Some((head, entry)) = queue.complete() { - found_completion = true; - println!("nvmed: Unhandled completion {:?}", entry); - //TODO: Handle errors - unsafe { - nvme.completion_queue_head(qid as u16, head as u16); - } - } - } - - found_completion - } -} - impl SchemeBlockMut for DiskScheme { fn open(&mut self, path: &[u8], flags: usize, uid: u32, _gid: u32) -> Result> { if uid == 0 { @@ -372,10 +347,7 @@ impl SchemeBlockMut for DiskScheme { Handle::Disk(number, ref mut size) => { let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?; let block_size = disk.as_ref().block_size; - if let Some(count) = unsafe { - self.nvme - .namespace_read(disk.as_ref().id, (*size as u64) / block_size, buf)? - } { + if let Some(count) = futures::executor::block_on(self.nvme.namespace_read(disk.as_ref().id, (*size as u64) / block_size, buf))? { *size += count; Ok(Some(count)) } else { @@ -400,9 +372,7 @@ impl SchemeBlockMut for DiskScheme { let abs_block = part.start_lba + rel_block; - if let Some(count) = - unsafe { self.nvme.namespace_read(disk.as_ref().id, abs_block, buf)? } - { + if let Some(count) = futures::executor::block_on(self.nvme.namespace_read(disk.as_ref().id, abs_block, buf))? { *offset += count; Ok(Some(count)) } else { @@ -419,8 +389,8 @@ impl SchemeBlockMut for DiskScheme { let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?; let block_size = disk.as_ref().block_size; if let Some(count) = unsafe { - self.nvme - .namespace_write(disk.as_ref().id, (*size as u64) / block_size, buf)? + futures::executor::block_on(self.nvme + .namespace_write(disk.as_ref().id, (*size as u64) / block_size, buf))? } { *size += count; Ok(Some(count)) @@ -446,10 +416,7 @@ impl SchemeBlockMut for DiskScheme { let abs_block = part.start_lba + rel_block; - if let Some(count) = unsafe { - self.nvme - .namespace_write(disk.as_ref().id, abs_block, buf)? - } { + if let Some(count) = futures::executor::block_on(self.nvme.namespace_write(disk.as_ref().id, abs_block, buf))? { *offset += count; Ok(Some(count)) } else {