From 3558397859f6ec470de387bf92a6743bc3c00f65 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 3 May 2020 15:47:09 +0200 Subject: [PATCH] ASYNC COMMAND SUBMISSION WORKS (partially). --- nvmed/src/main.rs | 166 ++++++++++++++-------------- nvmed/src/nvme/cq_reactor.rs | 202 +++++++++++++++++------------------ nvmed/src/nvme/identify.rs | 22 ++-- nvmed/src/nvme/mod.rs | 103 +++++------------- nvmed/src/nvme/queues.rs | 9 +- xhcid/src/main.rs | 8 +- 6 files changed, 232 insertions(+), 278 deletions(-) diff --git a/nvmed/src/main.rs b/nvmed/src/main.rs index 3691cc9a2d..2eb7eec7a2 100644 --- a/nvmed/src/main.rs +++ b/nvmed/src/main.rs @@ -48,7 +48,7 @@ impl Drop for Bar { #[derive(Default)] pub struct AllocatedBars(pub [Mutex>; 6]); -/// Get the most optimal yet functional interrupt mechanism: either (in the order preference): +/// Get the most optimal yet functional interrupt mechanism: either (in the order of preference): /// MSI-X, MSI, and INTx# pin. Returns both runtime interrupt structures (MSI/MSI-X capability /// structures), and the handles to the interrupts. fn get_int_method( @@ -56,6 +56,7 @@ fn get_int_method( function: &PciFunction, allocated_bars: &AllocatedBars, ) -> Result<(InterruptMethod, InterruptSources)> { + log::trace!("Begin get_int_method"); use pcid_interface::irq_helpers; let features = pcid_handle.fetch_all_features().unwrap(); @@ -214,7 +215,14 @@ fn setup_logging() -> Option<&'static RedoxLogger> { let mut logger = RedoxLogger::new() .with_output( OutputBuilder::stderr() - .with_filter(log::LevelFilter::Info) // limit global output to important info + .with_filter(log::LevelFilter::Trace) // limit global output to important info + .with_ansi_escape_codes() + .flush_on_newline(true) + .build() + ) + .with_output( + OutputBuilder::with_endpoint(File::open("debug:").unwrap()) + .with_filter(log::LevelFilter::Trace) .with_ansi_escape_codes() .flush_on_newline(true) .build() @@ -295,87 +303,89 @@ fn main() { bar_size: bar_size as usize, ptr: NonNull::new(address as *mut u8).expect("Physmapping BAR gave nullptr"), }); - { - let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC) - .expect("nvmed: failed to open event queue"); - let mut event_file = unsafe { File::from_raw_fd(event_fd as RawFd) }; + let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC) + .expect("nvmed: failed to open event queue"); + let mut event_file = unsafe { File::from_raw_fd(event_fd as RawFd) }; - let scheme_name = format!("disk/{}", name); - let socket_fd = syscall::open( - &format!(":{}", scheme_name), - syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK | syscall::O_CLOEXEC, - ) - .expect("nvmed: failed to create disk scheme"); + let scheme_name = format!("disk/{}", name); + let socket_fd = syscall::open( + &format!(":{}", scheme_name), + syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK | syscall::O_CLOEXEC, + ) + .expect("nvmed: failed to create disk scheme"); - syscall::write( - event_fd, - &syscall::Event { - id: socket_fd, - flags: syscall::EVENT_READ, - data: 0, + syscall::write( + event_fd, + &syscall::Event { + id: socket_fd, + flags: syscall::EVENT_READ, + data: 0, + }, + ) + .expect("nvmed: failed to watch disk scheme events"); + + std::thread::sleep(std::time::Duration::from_millis(1000)); + + let mut socket_file = unsafe { File::from_raw_fd(socket_fd as RawFd) }; + + let (reactor_sender, reactor_receiver) = crossbeam_channel::unbounded(); + let (interrupt_method, interrupt_sources) = + get_int_method(&mut pcid_handle, &pci_config.func, &allocated_bars) + .expect("nvmed: failed to find a suitable interrupt method"); + let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender) + .expect("nvmed: failed to allocate driver data"); + unsafe { nvme.init() } + log::debug!("Finished base initialization"); + let nvme = Arc::new(nvme); + let reactor_thread = nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver); + let namespaces = futures::executor::block_on(nvme.init_with_queues()); + + syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace"); + + let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces); + let mut todo = Vec::new(); + 'events: loop { + let mut event = Event::default(); + if event_file + .read(&mut event) + .expect("nvmed: failed to read event queue") + == 0 + { + break; + } + + match event.data { + 0 => loop { + let mut packet = Packet::default(); + match socket_file.read(&mut packet) { + Ok(0) => break 'events, + Ok(_) => (), + Err(err) => match err.kind() { + ErrorKind::WouldBlock => break, + _ => Err(err).expect("nvmed: failed to read disk scheme"), + }, + } + todo.push(packet); }, - ) - .expect("nvmed: failed to watch disk scheme events"); - - let mut socket_file = unsafe { File::from_raw_fd(socket_fd as RawFd) }; - - let (reactor_sender, reactor_receiver) = crossbeam_channel::unbounded(); - let (interrupt_method, interrupt_sources) = - get_int_method(&mut pcid_handle, &pci_config.func, &allocated_bars) - .expect("nvmed: failed to find a suitable interrupt method"); - let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender) - .expect("nvmed: failed to allocate driver data"); - unsafe { nvme.init() } - let nvme = Arc::new(nvme); - nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver); - let namespaces = futures::executor::block_on(nvme.init_with_queues()); - - syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace"); - - let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces); - let mut todo = Vec::new(); - 'events: loop { - let mut event = Event::default(); - if event_file - .read(&mut event) - .expect("nvmed: failed to read event queue") - == 0 - { - break; - } - - match event.data { - 0 => loop { - let mut packet = Packet::default(); - match socket_file.read(&mut packet) { - Ok(0) => break 'events, - Ok(_) => (), - Err(err) => match err.kind() { - ErrorKind::WouldBlock => break, - _ => Err(err).expect("nvmed: failed to read disk scheme"), - }, - } - todo.push(packet); - }, - unknown => { - panic!("nvmed: unknown event data {}", unknown); - } - } - - let mut i = 0; - while i < todo.len() { - if let Some(a) = scheme.handle(&todo[i]) { - let mut packet = todo.remove(i); - packet.a = a; - socket_file - .write(&packet) - .expect("nvmed: failed to write disk scheme"); - } else { - i += 1; - } + unknown => { + panic!("nvmed: unknown event data {}", unknown); } } - //TODO: destroy NVMe stuff + let mut i = 0; + while i < todo.len() { + if let Some(a) = scheme.handle(&todo[i]) { + let mut packet = todo.remove(i); + packet.a = a; + socket_file + .write(&packet) + .expect("nvmed: failed to write disk scheme"); + } else { + i += 1; + } + } } + + //TODO: destroy NVMe stuff + reactor_thread.join().expect("nvmed: failed to join reactor thread"); } diff --git a/nvmed/src/nvme/cq_reactor.rs b/nvmed/src/nvme/cq_reactor.rs index e4dbfefe87..f78547bb4b 100644 --- a/nvmed/src/nvme/cq_reactor.rs +++ b/nvmed/src/nvme/cq_reactor.rs @@ -5,6 +5,7 @@ //! can also be used for notifying when a full submission queue can submit a new command (see //! `AvailableSqEntryFuture`). +use std::convert::TryFrom; use std::fs::File; use std::future::Future; use std::io::prelude::*; @@ -14,15 +15,15 @@ use std::sync::{Arc, Mutex}; use std::{mem, task, thread}; use syscall::data::Event; -use syscall::flag::EVENT_READ; use syscall::Result; -use crossbeam_channel::{Receiver, Sender}; +use crossbeam_channel::Receiver; use super::{CmdId, CqId, InterruptSources, Nvme, NvmeComp, NvmeCmd, SqId}; /// A notification request, sent by the future in order to tell the completion thread that the /// current task wants a notification when a matching completion queue entry has been seen. +#[derive(Debug)] pub enum NotifReq { RequestCompletion { cq_id: CqId, @@ -58,6 +59,7 @@ struct CqReactor { int_sources: InterruptSources, nvme: Arc, pending_reqs: Vec, + // used to store commands that may be completed before a completion is requested receiver: Receiver, event_queue: File, } @@ -95,8 +97,20 @@ impl CqReactor { receiver, }) } - fn handle_notif_reqs(&mut self) { - for req in self.receiver.try_iter() { + fn handle_notif_reqs_raw(pending_reqs: &mut Vec, receiver: &Receiver, block_until_first: bool) { + let mut blocking_iter; + let mut nonblocking_iter; + + let iter: &mut dyn Iterator = if block_until_first { + blocking_iter = std::iter::once(receiver.recv().unwrap()).chain(receiver.try_iter()); + &mut blocking_iter + } else { + nonblocking_iter = receiver.try_iter(); + &mut nonblocking_iter + }; + + for req in iter { + log::trace!("Got notif req: {:?}", req); match req { NotifReq::RequestCompletion { sq_id, @@ -104,14 +118,14 @@ impl CqReactor { cmd_id, waker, message, - } => self.pending_reqs.push(PendingReq::PendingCompletion { + } => pending_reqs.push(PendingReq::PendingCompletion { sq_id, cq_id, cmd_id, message, waker, }), - NotifReq::RequestAvailSubmission { sq_id, waker } => self.pending_reqs.push(PendingReq::PendingAvailSubmission { sq_id, waker, }), + NotifReq::RequestAvailSubmission { sq_id, waker } => pending_reqs.push(PendingReq::PendingAvailSubmission { sq_id, waker, }), } } } @@ -127,18 +141,29 @@ impl CqReactor { let mut completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap(); let &mut (ref mut completion_queue, _) = &mut *completion_queue_guard; - let (head, entry) = match completion_queue.complete() { - Some(e) => e, - None => continue, - }; + while let Some((head, entry)) = completion_queue.complete() { + unsafe { self.nvme.completion_queue_head(cq_id, head) }; - unsafe { self.nvme.completion_queue_head(cq_id, head) }; + log::trace!("Got completion queue entry (CQID {}): {:?} at {}", cq_id, entry, head); - self.nvme.submission_queues.read().unwrap().get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist").lock().unwrap().head = entry.sq_head; + { + let submission_queues_read_lock = self.nvme.submission_queues.read().unwrap(); + // this lock is actually important, since it will block during submission from other + // threads. the lock won't be held for long by the submitters, but it still prevents + // the entry being lost before this reactor is actually able to respond: + let &(ref sq_lock, corresponding_cq_id) = submission_queues_read_lock.get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist"); + assert_eq!(cq_id, corresponding_cq_id); + let mut sq_guard = sq_lock.lock().unwrap(); + sq_guard.head = entry.sq_head; + // the channel still has to be polled twice though: + Self::handle_notif_reqs_raw(&mut self.pending_reqs, &self.receiver, false); + } - Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry); - entry_count += 1; + Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry); + + entry_count += 1; + } } if entry_count == 0 {} @@ -199,24 +224,24 @@ impl CqReactor { } fn run(mut self) { + log::debug!("Running CQ reactor"); let mut event = Event::default(); let mut irq_word = [0u8; 8]; // stores the IRQ count const WORD_SIZE: usize = mem::size_of::(); loop { - self.handle_notif_reqs(); + let block_until_first = self.pending_reqs.is_empty(); + Self::handle_notif_reqs_raw(&mut self.pending_reqs, &self.receiver, block_until_first); + log::trace!("Handled notif reqs"); // block on getting the next event if self.event_queue.read(&mut event).unwrap() == 0 { // event queue has been destroyed break; } - if event.flags & EVENT_READ != EVENT_READ { - continue; - } - let (vector, irq_handle) = match self.int_sources.iter_mut().nth(event.id) { + let (vector, irq_handle) = match self.int_sources.iter_mut().nth(event.data) { Some(s) => s, None => continue, }; @@ -227,6 +252,7 @@ impl CqReactor { if irq_handle.write(&irq_word[..WORD_SIZE]).unwrap() == 0 { continue; } + log::trace!("NVME IRQ: vector {}", vector); self.nvme.set_vector_masked(vector, true); self.poll_completion_queues(vector); self.nvme.set_vector_masked(vector, false); @@ -252,14 +278,22 @@ pub fn start_cq_reactor_thread( }) } +#[derive(Debug)] pub struct CompletionMessage { cq_entry: NvmeComp, } -enum CompletionFutureState { - // not really required, but makes futures inert - Pending { - sender: Sender, +enum CompletionFutureState<'a, F> { + // the future is in its initial state: the command has not been submitted yet, and no interest + // has been registered. this state will repeat until a free submission queue entry appears to + // it, which it probably will since queues aren't supposed to be nearly always be full. + PendingSubmission { + cmd_init: F, + nvme: &'a Nvme, + sq_id: SqId, + }, + PendingCompletion { + nvme: &'a Nvme, cq_id: CqId, cmd_id: CmdId, sq_id: SqId, @@ -268,39 +302,72 @@ enum CompletionFutureState { Finished, Placeholder, } -pub struct CompletionFuture { - state: CompletionFutureState, +pub struct CompletionFuture<'a, F> { + state: CompletionFutureState<'a, F>, } // enum not self-referential -impl Unpin for CompletionFuture {} +impl Unpin for CompletionFuture<'_, F> {} -impl Future for CompletionFuture { +impl Future for CompletionFuture<'_, F> +where + F: FnOnce(CmdId) -> NvmeCmd, +{ type Output = NvmeComp; fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll { let this = &mut self.get_mut().state; match mem::replace(this, CompletionFutureState::Placeholder) { - CompletionFutureState::Pending { + CompletionFutureState::PendingSubmission { cmd_init, nvme, sq_id } => { + let sqs_read_guard = nvme.submission_queues.read().unwrap(); + let &(ref sq_lock, cq_id) = sqs_read_guard + .get(&sq_id) + .expect("nvmed: internal error: given SQ for SQ ID not there"); + let mut sq_guard = sq_lock.lock().unwrap(); + let sq = &mut *sq_guard; + + if sq.is_full() { + // when the CQ reactor gets a new completion queue entry, it'll lock the + // submisson queue it came from. since we're holding the same lock, this + // message will always be sent before the reactor is done with the entry. + nvme.reactor_sender.send(NotifReq::RequestAvailSubmission { sq_id, waker: context.waker().clone() }).unwrap(); + *this = CompletionFutureState::PendingSubmission { cmd_init, nvme, sq_id }; + return task::Poll::Pending; + } + + let cmd_id = + u16::try_from(sq.tail).expect("nvmed: internal error: CQ has more than 2^16 entries"); + let tail = sq.submit_unchecked(cmd_init(cmd_id)); + let tail = u16::try_from(tail).unwrap(); + + // make sure that we register interest before the reactor can get notified + let message = Arc::new(Mutex::new(None)); + *this = CompletionFutureState::PendingCompletion { nvme, cq_id, cmd_id, sq_id, message: Arc::clone(&message), }; + nvme.reactor_sender.send(NotifReq::RequestCompletion { cq_id, sq_id, cmd_id, message, waker: context.waker().clone() }).expect("reactor dead"); + unsafe { nvme.submission_queue_tail(sq_id, tail) }; + task::Poll::Pending + } + CompletionFutureState::PendingCompletion { message, cq_id, cmd_id, sq_id, - sender, + nvme, } => { + println!("{:p}", &mut nvme.completion_queues.read().unwrap().get(&cq_id).unwrap().lock().unwrap().0); if let Some(value) = message.lock().unwrap().take() { *this = CompletionFutureState::Finished; return task::Poll::Ready(value.cq_entry); } - sender.send(NotifReq::RequestCompletion { + nvme.reactor_sender.send(NotifReq::RequestCompletion { cq_id, sq_id, cmd_id, waker: context.waker().clone(), message: Arc::clone(&message), }).expect("reactor dead"); - *this = CompletionFutureState::Pending { message, cq_id, cmd_id, sq_id, sender }; + *this = CompletionFutureState::PendingCompletion { message, cq_id, cmd_id, sq_id, nvme }; task::Poll::Pending } CompletionFutureState::Finished => { @@ -312,78 +379,9 @@ impl Future for CompletionFuture { } impl Nvme { - /// Returns a future representing an eventual completion queue event, in `cq_id`, from `sq_id`, - /// with the individual command identified by `cmd_id`. - pub fn completion(&self, sq_id: SqId, cmd_id: CmdId, cq_id: SqId) -> CompletionFuture { + pub fn submit_and_complete_command NvmeCmd>(&self, sq_id: SqId, cmd_init: F) -> CompletionFuture { CompletionFuture { - state: CompletionFutureState::Pending { - sender: self.reactor_sender.clone(), - cq_id, - cmd_id, - sq_id, - message: Arc::new(Mutex::new(None)), - }, - } - } - /// Returns a future representing a submission queue becoming non-full. Make sure that the - /// queue doesn't have any additional free entries first though, so that the reactor doesn't - /// have to interfere. - pub fn wait_for_available_submission<'a, F: FnOnce(CmdId) -> NvmeCmd>(&'a self, sq_id: SqId, f: F) -> SubmissionFuture<'a, F> { - SubmissionFuture { - state: SubmissionFutureState::Pending { - sq_id, - cmd_init: f, - nvme: &self, - }, - } - } -} - -pub(crate) enum SubmissionFutureState<'a, F> { - // the queue was known to be full when checked, thus the reactor is asked - Pending { - sq_id: SqId, - cmd_init: F, - nvme: &'a Nvme, - }, - // returned when there was an available submission entry from the beginning - Ready(CmdId), - Finished, - Placeholder, -} - -/// A future representing a submission queue eventually becoming non-full. In most cases this -/// future will finish directly, since all entries in the queue have to be occupied for it to block. -pub struct SubmissionFuture<'a, F> { - pub(crate) state: SubmissionFutureState<'a, F>, -} - -impl Unpin for SubmissionFuture<'_, F> {} - -impl NvmeCmd> Future for SubmissionFuture<'_, F> { - type Output = CmdId; - - fn poll(self: Pin<&mut Self>, context: &mut task::Context<'_>) -> task::Poll { - let state = &mut self.get_mut().state; - - match mem::replace(state, SubmissionFutureState::Placeholder) { - SubmissionFutureState::Pending { sq_id, cmd_init, nvme } => match nvme.try_submit_command(sq_id, cmd_init) { - Ok(cmd_id) => { - *state = SubmissionFutureState::Finished; - task::Poll::Ready(cmd_id) - } - Err(closure) => { - nvme.reactor_sender.send(NotifReq::RequestAvailSubmission { sq_id, waker: context.waker().clone() }); - *state = SubmissionFutureState::Pending { sq_id, cmd_init: closure, nvme }; - task::Poll::Pending - } - } - SubmissionFutureState::Ready(value) => { - *state = SubmissionFutureState::Finished; - task::Poll::Ready(value) - } - SubmissionFutureState::Finished => panic!("calling poll() on an already finished SubmissionFuture"), - SubmissionFutureState::Placeholder => unreachable!(), + state: CompletionFutureState::PendingSubmission { cmd_init, nvme: &self, sq_id }, } } } diff --git a/nvmed/src/nvme/identify.rs b/nvmed/src/nvme/identify.rs index 98937ec72f..8dff534a27 100644 --- a/nvmed/src/nvme/identify.rs +++ b/nvmed/src/nvme/identify.rs @@ -149,12 +149,10 @@ impl Nvme { let data: Dma = Dma::zeroed().unwrap(); // println!(" - Attempting to identify controller"); - let cid = self - .submit_admin_command(|cid| NvmeCmd::identify_controller(cid, data.physical())) + let comp = self + .submit_and_complete_admin_command(|cid| NvmeCmd::identify_controller(cid, data.physical())) .await; - - // println!(" - Waiting to identify controller"); - let comp = self.admin_queue_completion(cid).await; + log::trace!("Completion: {:?}", comp); // println!(" - Dumping identify controller"); @@ -176,14 +174,13 @@ impl Nvme { let data: Dma<[u32; 1024]> = Dma::zeroed().unwrap(); // println!(" - Attempting to retrieve namespace ID list"); - let cmd_id = self - .submit_admin_command(|cid| { + let comp = self + .submit_and_complete_admin_command(|cid| { NvmeCmd::identify_namespace_list(cid, data.physical(), base) }) .await; - // println!(" - Waiting to retrieve namespace ID list"); - let comp = self.admin_queue_completion(cmd_id).await; + log::trace!("Completion2: {:?}", comp); // println!(" - Dumping namespace ID list"); data.iter().copied().take_while(|&nsid| nsid != 0).collect() @@ -193,13 +190,10 @@ impl Nvme { let data: Dma = Dma::zeroed().unwrap(); // println!(" - Attempting to identify namespace {}", nsid); - let cmd_id = self - .submit_admin_command(|cid| NvmeCmd::identify_namespace(cid, data.physical(), nsid)) + let comp = self + .submit_and_complete_admin_command(|cid| NvmeCmd::identify_namespace(cid, data.physical(), nsid)) .await; - // println!(" - Waiting to identify namespace {}", nsid); - let comp = self.admin_queue_completion(cmd_id).await; - // println!(" - Dumping identify namespace"); let size = data.size_in_blocks(); diff --git a/nvmed/src/nvme/mod.rs b/nvmed/src/nvme/mod.rs index 96f86e1bd5..f5af0f0c37 100644 --- a/nvmed/src/nvme/mod.rs +++ b/nvmed/src/nvme/mod.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::convert::TryFrom; use std::fs::File; use std::ptr; -use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Mutex, RwLock}; use crossbeam_channel::Sender; @@ -159,7 +159,7 @@ pub struct Nvme { pcid_interface: Mutex, regs: RwLock<&'static mut NvmeRegs>, - pub(crate) submission_queues: RwLock>>, + pub(crate) submission_queues: RwLock, CqId)>>, pub(crate) completion_queues: RwLock)>>>, @@ -172,6 +172,8 @@ pub struct Nvme { next_sqid: AtomicSqId, next_cqid: AtomicCqId, + + next_avail_submission_epoch: AtomicU64, } unsafe impl Send for Nvme {} unsafe impl Sync for Nvme {} @@ -186,11 +188,6 @@ pub enum FullSqHandling { Wait, } -pub enum SubmissionBehavior<'a, F: FnOnce(CmdId) -> NvmeCmd> { - Nonblocking(Result), // TODO: Add full error - Future(self::cq_reactor::SubmissionFuture<'a, F>), -} - impl Nvme { pub fn new( address: usize, @@ -201,10 +198,10 @@ impl Nvme { Ok(Nvme { regs: RwLock::new(unsafe { &mut *(address as *mut NvmeRegs) }), submission_queues: RwLock::new( - std::iter::once((0u16, Mutex::new(NvmeCmdQueue::new()?))).collect(), + std::iter::once((0u16, (Mutex::new(NvmeCmdQueue::new()?), 0u16))).collect(), ), completion_queues: RwLock::new( - std::iter::once((0u16, Mutex::new((NvmeCompQueue::new()?, smallvec!())))).collect(), + std::iter::once((0u16, Mutex::new((NvmeCompQueue::new()?, smallvec!(0))))).collect(), ), // map the zero interrupt vector (which according to the spec shall always point to the // admin completion queue) to CQID 0 (admin completion queue) @@ -217,6 +214,7 @@ impl Nvme { next_sqid: AtomicSqId::new(0), next_cqid: AtomicCqId::new(0), + next_avail_submission_epoch: AtomicU64::new(0), }) } /// Write to a doorbell register. @@ -288,9 +286,9 @@ impl Nvme { log::debug!("completion queue {}: {:X}, {}, (submission queue ids: {:?}", qid, data.physical(), data.len(), sq_ids); } - for (qid, queue) in self.submission_queues.get_mut().unwrap().iter_mut() { + for (qid, (queue, cq_id)) in self.submission_queues.get_mut().unwrap().iter_mut() { let data = &queue.get_mut().unwrap().data; - log::debug!("submission queue {}: {:X}, {}", qid, data.physical(), data.len()); + log::debug!("submission queue {}: {:X}, {}, attached to CQID: {}", qid, data.physical(), data.len(), cq_id); } { @@ -298,7 +296,7 @@ impl Nvme { let submission_queues = self.submission_queues.get_mut().unwrap(); let completion_queues = self.completion_queues.get_mut().unwrap(); - let asq = submission_queues.get_mut(&0).unwrap().get_mut().unwrap(); + let asq = submission_queues.get_mut(&0).unwrap().0.get_mut().unwrap(); let (acq, _) = completion_queues.get_mut(&0).unwrap().get_mut().unwrap(); regs.aqa .write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1)); @@ -402,55 +400,8 @@ impl Nvme { self.set_vectors_masked(std::iter::once((vector, masked))) } - pub fn submit_command_generic<'a, F: FnOnce(CmdId) -> NvmeCmd>(&'a self, sq_id: SqId, full_sq_handling: FullSqHandling, cmd_init: F) -> SubmissionBehavior<'a, F> { - let sqs_read_guard = self.submission_queues.read().unwrap(); - let mut sq_lock = sqs_read_guard - .get(&sq_id) - .expect("nvmed: internal error: given SQ for SQ ID not there") - .lock() - .unwrap(); - if sq_lock.is_full() { - match full_sq_handling { - FullSqHandling::ErrorDirectly => return SubmissionBehavior::Nonblocking(Err(cmd_init)), - FullSqHandling::Wait => return SubmissionBehavior::Future(self.wait_for_available_submission(sq_id, cmd_init)), - } - } - let cmd_id = - u16::try_from(sq_lock.tail).expect("nvmed: internal error: CQ has more than 2^16 entries"); - let tail = sq_lock.submit_unchecked(cmd_init(cmd_id)); - let tail = u16::try_from(tail).unwrap(); - - unsafe { self.submission_queue_tail(sq_id, tail) }; - - match full_sq_handling { - FullSqHandling::ErrorDirectly => SubmissionBehavior::Nonblocking(Ok(cmd_id)), - FullSqHandling::Wait => SubmissionBehavior::Future(self::cq_reactor::SubmissionFuture { state: self::cq_reactor::SubmissionFutureState::Ready(cmd_id) }) - } - } - - /// Try submitting a new entry to the specified submission queue, or return None if the queue - /// was full. - pub fn try_submit_command NvmeCmd>( - &self, - sq_id: SqId, - f: F, - ) -> Result { - match self.submit_command_generic(sq_id, FullSqHandling::ErrorDirectly, f) { - SubmissionBehavior::Nonblocking(opt) => opt, - _ => unreachable!(), - } - } - pub async fn submit_command_async NvmeCmd>(&self, sq_id: SqId, f: F) -> CmdId { - match self.submit_command_generic(sq_id, FullSqHandling::Wait, f) { - SubmissionBehavior::Future(future) => future.await, - _ => unreachable!(), - } - } - pub async fn submit_admin_command NvmeCmd>(&self, f: F) -> CmdId { - self.submit_command_async(0, f).await - } - pub async fn admin_queue_completion(&self, cmd_id: CmdId) -> NvmeComp { - self.completion(0, cmd_id, 0).await + pub async fn submit_and_complete_admin_command NvmeCmd>(&self, cmd_init: F) -> NvmeComp { + self.submit_and_complete_command(0, cmd_init).await } pub async fn create_io_completion_queue(&self, io_cq_id: CqId, vector: Option) { @@ -478,12 +429,11 @@ impl Nvme { .checked_sub(1) .expect("nvmed: internal error: CQID 0 for I/O CQ"); - let cmd_id = self - .submit_admin_command(|cid| { + let comp = self + .submit_and_complete_admin_command(|cid| { NvmeCmd::create_io_completion_queue(cid, io_cq_id, ptr, raw_len, vector) }) .await; - let comp = self.admin_queue_completion(cmd_id).await; if let Some(vector) = vector { self.cqs_for_ivs @@ -498,17 +448,17 @@ impl Nvme { let (ptr, len) = { let mut submission_queues_guard = self.submission_queues.write().unwrap(); - let queue_guard = submission_queues_guard + let (queue_lock, _) = submission_queues_guard .entry(io_sq_id) .or_insert_with(|| { - Mutex::new( + (Mutex::new( NvmeCmdQueue::new() .expect("nvmed: failed to allocate I/O completion queue"), - ) - }) - .get_mut() - .unwrap(); - (queue_guard.data.physical(), queue_guard.data.len()) + ), io_cq_id) + }); + let queue = queue_lock.get_mut().unwrap(); + + (queue.data.physical(), queue.data.len()) }; let len = @@ -517,17 +467,18 @@ impl Nvme { .checked_sub(1) .expect("nvmed: internal error: SQID 0 for I/O SQ"); - let cmd_id = self - .submit_admin_command(|cid| { + let comp = self + .submit_and_complete_admin_command(|cid| { NvmeCmd::create_io_submission_queue(cid, io_sq_id, ptr, raw_len, io_cq_id) }) .await; - let comp = self.admin_queue_completion(cmd_id).await; } pub async fn init_with_queues(&self) -> BTreeMap { + log::trace!("preinit"); let ((), nsids) = futures::join!(self.identify_controller(), self.identify_namespace_list(0)); + log::debug!("first commands"); let mut namespaces = BTreeMap::new(); @@ -563,15 +514,13 @@ impl Nvme { (buffer_prp_guard[0], (buffer_prp_guard.physical() + 8) as u64) }; - let cmd_id = self.submit_command_async(1, |cid| { + let comp = self.submit_and_complete_command(1, |cid| { if write { NvmeCmd::io_write(cid, nsid, lba, blocks_1, ptr0, ptr1) } else { NvmeCmd::io_read(cid, nsid, lba, blocks_1, ptr0, ptr1) } }).await; - - let comp = self.completion(1, cmd_id, 1).await; // TODO: Handle errors Ok(()) diff --git a/nvmed/src/nvme/queues.rs b/nvmed/src/nvme/queues.rs index cac0001b7f..f6fe72a279 100644 --- a/nvmed/src/nvme/queues.rs +++ b/nvmed/src/nvme/queues.rs @@ -63,10 +63,11 @@ impl NvmeCompQueue { /// Get a new completion queue entry, or return None if no entry is available yet. pub(crate) fn complete(&mut self) -> Option<(u16, NvmeComp)> { + println!("PTR: {:p}", &*self as *const _); let entry = unsafe { ptr::read_volatile(self.data.as_ptr().add(self.head as usize)) }; // println!("{:?}", entry); - if ((entry.status & 1) == 1) == self.phase { - self.head = (self.head + 1) % (self.data.len() as u16); + if ((dbg!(entry.status) & 1) == 1) == dbg!(self.phase) { + self.head = dbg!(self.head + 1) % dbg!(self.data.len() as u16); if self.head == 0 { self.phase = !self.phase; } @@ -108,13 +109,13 @@ impl NvmeCmdQueue { self.head == self.tail } pub fn is_full(&self) -> bool { - self.head + 1 == self.tail + self.head == self.tail + 1 } /// Add a new submission command entry to the queue. The caller must ensure that the queue have free /// entries; this can be checked using `is_full`. pub fn submit_unchecked(&mut self, entry: NvmeCmd) -> u16 { - self.data[self.tail as usize] = entry; + unsafe { ptr::write_volatile(&mut self.data[self.tail as usize] as *mut _, entry) } self.tail = (self.tail + 1) % (self.data.len() as u16); self.tail } diff --git a/xhcid/src/main.rs b/xhcid/src/main.rs index 66faca6e95..f56aacaf4e 100644 --- a/xhcid/src/main.rs +++ b/xhcid/src/main.rs @@ -40,14 +40,15 @@ async fn handle_packet(hci: Arc, packet: Packet) -> Packet { fn setup_logging() -> Option<&'static RedoxLogger> { let mut logger = RedoxLogger::new() - .with_output( + /*.with_output( OutputBuilder::stderr() .with_filter(log::LevelFilter::Info) // limit global output to important info .with_ansi_escape_codes() .flush_on_newline(true) .build() - ); + )*/; + #[cfg(target_os = "redox")] match OutputBuilder::in_redox_logging_scheme("usb", "host", "xhci.log") { Ok(b) => logger = logger.with_output( // TODO: Add a configuration file for this @@ -58,6 +59,7 @@ fn setup_logging() -> Option<&'static RedoxLogger> { Err(error) => eprintln!("Failed to create xhci.log: {}", error), } + #[cfg(target_os = "redox")] match OutputBuilder::in_redox_logging_scheme("usb", "host", "xhci.ansi.log") { Ok(b) => logger = logger.with_output( b.with_filter(log::LevelFilter::Trace) @@ -221,7 +223,7 @@ fn main() { (None, InterruptMethod::Polling) }; - std::thread::sleep(std::time::Duration::from_millis(300)); + //std::thread::sleep(std::time::Duration::from_millis(300)); print!( "{}",