vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ASYNC COMMAND SUBMISSION WORKS (partially).

Browse Source
This commit is contained in:
4lDO2
2020-05-03 15:47:09 +02:00
parent cea6ce7d7a
commit 3558397859
6 changed files with 232 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nvmed/src/main.rs
+88 -78
View File
@@ -48,7 +48,7 @@ impl Drop for Bar {
#[derive(Default)]
pub struct AllocatedBars(pub [Mutex<Option<Bar>>; 6]);
/// Get the most optimal yet functional interrupt mechanism: either (in the order preference):
/// Get the most optimal yet functional interrupt mechanism: either (in the order of preference):
/// MSI-X, MSI, and INTx# pin. Returns both runtime interrupt structures (MSI/MSI-X capability
/// structures), and the handles to the interrupts.
fn get_int_method(
@@ -56,6 +56,7 @@ fn get_int_method(
function: &PciFunction,
allocated_bars: &AllocatedBars,
) -> Result<(InterruptMethod, InterruptSources)> {
log::trace!("Begin get_int_method");
use pcid_interface::irq_helpers;
let features = pcid_handle.fetch_all_features().unwrap();
@@ -214,7 +215,14 @@ fn setup_logging() -> Option<&'static RedoxLogger> {
let mut logger = RedoxLogger::new()
.with_output(
OutputBuilder::stderr()
.with_filter(log::LevelFilter::Info) // limit global output to important info
.with_filter(log::LevelFilter::Trace) // limit global output to important info
.with_ansi_escape_codes()
.flush_on_newline(true)
.build()
)
.with_output(
OutputBuilder::with_endpoint(File::open("debug:").unwrap())
.with_filter(log::LevelFilter::Trace)
.with_ansi_escape_codes()
.flush_on_newline(true)
.build()
@@ -295,87 +303,89 @@ fn main() {
bar_size: bar_size as usize,
ptr: NonNull::new(address as *mut u8).expect("Physmapping BAR gave nullptr"),
});
{
let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC)
.expect("nvmed: failed to open event queue");
let mut event_file = unsafe { File::from_raw_fd(event_fd as RawFd) };
let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC)
.expect("nvmed: failed to open event queue");
let mut event_file = unsafe { File::from_raw_fd(event_fd as RawFd) };
let scheme_name = format!("disk/{}", name);
let socket_fd = syscall::open(
&format!(":{}", scheme_name),
syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK | syscall::O_CLOEXEC,
)
.expect("nvmed: failed to create disk scheme");
let scheme_name = format!("disk/{}", name);
let socket_fd = syscall::open(
&format!(":{}", scheme_name),
syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK | syscall::O_CLOEXEC,
)
.expect("nvmed: failed to create disk scheme");
syscall::write(
event_fd,
&syscall::Event {
id: socket_fd,
flags: syscall::EVENT_READ,
data: 0,
syscall::write(
event_fd,
&syscall::Event {
id: socket_fd,
flags: syscall::EVENT_READ,
data: 0,
},
)
.expect("nvmed: failed to watch disk scheme events");
std::thread::sleep(std::time::Duration::from_millis(1000));
let mut socket_file = unsafe { File::from_raw_fd(socket_fd as RawFd) };
let (reactor_sender, reactor_receiver) = crossbeam_channel::unbounded();
let (interrupt_method, interrupt_sources) =
get_int_method(&mut pcid_handle, &pci_config.func, &allocated_bars)
.expect("nvmed: failed to find a suitable interrupt method");
let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender)
.expect("nvmed: failed to allocate driver data");
unsafe { nvme.init() }
log::debug!("Finished base initialization");
let nvme = Arc::new(nvme);
let reactor_thread = nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver);
let namespaces = futures::executor::block_on(nvme.init_with_queues());
syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace");
let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces);
let mut todo = Vec::new();
'events: loop {
let mut event = Event::default();
if event_file
.read(&mut event)
.expect("nvmed: failed to read event queue")
== 0
{
break;
}
match event.data {
0 => loop {
let mut packet = Packet::default();
match socket_file.read(&mut packet) {
Ok(0) => break 'events,
Ok(_) => (),
Err(err) => match err.kind() {
ErrorKind::WouldBlock => break,
_ => Err(err).expect("nvmed: failed to read disk scheme"),
},
}
todo.push(packet);
},
)
.expect("nvmed: failed to watch disk scheme events");
let mut socket_file = unsafe { File::from_raw_fd(socket_fd as RawFd) };
let (reactor_sender, reactor_receiver) = crossbeam_channel::unbounded();
let (interrupt_method, interrupt_sources) =
get_int_method(&mut pcid_handle, &pci_config.func, &allocated_bars)
.expect("nvmed: failed to find a suitable interrupt method");
let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender)
.expect("nvmed: failed to allocate driver data");
unsafe { nvme.init() }
let nvme = Arc::new(nvme);
nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver);
let namespaces = futures::executor::block_on(nvme.init_with_queues());
syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace");
let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces);
let mut todo = Vec::new();
'events: loop {
let mut event = Event::default();
if event_file
.read(&mut event)
.expect("nvmed: failed to read event queue")
== 0
{
break;
}
match event.data {
0 => loop {
let mut packet = Packet::default();
match socket_file.read(&mut packet) {
Ok(0) => break 'events,
Ok(_) => (),
Err(err) => match err.kind() {
ErrorKind::WouldBlock => break,
_ => Err(err).expect("nvmed: failed to read disk scheme"),
},
}
todo.push(packet);
},
unknown => {
panic!("nvmed: unknown event data {}", unknown);
}
}
let mut i = 0;
while i < todo.len() {
if let Some(a) = scheme.handle(&todo[i]) {
let mut packet = todo.remove(i);
packet.a = a;
socket_file
.write(&packet)
.expect("nvmed: failed to write disk scheme");
} else {
i += 1;
}
unknown => {
panic!("nvmed: unknown event data {}", unknown);
}
}
//TODO: destroy NVMe stuff
let mut i = 0;
while i < todo.len() {
if let Some(a) = scheme.handle(&todo[i]) {
let mut packet = todo.remove(i);
packet.a = a;
socket_file
.write(&packet)
.expect("nvmed: failed to write disk scheme");
} else {
i += 1;
}
}
}
//TODO: destroy NVMe stuff
reactor_thread.join().expect("nvmed: failed to join reactor thread");
}
nvmed/src/nvme/cq_reactor.rs
+100 -102
View File
@@ -5,6 +5,7 @@
//! can also be used for notifying when a full submission queue can submit a new command (see
//! `AvailableSqEntryFuture`).
use std::convert::TryFrom;
use std::fs::File;
use std::future::Future;
use std::io::prelude::*;
@@ -14,15 +15,15 @@ use std::sync::{Arc, Mutex};
use std::{mem, task, thread};
use syscall::data::Event;
use syscall::flag::EVENT_READ;
use syscall::Result;
use crossbeam_channel::{Receiver, Sender};
use crossbeam_channel::Receiver;
use super::{CmdId, CqId, InterruptSources, Nvme, NvmeComp, NvmeCmd, SqId};
/// A notification request, sent by the future in order to tell the completion thread that the
/// current task wants a notification when a matching completion queue entry has been seen.
#[derive(Debug)]
pub enum NotifReq {
RequestCompletion {
cq_id: CqId,
@@ -58,6 +59,7 @@ struct CqReactor {
int_sources: InterruptSources,
nvme: Arc<Nvme>,
pending_reqs: Vec<PendingReq>,
// used to store commands that may be completed before a completion is requested
receiver: Receiver<NotifReq>,
event_queue: File,
}
@@ -95,8 +97,20 @@ impl CqReactor {
receiver,
})
}
fn handle_notif_reqs(&mut self) {
for req in self.receiver.try_iter() {
fn handle_notif_reqs_raw(pending_reqs: &mut Vec<PendingReq>, receiver: &Receiver<NotifReq>, block_until_first: bool) {
let mut blocking_iter;
let mut nonblocking_iter;
let iter: &mut dyn Iterator<Item = NotifReq> = if block_until_first {
blocking_iter = std::iter::once(receiver.recv().unwrap()).chain(receiver.try_iter());
&mut blocking_iter
} else {
nonblocking_iter = receiver.try_iter();
&mut nonblocking_iter
};
for req in iter {
log::trace!("Got notif req: {:?}", req);
match req {
NotifReq::RequestCompletion {
sq_id,
@@ -104,14 +118,14 @@ impl CqReactor {
cmd_id,
waker,
message,
} => self.pending_reqs.push(PendingReq::PendingCompletion {
} => pending_reqs.push(PendingReq::PendingCompletion {
sq_id,
cq_id,
cmd_id,
message,
waker,
}),
NotifReq::RequestAvailSubmission { sq_id, waker } => self.pending_reqs.push(PendingReq::PendingAvailSubmission { sq_id, waker, }),
NotifReq::RequestAvailSubmission { sq_id, waker } => pending_reqs.push(PendingReq::PendingAvailSubmission { sq_id, waker, }),
}
}
}
@@ -127,18 +141,29 @@ impl CqReactor {
let mut completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap();
let &mut (ref mut completion_queue, _) = &mut *completion_queue_guard;
let (head, entry) = match completion_queue.complete() {
Some(e) => e,
None => continue,
};
while let Some((head, entry)) = completion_queue.complete() {
unsafe { self.nvme.completion_queue_head(cq_id, head) };
unsafe { self.nvme.completion_queue_head(cq_id, head) };
log::trace!("Got completion queue entry (CQID {}): {:?} at {}", cq_id, entry, head);
self.nvme.submission_queues.read().unwrap().get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist").lock().unwrap().head = entry.sq_head;
{
let submission_queues_read_lock = self.nvme.submission_queues.read().unwrap();
// this lock is actually important, since it will block during submission from other
// threads. the lock won't be held for long by the submitters, but it still prevents
// the entry being lost before this reactor is actually able to respond:
let &(ref sq_lock, corresponding_cq_id) = submission_queues_read_lock.get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist");
assert_eq!(cq_id, corresponding_cq_id);
let mut sq_guard = sq_lock.lock().unwrap();
sq_guard.head = entry.sq_head;
// the channel still has to be polled twice though:
Self::handle_notif_reqs_raw(&mut self.pending_reqs, &self.receiver, false);
}
Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry);
entry_count += 1;
Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry);
entry_count += 1;
}
}
if entry_count == 0 {}
@@ -199,24 +224,24 @@ impl CqReactor {
}
fn run(mut self) {
log::debug!("Running CQ reactor");
let mut event = Event::default();
let mut irq_word = [0u8; 8]; // stores the IRQ count
const WORD_SIZE: usize = mem::size_of::<usize>();
loop {
self.handle_notif_reqs();
let block_until_first = self.pending_reqs.is_empty();
Self::handle_notif_reqs_raw(&mut self.pending_reqs, &self.receiver, block_until_first);
log::trace!("Handled notif reqs");
// block on getting the next event
if self.event_queue.read(&mut event).unwrap() == 0 {
// event queue has been destroyed
break;
}
if event.flags & EVENT_READ != EVENT_READ {
continue;
}
let (vector, irq_handle) = match self.int_sources.iter_mut().nth(event.id) {
let (vector, irq_handle) = match self.int_sources.iter_mut().nth(event.data) {
Some(s) => s,
None => continue,
};
@@ -227,6 +252,7 @@ impl CqReactor {
if irq_handle.write(&irq_word[..WORD_SIZE]).unwrap() == 0 {
continue;
}
log::trace!("NVME IRQ: vector {}", vector);
self.nvme.set_vector_masked(vector, true);
self.poll_completion_queues(vector);
self.nvme.set_vector_masked(vector, false);
@@ -252,14 +278,22 @@ pub fn start_cq_reactor_thread(
})
}
#[derive(Debug)]
pub struct CompletionMessage {
cq_entry: NvmeComp,
}
enum CompletionFutureState {
// not really required, but makes futures inert
Pending {
sender: Sender<NotifReq>,
enum CompletionFutureState<'a, F> {
// the future is in its initial state: the command has not been submitted yet, and no interest
// has been registered. this state will repeat until a free submission queue entry appears to
// it, which it probably will since queues aren't supposed to be nearly always be full.
PendingSubmission {
cmd_init: F,
nvme: &'a Nvme,
sq_id: SqId,
},
PendingCompletion {
nvme: &'a Nvme,
cq_id: CqId,
cmd_id: CmdId,
sq_id: SqId,
@@ -268,39 +302,72 @@ enum CompletionFutureState {
Finished,
Placeholder,
}
pub struct CompletionFuture {
state: CompletionFutureState,
pub struct CompletionFuture<'a, F> {
state: CompletionFutureState<'a, F>,
}
// enum not self-referential
impl Unpin for CompletionFuture {}
impl<F> Unpin for CompletionFuture<'_, F> {}
impl Future for CompletionFuture {
impl<F> Future for CompletionFuture<'_, F>
where
F: FnOnce(CmdId) -> NvmeCmd,
{
type Output = NvmeComp;
fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll<Self::Output> {
let this = &mut self.get_mut().state;
match mem::replace(this, CompletionFutureState::Placeholder) {
CompletionFutureState::Pending {
CompletionFutureState::PendingSubmission { cmd_init, nvme, sq_id } => {
let sqs_read_guard = nvme.submission_queues.read().unwrap();
let &(ref sq_lock, cq_id) = sqs_read_guard
.get(&sq_id)
.expect("nvmed: internal error: given SQ for SQ ID not there");
let mut sq_guard = sq_lock.lock().unwrap();
let sq = &mut *sq_guard;
if sq.is_full() {
// when the CQ reactor gets a new completion queue entry, it'll lock the
// submisson queue it came from. since we're holding the same lock, this
// message will always be sent before the reactor is done with the entry.
nvme.reactor_sender.send(NotifReq::RequestAvailSubmission { sq_id, waker: context.waker().clone() }).unwrap();
*this = CompletionFutureState::PendingSubmission { cmd_init, nvme, sq_id };
return task::Poll::Pending;
}
let cmd_id =
u16::try_from(sq.tail).expect("nvmed: internal error: CQ has more than 2^16 entries");
let tail = sq.submit_unchecked(cmd_init(cmd_id));
let tail = u16::try_from(tail).unwrap();
// make sure that we register interest before the reactor can get notified
let message = Arc::new(Mutex::new(None));
*this = CompletionFutureState::PendingCompletion { nvme, cq_id, cmd_id, sq_id, message: Arc::clone(&message), };
nvme.reactor_sender.send(NotifReq::RequestCompletion { cq_id, sq_id, cmd_id, message, waker: context.waker().clone() }).expect("reactor dead");
unsafe { nvme.submission_queue_tail(sq_id, tail) };
task::Poll::Pending
}
CompletionFutureState::PendingCompletion {
message,
cq_id,
cmd_id,
sq_id,
sender,
nvme,
} => {
println!("{:p}", &mut nvme.completion_queues.read().unwrap().get(&cq_id).unwrap().lock().unwrap().0);
if let Some(value) = message.lock().unwrap().take() {
*this = CompletionFutureState::Finished;
return task::Poll::Ready(value.cq_entry);
}
sender.send(NotifReq::RequestCompletion {
nvme.reactor_sender.send(NotifReq::RequestCompletion {
cq_id,
sq_id,
cmd_id,
waker: context.waker().clone(),
message: Arc::clone(&message),
}).expect("reactor dead");
*this = CompletionFutureState::Pending { message, cq_id, cmd_id, sq_id, sender };
*this = CompletionFutureState::PendingCompletion { message, cq_id, cmd_id, sq_id, nvme };
task::Poll::Pending
}
CompletionFutureState::Finished => {
@@ -312,78 +379,9 @@ impl Future for CompletionFuture {
}
impl Nvme {
/// Returns a future representing an eventual completion queue event, in `cq_id`, from `sq_id`,
/// with the individual command identified by `cmd_id`.
pub fn completion(&self, sq_id: SqId, cmd_id: CmdId, cq_id: SqId) -> CompletionFuture {
pub fn submit_and_complete_command<F: FnOnce(CmdId) -> NvmeCmd>(&self, sq_id: SqId, cmd_init: F) -> CompletionFuture<F> {
CompletionFuture {
state: CompletionFutureState::Pending {
sender: self.reactor_sender.clone(),
cq_id,
cmd_id,
sq_id,
message: Arc::new(Mutex::new(None)),
},
}
}
/// Returns a future representing a submission queue becoming non-full. Make sure that the
/// queue doesn't have any additional free entries first though, so that the reactor doesn't
/// have to interfere.
pub fn wait_for_available_submission<'a, F: FnOnce(CmdId) -> NvmeCmd>(&'a self, sq_id: SqId, f: F) -> SubmissionFuture<'a, F> {
SubmissionFuture {
state: SubmissionFutureState::Pending {
sq_id,
cmd_init: f,
nvme: &self,
},
}
}
}
pub(crate) enum SubmissionFutureState<'a, F> {
// the queue was known to be full when checked, thus the reactor is asked
Pending {
sq_id: SqId,
cmd_init: F,
nvme: &'a Nvme,
},
// returned when there was an available submission entry from the beginning
Ready(CmdId),
Finished,
Placeholder,
}
/// A future representing a submission queue eventually becoming non-full. In most cases this
/// future will finish directly, since all entries in the queue have to be occupied for it to block.
pub struct SubmissionFuture<'a, F> {
pub(crate) state: SubmissionFutureState<'a, F>,
}
impl<F> Unpin for SubmissionFuture<'_, F> {}
impl<F: FnOnce(CmdId) -> NvmeCmd> Future for SubmissionFuture<'_, F> {
type Output = CmdId;
fn poll(self: Pin<&mut Self>, context: &mut task::Context<'_>) -> task::Poll<Self::Output> {
let state = &mut self.get_mut().state;
match mem::replace(state, SubmissionFutureState::Placeholder) {
SubmissionFutureState::Pending { sq_id, cmd_init, nvme } => match nvme.try_submit_command(sq_id, cmd_init) {
Ok(cmd_id) => {
*state = SubmissionFutureState::Finished;
task::Poll::Ready(cmd_id)
}
Err(closure) => {
nvme.reactor_sender.send(NotifReq::RequestAvailSubmission { sq_id, waker: context.waker().clone() });
*state = SubmissionFutureState::Pending { sq_id, cmd_init: closure, nvme };
task::Poll::Pending
}
}
SubmissionFutureState::Ready(value) => {
*state = SubmissionFutureState::Finished;
task::Poll::Ready(value)
}
SubmissionFutureState::Finished => panic!("calling poll() on an already finished SubmissionFuture"),
SubmissionFutureState::Placeholder => unreachable!(),
state: CompletionFutureState::PendingSubmission { cmd_init, nvme: &self, sq_id },
}
}
}
nvmed/src/nvme/identify.rs
+8 -14
View File
@@ -149,12 +149,10 @@ impl Nvme {
let data: Dma<IdentifyControllerData> = Dma::zeroed().unwrap();
// println!(" - Attempting to identify controller");
let cid = self
.submit_admin_command(|cid| NvmeCmd::identify_controller(cid, data.physical()))
let comp = self
.submit_and_complete_admin_command(|cid| NvmeCmd::identify_controller(cid, data.physical()))
.await;
// println!(" - Waiting to identify controller");
let comp = self.admin_queue_completion(cid).await;
log::trace!("Completion: {:?}", comp);
// println!(" - Dumping identify controller");
@@ -176,14 +174,13 @@ impl Nvme {
let data: Dma<[u32; 1024]> = Dma::zeroed().unwrap();
// println!(" - Attempting to retrieve namespace ID list");
let cmd_id = self
.submit_admin_command(|cid| {
let comp = self
.submit_and_complete_admin_command(|cid| {
NvmeCmd::identify_namespace_list(cid, data.physical(), base)
})
.await;
// println!(" - Waiting to retrieve namespace ID list");
let comp = self.admin_queue_completion(cmd_id).await;
log::trace!("Completion2: {:?}", comp);
// println!(" - Dumping namespace ID list");
data.iter().copied().take_while(|&nsid| nsid != 0).collect()
@@ -193,13 +190,10 @@ impl Nvme {
let data: Dma<IdentifyNamespaceData> = Dma::zeroed().unwrap();
// println!(" - Attempting to identify namespace {}", nsid);
let cmd_id = self
.submit_admin_command(|cid| NvmeCmd::identify_namespace(cid, data.physical(), nsid))
let comp = self
.submit_and_complete_admin_command(|cid| NvmeCmd::identify_namespace(cid, data.physical(), nsid))
.await;
// println!(" - Waiting to identify namespace {}", nsid);
let comp = self.admin_queue_completion(cmd_id).await;
// println!(" - Dumping identify namespace");
let size = data.size_in_blocks();
nvmed/src/nvme/mod.rs
+26 -77
View File
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::fs::File;
use std::ptr;
use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Mutex, RwLock};
use crossbeam_channel::Sender;
@@ -159,7 +159,7 @@ pub struct Nvme {
pcid_interface: Mutex<PcidServerHandle>,
regs: RwLock<&'static mut NvmeRegs>,
pub(crate) submission_queues: RwLock<BTreeMap<SqId, Mutex<NvmeCmdQueue>>>,
pub(crate) submission_queues: RwLock<BTreeMap<SqId, (Mutex<NvmeCmdQueue>, CqId)>>,
pub(crate) completion_queues:
RwLock<BTreeMap<CqId, Mutex<(NvmeCompQueue, SmallVec<[SqId; 16]>)>>>,
@@ -172,6 +172,8 @@ pub struct Nvme {
next_sqid: AtomicSqId,
next_cqid: AtomicCqId,
next_avail_submission_epoch: AtomicU64,
}
unsafe impl Send for Nvme {}
unsafe impl Sync for Nvme {}
@@ -186,11 +188,6 @@ pub enum FullSqHandling {
Wait,
}
pub enum SubmissionBehavior<'a, F: FnOnce(CmdId) -> NvmeCmd> {
Nonblocking(Result<CmdId, F>), // TODO: Add full error
Future(self::cq_reactor::SubmissionFuture<'a, F>),
}
impl Nvme {
pub fn new(
address: usize,
@@ -201,10 +198,10 @@ impl Nvme {
Ok(Nvme {
regs: RwLock::new(unsafe { &mut *(address as *mut NvmeRegs) }),
submission_queues: RwLock::new(
std::iter::once((0u16, Mutex::new(NvmeCmdQueue::new()?))).collect(),
std::iter::once((0u16, (Mutex::new(NvmeCmdQueue::new()?), 0u16))).collect(),
),
completion_queues: RwLock::new(
std::iter::once((0u16, Mutex::new((NvmeCompQueue::new()?, smallvec!())))).collect(),
std::iter::once((0u16, Mutex::new((NvmeCompQueue::new()?, smallvec!(0))))).collect(),
),
// map the zero interrupt vector (which according to the spec shall always point to the
// admin completion queue) to CQID 0 (admin completion queue)
@@ -217,6 +214,7 @@ impl Nvme {
next_sqid: AtomicSqId::new(0),
next_cqid: AtomicCqId::new(0),
next_avail_submission_epoch: AtomicU64::new(0),
})
}
/// Write to a doorbell register.
@@ -288,9 +286,9 @@ impl Nvme {
log::debug!("completion queue {}: {:X}, {}, (submission queue ids: {:?}", qid, data.physical(), data.len(), sq_ids);
}
for (qid, queue) in self.submission_queues.get_mut().unwrap().iter_mut() {
for (qid, (queue, cq_id)) in self.submission_queues.get_mut().unwrap().iter_mut() {
let data = &queue.get_mut().unwrap().data;
log::debug!("submission queue {}: {:X}, {}", qid, data.physical(), data.len());
log::debug!("submission queue {}: {:X}, {}, attached to CQID: {}", qid, data.physical(), data.len(), cq_id);
}
{
@@ -298,7 +296,7 @@ impl Nvme {
let submission_queues = self.submission_queues.get_mut().unwrap();
let completion_queues = self.completion_queues.get_mut().unwrap();
let asq = submission_queues.get_mut(&0).unwrap().get_mut().unwrap();
let asq = submission_queues.get_mut(&0).unwrap().0.get_mut().unwrap();
let (acq, _) = completion_queues.get_mut(&0).unwrap().get_mut().unwrap();
regs.aqa
.write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1));
@@ -402,55 +400,8 @@ impl Nvme {
self.set_vectors_masked(std::iter::once((vector, masked)))
}
pub fn submit_command_generic<'a, F: FnOnce(CmdId) -> NvmeCmd>(&'a self, sq_id: SqId, full_sq_handling: FullSqHandling, cmd_init: F) -> SubmissionBehavior<'a, F> {
let sqs_read_guard = self.submission_queues.read().unwrap();
let mut sq_lock = sqs_read_guard
.get(&sq_id)
.expect("nvmed: internal error: given SQ for SQ ID not there")
.lock()
.unwrap();
if sq_lock.is_full() {
match full_sq_handling {
FullSqHandling::ErrorDirectly => return SubmissionBehavior::Nonblocking(Err(cmd_init)),
FullSqHandling::Wait => return SubmissionBehavior::Future(self.wait_for_available_submission(sq_id, cmd_init)),
}
}
let cmd_id =
u16::try_from(sq_lock.tail).expect("nvmed: internal error: CQ has more than 2^16 entries");
let tail = sq_lock.submit_unchecked(cmd_init(cmd_id));
let tail = u16::try_from(tail).unwrap();
unsafe { self.submission_queue_tail(sq_id, tail) };
match full_sq_handling {
FullSqHandling::ErrorDirectly => SubmissionBehavior::Nonblocking(Ok(cmd_id)),
FullSqHandling::Wait => SubmissionBehavior::Future(self::cq_reactor::SubmissionFuture { state: self::cq_reactor::SubmissionFutureState::Ready(cmd_id) })
}
}
/// Try submitting a new entry to the specified submission queue, or return None if the queue
/// was full.
pub fn try_submit_command<F: FnOnce(CmdId) -> NvmeCmd>(
&self,
sq_id: SqId,
f: F,
) -> Result<CmdId, F> {
match self.submit_command_generic(sq_id, FullSqHandling::ErrorDirectly, f) {
SubmissionBehavior::Nonblocking(opt) => opt,
_ => unreachable!(),
}
}
pub async fn submit_command_async<F: FnOnce(CmdId) -> NvmeCmd>(&self, sq_id: SqId, f: F) -> CmdId {
match self.submit_command_generic(sq_id, FullSqHandling::Wait, f) {
SubmissionBehavior::Future(future) => future.await,
_ => unreachable!(),
}
}
pub async fn submit_admin_command<F: FnOnce(CmdId) -> NvmeCmd>(&self, f: F) -> CmdId {
self.submit_command_async(0, f).await
}
pub async fn admin_queue_completion(&self, cmd_id: CmdId) -> NvmeComp {
self.completion(0, cmd_id, 0).await
pub async fn submit_and_complete_admin_command<F: FnOnce(CmdId) -> NvmeCmd>(&self, cmd_init: F) -> NvmeComp {
self.submit_and_complete_command(0, cmd_init).await
}
pub async fn create_io_completion_queue(&self, io_cq_id: CqId, vector: Option<u16>) {
@@ -478,12 +429,11 @@ impl Nvme {
.checked_sub(1)
.expect("nvmed: internal error: CQID 0 for I/O CQ");
let cmd_id = self
.submit_admin_command(|cid| {
let comp = self
.submit_and_complete_admin_command(|cid| {
NvmeCmd::create_io_completion_queue(cid, io_cq_id, ptr, raw_len, vector)
})
.await;
let comp = self.admin_queue_completion(cmd_id).await;
if let Some(vector) = vector {
self.cqs_for_ivs
@@ -498,17 +448,17 @@ impl Nvme {
let (ptr, len) = {
let mut submission_queues_guard = self.submission_queues.write().unwrap();
let queue_guard = submission_queues_guard
let (queue_lock, _) = submission_queues_guard
.entry(io_sq_id)
.or_insert_with(|| {
Mutex::new(
(Mutex::new(
NvmeCmdQueue::new()
.expect("nvmed: failed to allocate I/O completion queue"),
)
})
.get_mut()
.unwrap();
(queue_guard.data.physical(), queue_guard.data.len())
), io_cq_id)
});
let queue = queue_lock.get_mut().unwrap();
(queue.data.physical(), queue.data.len())
};
let len =
@@ -517,17 +467,18 @@ impl Nvme {
.checked_sub(1)
.expect("nvmed: internal error: SQID 0 for I/O SQ");
let cmd_id = self
.submit_admin_command(|cid| {
let comp = self
.submit_and_complete_admin_command(|cid| {
NvmeCmd::create_io_submission_queue(cid, io_sq_id, ptr, raw_len, io_cq_id)
})
.await;
let comp = self.admin_queue_completion(cmd_id).await;
}
pub async fn init_with_queues(&self) -> BTreeMap<u32, NvmeNamespace> {
log::trace!("preinit");
let ((), nsids) =
futures::join!(self.identify_controller(), self.identify_namespace_list(0));
log::debug!("first commands");
let mut namespaces = BTreeMap::new();
@@ -563,15 +514,13 @@ impl Nvme {
(buffer_prp_guard[0], (buffer_prp_guard.physical() + 8) as u64)
};
let cmd_id = self.submit_command_async(1, |cid| {
let comp = self.submit_and_complete_command(1, |cid| {
if write {
NvmeCmd::io_write(cid, nsid, lba, blocks_1, ptr0, ptr1)
} else {
NvmeCmd::io_read(cid, nsid, lba, blocks_1, ptr0, ptr1)
}
}).await;
let comp = self.completion(1, cmd_id, 1).await;
// TODO: Handle errors
Ok(())
nvmed/src/nvme/queues.rs
+5 -4
View File
@@ -63,10 +63,11 @@ impl NvmeCompQueue {
/// Get a new completion queue entry, or return None if no entry is available yet.
pub(crate) fn complete(&mut self) -> Option<(u16, NvmeComp)> {
println!("PTR: {:p}", &*self as *const _);
let entry = unsafe { ptr::read_volatile(self.data.as_ptr().add(self.head as usize)) };
// println!("{:?}", entry);
if ((entry.status & 1) == 1) == self.phase {
self.head = (self.head + 1) % (self.data.len() as u16);
if ((dbg!(entry.status) & 1) == 1) == dbg!(self.phase) {
self.head = dbg!(self.head + 1) % dbg!(self.data.len() as u16);
if self.head == 0 {
self.phase = !self.phase;
}
@@ -108,13 +109,13 @@ impl NvmeCmdQueue {
self.head == self.tail
}
pub fn is_full(&self) -> bool {
self.head + 1 == self.tail
self.head == self.tail + 1
}
/// Add a new submission command entry to the queue. The caller must ensure that the queue have free
/// entries; this can be checked using `is_full`.
pub fn submit_unchecked(&mut self, entry: NvmeCmd) -> u16 {
self.data[self.tail as usize] = entry;
unsafe { ptr::write_volatile(&mut self.data[self.tail as usize] as *mut _, entry) }
self.tail = (self.tail + 1) % (self.data.len() as u16);
self.tail
}
xhcid/src/main.rs
+5 -3
View File
@@ -40,14 +40,15 @@ async fn handle_packet(hci: Arc<Xhci>, packet: Packet) -> Packet {
fn setup_logging() -> Option<&'static RedoxLogger> {
let mut logger = RedoxLogger::new()
.with_output(
/*.with_output(
OutputBuilder::stderr()
.with_filter(log::LevelFilter::Info) // limit global output to important info
.with_ansi_escape_codes()
.flush_on_newline(true)
.build()
);
)*/;
#[cfg(target_os = "redox")]
match OutputBuilder::in_redox_logging_scheme("usb", "host", "xhci.log") {
Ok(b) => logger = logger.with_output(
// TODO: Add a configuration file for this
@@ -58,6 +59,7 @@ fn setup_logging() -> Option<&'static RedoxLogger> {
Err(error) => eprintln!("Failed to create xhci.log: {}", error),
}
#[cfg(target_os = "redox")]
match OutputBuilder::in_redox_logging_scheme("usb", "host", "xhci.ansi.log") {
Ok(b) => logger = logger.with_output(
b.with_filter(log::LevelFilter::Trace)
@@ -221,7 +223,7 @@ fn main() {
(None, InterruptMethod::Polling)
};
std::thread::sleep(std::time::Duration::from_millis(300));
//std::thread::sleep(std::time::Duration::from_millis(300));
print!(
"{}",