vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

NVME WORKS WITH IRQs

Browse Source
This commit is contained in:
4lDO2
2020-05-01 14:14:47 +02:00
parent 64e9eea9b0
commit 1936f05030
6 changed files with 107 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nvmed/src/main.rs
+7 -7
View File
@@ -32,7 +32,7 @@ impl Bar {
pub fn allocate(bar: usize, bar_size: usize) -> Result<Self> {
Ok(Self {
ptr: NonNull::new(
syscall::physmap(bar, bar_size, PHYSMAP_NO_CACHE | PHYSMAP_WRITE)? as *mut u8,
unsafe { syscall::physmap(bar, bar_size, PHYSMAP_NO_CACHE | PHYSMAP_WRITE)? as *mut u8 },
)
.expect("Mapping a BAR resulted in a nullptr"),
physical: bar,
@@ -43,7 +43,7 @@ impl Bar {
impl Drop for Bar {
fn drop(&mut self) {
let _ = syscall::physunmap(self.physical);
let _ = unsafe { syscall::physunmap(self.physical) };
}
}
@@ -82,7 +82,7 @@ fn get_int_method(
bir: u8,
) -> Result<NonNull<u8>> {
let bir = usize::from(bir);
let bar_guard = allocated_bars.0[bir].lock().unwrap();
let mut bar_guard = allocated_bars.0[bir].lock().unwrap();
match &mut *bar_guard {
&mut Some(ref bar) => Ok(bar.ptr),
bar_to_set @ &mut None => {
@@ -119,7 +119,7 @@ fn get_int_method(
// Mask all interrupts in case some earlier driver/os already unmasked them (according to
// the PCI Local Bus spec 3.0, they are masked after system reset).
for table_entry in table_entries {
for table_entry in table_entries.iter_mut() {
table_entry.mask();
}
@@ -284,10 +284,10 @@ fn main() {
.expect("nvmed: failed to find a suitable interrupt method");
let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender)
.expect("nvmed: failed to allocate driver data");
let nvme = Arc::new(nvme);
unsafe { nvme.init() }
nvme::cq_reactor::start_cq_reactor_thread(nvme, interrupt_sources, reactor_receiver);
let namespaces = unsafe { futures::executor::block_on(nvme.init_with_queues()) };
let nvme = Arc::new(nvme);
nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver);
let namespaces = futures::executor::block_on(nvme.init_with_queues());
let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces);
let mut todo = Vec::new();
nvmed/src/nvme/cq_reactor.rs
+41 -35
View File
@@ -62,7 +62,7 @@ struct CqReactor {
event_queue: File,
}
impl CqReactor {
fn create_event_queue(int_sources: &InterruptSources) -> Result<File> {
fn create_event_queue(int_sources: &mut InterruptSources) -> Result<File> {
use syscall::flag::*;
let fd = syscall::open("event:", O_CLOEXEC | O_RDWR)?;
let mut file = unsafe { File::from_raw_fd(fd as RawFd) };
@@ -84,11 +84,11 @@ impl CqReactor {
}
fn new(
nvme: Arc<Nvme>,
int_sources: InterruptSources,
mut int_sources: InterruptSources,
receiver: Receiver<NotifReq>,
) -> Result<Self> {
Ok(Self {
event_queue: Self::create_event_queue(&int_sources)?,
event_queue: Self::create_event_queue(&mut int_sources)?,
int_sources,
nvme,
pending_reqs: Vec::new(),
@@ -121,8 +121,10 @@ impl CqReactor {
let mut entry_count = 0;
for cq_id in ivs_read_guard.get(&iv)?.iter().copied() {
let completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap();
let cq_ids = ivs_read_guard.get(&iv)?;
for cq_id in cq_ids.iter().copied() {
let mut completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap();
let &mut (ref mut completion_queue, _) = &mut *completion_queue_guard;
let (head, entry) = match completion_queue.complete() {
@@ -130,11 +132,11 @@ impl CqReactor {
None => continue,
};
self.nvme.completion_queue_head(cq_id, head);
unsafe { self.nvme.completion_queue_head(cq_id, head) };
self.nvme.submission_queues.read().unwrap().get(&entry.sq_id).expect("nvmed: internal error: queue returned from controller doesn't exist").lock().unwrap().head = entry.sq_head;
self.nvme.submission_queues.read().unwrap().get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist").lock().unwrap().head = entry.sq_head;
self.try_notify_futures(cq_id, &entry);
Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry);
entry_count += 1;
}
@@ -142,12 +144,12 @@ impl CqReactor {
Some(())
}
fn finish_pending_completion(&mut self, req_cq_id: CqId, cq_id: CqId, sq_id: SqId, cmd_id: CmdId, entry: &NvmeComp, i: usize) -> bool {
fn finish_pending_completion(pending_reqs: &mut Vec<PendingReq>, req_cq_id: CqId, cq_id: CqId, sq_id: SqId, cmd_id: CmdId, entry: &NvmeComp, i: usize) -> bool {
if req_cq_id == cq_id
&& sq_id == entry.sq_id
&& cmd_id == entry.cid
{
let (waker, message) = match self.pending_reqs.remove(i) {
let (waker, message) = match pending_reqs.remove(i) {
PendingReq::PendingCompletion { waker, message, .. } => (waker, message),
_ => unreachable!(),
};
@@ -160,9 +162,9 @@ impl CqReactor {
false
}
}
fn finish_pending_avail_submission(&mut self, sq_id: SqId, entry: &NvmeComp, i: usize) -> bool {
fn finish_pending_avail_submission(pending_reqs: &mut Vec<PendingReq>, sq_id: SqId, entry: &NvmeComp, i: usize) -> bool {
if sq_id == entry.sq_id {
let waker = match self.pending_reqs.remove(i) {
let waker = match pending_reqs.remove(i) {
PendingReq::PendingAvailSubmission { waker, .. } => waker,
_ => unreachable!(),
};
@@ -173,19 +175,19 @@ impl CqReactor {
false
}
}
fn try_notify_futures(&mut self, cq_id: CqId, entry: &NvmeComp) -> Option<()> {
fn try_notify_futures(pending_reqs: &mut Vec<PendingReq>, cq_id: CqId, entry: &NvmeComp) -> Option<()> {
let mut i = 0usize;
let mut futures_notified = 0;
while i < self.pending_reqs.len() {
match &self.pending_reqs[i] {
&PendingReq::PendingCompletion { cq_id: req_cq_id, sq_id, cmd_id, .. } => if self.finish_pending_completion(req_cq_id, cq_id, sq_id, cmd_id, entry, i) {
while i < pending_reqs.len() {
match &pending_reqs[i] {
&PendingReq::PendingCompletion { cq_id: req_cq_id, sq_id, cmd_id, .. } => if Self::finish_pending_completion(pending_reqs, req_cq_id, cq_id, sq_id, cmd_id, entry, i) {
futures_notified += 1;
} else {
i += 1;
}
&PendingReq::PendingAvailSubmission { sq_id, .. } => if self.finish_pending_avail_submission(sq_id, entry, i) {
&PendingReq::PendingAvailSubmission { sq_id, .. } => if Self::finish_pending_avail_submission(pending_reqs, sq_id, entry, i) {
futures_notified += 1;
} else {
i += 1;
@@ -250,7 +252,7 @@ pub fn start_cq_reactor_thread(
})
}
struct CompletionMessage {
pub struct CompletionMessage {
cq_entry: NvmeComp,
}
@@ -264,6 +266,7 @@ enum CompletionFutureState {
message: Arc<Mutex<Option<CompletionMessage>>>,
},
Finished,
Placeholder,
}
pub struct CompletionFuture {
state: CompletionFutureState,
@@ -278,8 +281,8 @@ impl Future for CompletionFuture {
fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll<Self::Output> {
let this = &mut self.get_mut().state;
match this {
&mut CompletionFutureState::Pending {
match mem::replace(this, CompletionFutureState::Placeholder) {
CompletionFutureState::Pending {
message,
cq_id,
cmd_id,
@@ -288,21 +291,22 @@ impl Future for CompletionFuture {
} => {
if let Some(value) = message.lock().unwrap().take() {
*this = CompletionFutureState::Finished;
task::Poll::Ready(value.cq_entry)
} else {
sender.send(NotifReq::RequestCompletion {
cq_id,
sq_id,
cmd_id,
waker: context.waker().clone(),
message: Arc::clone(&message),
});
task::Poll::Pending
return task::Poll::Ready(value.cq_entry);
}
sender.send(NotifReq::RequestCompletion {
cq_id,
sq_id,
cmd_id,
waker: context.waker().clone(),
message: Arc::clone(&message),
}).expect("reactor dead");
*this = CompletionFutureState::Pending { message, cq_id, cmd_id, sq_id, sender };
task::Poll::Pending
}
&mut CompletionFutureState::Finished => {
CompletionFutureState::Finished => {
panic!("calling poll() on an already finished CompletionFuture")
}
CompletionFutureState::Placeholder => unreachable!(),
}
}
}
@@ -345,6 +349,7 @@ pub(crate) enum SubmissionFutureState<'a, F> {
// returned when there was an available submission entry from the beginning
Ready(CmdId),
Finished,
Placeholder,
}
/// A future representing a submission queue eventually becoming non-full. In most cases this
@@ -361,8 +366,8 @@ impl<F: FnOnce(CmdId) -> NvmeCmd> Future for SubmissionFuture<'_, F> {
fn poll(self: Pin<&mut Self>, context: &mut task::Context<'_>) -> task::Poll<Self::Output> {
let state = &mut self.get_mut().state;
match state {
&mut SubmissionFutureState::Pending { sq_id, cmd_init, nvme } => match nvme.try_submit_command(sq_id, cmd_init) {
match mem::replace(state, SubmissionFutureState::Placeholder) {
SubmissionFutureState::Pending { sq_id, cmd_init, nvme } => match nvme.try_submit_command(sq_id, cmd_init) {
Ok(cmd_id) => {
*state = SubmissionFutureState::Finished;
task::Poll::Ready(cmd_id)
@@ -373,11 +378,12 @@ impl<F: FnOnce(CmdId) -> NvmeCmd> Future for SubmissionFuture<'_, F> {
task::Poll::Pending
}
}
&mut SubmissionFutureState::Ready(value) => {
SubmissionFutureState::Ready(value) => {
*state = SubmissionFutureState::Finished;
task::Poll::Ready(value)
}
&mut SubmissionFutureState::Finished => panic!("calling poll() on an already finished SubmissionFuture"),
SubmissionFutureState::Finished => panic!("calling poll() on an already finished SubmissionFuture"),
SubmissionFutureState::Placeholder => unreachable!(),
}
}
}
nvmed/src/nvme/identify.rs
+3 -2
View File
@@ -78,8 +78,9 @@ impl Nvme {
// println!(" - Dumping identify namespace");
let size = *(data.as_ptr().offset(0) as *const u64);
let capacity = *(data.as_ptr().offset(8) as *const u64);
// TODO: Use struct
let size = unsafe { *(data.as_ptr().offset(0) as *const u64) };
let capacity = unsafe { *(data.as_ptr().offset(8) as *const u64) };
println!(" - ID: {} Size: {} Capacity: {}", nsid, size, capacity);
//TODO: Read block size
nvmed/src/nvme/mod.rs
+47 -49
View File
@@ -55,9 +55,9 @@ impl InterruptSources {
}
fn size_hint(&self) -> (usize, Option<usize>) {
match self {
&Self::Msi(mut iter) => iter.size_hint(),
&Self::MsiX(mut iter) => iter.size_hint(),
&Self::Intx(mut iter) => iter.size_hint(),
&Self::Msi(ref iter) => iter.size_hint(),
&Self::MsiX(ref iter) => iter.size_hint(),
&Self::Intx(ref iter) => iter.size_hint(),
}
}
}
@@ -224,10 +224,13 @@ impl Nvme {
/// # Locking
/// Locks `regs`.
unsafe fn doorbell_write(&self, index: usize, value: u32) {
let mut regs_guard = self.regs.write().unwrap();
use std::ops::DerefMut;
let dstrd = ((regs_guard.cap.read() >> 32) & 0b1111) as usize;
let addr = ((*regs_guard) as *mut NvmeRegs as usize) + 0x1000 + index * (4 << dstrd);
let mut regs_guard = self.regs.write().unwrap();
let mut regs: &mut NvmeRegs = regs_guard.deref_mut();
let dstrd = ((regs.cap.read() >> 32) & 0b1111) as usize;
let addr = (regs as *mut NvmeRegs as usize) + 0x1000 + index * (4 << dstrd);
(&mut *(addr as *mut Mmio<u32>)).write(value);
}
@@ -260,7 +263,7 @@ impl Nvme {
let csts = self.regs.get_mut().unwrap().csts.read();
// println!("CSTS: {:X}", csts);
if csts & 1 == 1 {
unsafe { std::arch::x86_64::_mm_pause() }
std::arch::x86_64::_mm_pause();
} else {
break;
}
@@ -276,13 +279,13 @@ impl Nvme {
}
}
for (qid, queue) in self.completion_queues.get_mut().unwrap().iter() {
for (qid, queue) in self.completion_queues.get_mut().unwrap().iter_mut() {
let &(ref cq, ref sq_ids) = &*queue.get_mut().unwrap();
let data = &cq.data;
// println!(" - completion queue {}: {:X}, {}", qid, data.physical(), data.len());
}
for (qid, queue) in self.submission_queues.get_mut().unwrap().iter() {
for (qid, queue) in self.submission_queues.get_mut().unwrap().iter_mut() {
let data = &queue.get_mut().unwrap().data;
// println!(" - submission queue {}: {:X}, {}", qid, data.physical(), data.len());
}
@@ -290,10 +293,10 @@ impl Nvme {
{
let regs = self.regs.get_mut().unwrap();
let submission_queues = self.submission_queues.get_mut().unwrap();
let completion_queues = self.submission_queues.get_mut().unwrap();
let completion_queues = self.completion_queues.get_mut().unwrap();
let asq = submission_queues.get(&0).unwrap().get_mut().unwrap();
let acq = completion_queues.get(&0).unwrap().get_mut().unwrap();
let asq = submission_queues.get_mut(&0).unwrap().get_mut().unwrap();
let (acq, _) = completion_queues.get_mut(&0).unwrap().get_mut().unwrap();
regs.aqa
.write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1));
regs.asq.write(asq.data.physical() as u64);
@@ -314,7 +317,7 @@ impl Nvme {
let csts = self.regs.get_mut().unwrap().csts.read();
// println!("CSTS: {:X}", csts);
if csts & 1 == 0 {
unsafe { std::arch::x86_64::_mm_pause() }
std::arch::x86_64::_mm_pause();
} else {
break;
}
@@ -326,7 +329,7 @@ impl Nvme {
/// # Panics
/// Will panic if the same vector is called twice with different mask flags.
pub fn set_vectors_masked(&self, vectors: impl IntoIterator<Item = (u16, bool)>) {
let interrupt_method_guard = self.interrupt_method.lock().unwrap();
let mut interrupt_method_guard = self.interrupt_method.lock().unwrap();
match &mut *interrupt_method_guard {
&mut InterruptMethod::Intx => {
@@ -398,7 +401,7 @@ impl Nvme {
pub fn submit_command_generic<'a, F: FnOnce(CmdId) -> NvmeCmd>(&'a self, sq_id: SqId, full_sq_handling: FullSqHandling, cmd_init: F) -> SubmissionBehavior<'a, F> {
let sqs_read_guard = self.submission_queues.read().unwrap();
let sq_lock = sqs_read_guard
let mut sq_lock = sqs_read_guard
.get(&sq_id)
.expect("nvmed: internal error: given SQ for SQ ID not there")
.lock()
@@ -413,7 +416,8 @@ impl Nvme {
u16::try_from(sq_lock.tail).expect("nvmed: internal error: CQ has more than 2^16 entries");
let tail = sq_lock.submit_unchecked(cmd_init(cmd_id));
let tail = u16::try_from(tail).unwrap();
self.submission_queue_tail(sq_id, tail);
unsafe { self.submission_queue_tail(sq_id, tail) };
match full_sq_handling {
FullSqHandling::ErrorDirectly => SubmissionBehavior::Nonblocking(Ok(cmd_id)),
@@ -535,8 +539,8 @@ impl Nvme {
namespaces
}
unsafe fn namespace_rw(
&mut self,
async fn namespace_rw(
&self,
nsid: u32,
lba: u64,
blocks_1: u16,
@@ -545,59 +549,51 @@ impl Nvme {
//TODO: Get real block size
let block_size = 512;
let buffer_prp_guard = self.buffer_prp.lock().unwrap();
let bytes = ((blocks_1 as u64) + 1) * block_size;
let (ptr0, ptr1) = if bytes <= 4096 {
(self.buffer_prp[0], 0)
(buffer_prp_guard[0], 0)
} else if bytes <= 8192 {
(self.buffer_prp[0], self.buffer_prp[1])
(buffer_prp_guard[0], buffer_prp_guard[1])
} else {
(self.buffer_prp[0], (self.buffer_prp.physical() + 8) as u64)
(buffer_prp_guard[0], (buffer_prp_guard.physical() + 8) as u64)
};
{
let qid = 1;
let queue = &mut self.submission_queues[qid];
let cid = queue.i as u16;
let entry = if write {
let cmd_id = self.submit_command_async(1, |cid| {
if write {
NvmeCmd::io_write(cid, nsid, lba, blocks_1, ptr0, ptr1)
} else {
NvmeCmd::io_read(cid, nsid, lba, blocks_1, ptr0, ptr1)
};
let tail = queue.submit(entry);
self.submission_queue_tail(qid as u16, tail as u16);
}
}
}).await;
{
let qid = 1;
let queue = &mut self.completion_queues[qid];
let (head, entry) = queue.complete_spin();
//TODO: Handle errors
self.completion_queue_head(qid as u16, head as u16);
}
let comp = self.completion(1, cmd_id, 1).await;
// TODO: Handle errors
Ok(())
}
pub unsafe fn namespace_read(
&mut self,
pub async fn namespace_read(
&self,
nsid: u32,
mut lba: u64,
buf: &mut [u8],
) -> Result<Option<usize>> {
//TODO: Use interrupts
//TODO: Get real block size
let block_size = 512;
for chunk in buf.chunks_mut(self.buffer.len()) {
let mut buffer_guard = self.buffer.lock().unwrap();
for chunk in buf.chunks_mut(buffer_guard.len()) {
let blocks = (chunk.len() + block_size - 1) / block_size;
assert!(blocks > 0);
assert!(blocks <= 0x1_0000);
self.namespace_rw(nsid, lba, (blocks - 1) as u16, false)?;
self.namespace_rw(nsid, lba, (blocks - 1) as u16, false).await?;
chunk.copy_from_slice(&self.buffer[..chunk.len()]);
chunk.copy_from_slice(&buffer_guard[..chunk.len()]);
lba += blocks as u64;
}
@@ -605,8 +601,8 @@ impl Nvme {
Ok(Some(buf.len()))
}
pub unsafe fn namespace_write(
&mut self,
pub async fn namespace_write(
&self,
nsid: u32,
mut lba: u64,
buf: &[u8],
@@ -616,15 +612,17 @@ impl Nvme {
//TODO: Get real block size
let block_size = 512;
for chunk in buf.chunks(self.buffer.len()) {
let mut buffer_guard = self.buffer.lock().unwrap();
for chunk in buf.chunks(buffer_guard.len()) {
let blocks = (chunk.len() + block_size - 1) / block_size;
assert!(blocks > 0);
assert!(blocks <= 0x1_0000);
self.buffer[..chunk.len()].copy_from_slice(chunk);
buffer_guard[..chunk.len()].copy_from_slice(chunk);
self.namespace_rw(nsid, lba, (blocks - 1) as u16, true)?;
self.namespace_rw(nsid, lba, (blocks - 1) as u16, true).await?;
lba += blocks as u64;
}
nvmed/src/nvme/queues.rs
+2 -2
View File
@@ -55,7 +55,7 @@ pub struct NvmeCompQueue {
impl NvmeCompQueue {
pub fn new() -> Result<Self> {
Ok(Self {
data: Dma::zeroed_unsized(256)?,
data: unsafe { Dma::zeroed_unsized(256)? },
head: 0,
phase: true,
})
@@ -98,7 +98,7 @@ pub struct NvmeCmdQueue {
impl NvmeCmdQueue {
pub fn new() -> Result<Self> {
Ok(Self {
data: Dma::zeroed_unsized(64)?,
data: unsafe { Dma::zeroed_unsized(64)? },
tail: 0,
head: 0,
})
nvmed/src/scheme.rs
+7 -40
View File
@@ -82,10 +82,8 @@ impl DiskWrapper {
return Err(io::Error::from_raw_os_error(syscall::EOVERFLOW));
}
loop {
match unsafe {
nvme.namespace_read(disk.id, block, block_bytes)
.map_err(|err| io::Error::from_raw_os_error(err.errno))?
} {
match futures::executor::block_on(nvme.namespace_read(disk.id, block, block_bytes))
.map_err(|err| io::Error::from_raw_os_error(err.errno))? {
Some(bytes) => {
assert_eq!(bytes, block_bytes.len());
assert_eq!(bytes, blksize as usize);
@@ -161,29 +159,6 @@ impl DiskScheme {
}
}
impl DiskScheme {
pub fn irq(&mut self) -> bool {
let mut found_completion = false;
let nvme = &mut self.nvme;
let completion_queues = nvme.completion_queues.read().unwrap();
for qid in 0..completion_queues.len() {
let queue = completion_queues[qid].lock().unwrap();
while let Some((head, entry)) = queue.complete() {
found_completion = true;
println!("nvmed: Unhandled completion {:?}", entry);
//TODO: Handle errors
unsafe {
nvme.completion_queue_head(qid as u16, head as u16);
}
}
}
found_completion
}
}
impl SchemeBlockMut for DiskScheme {
fn open(&mut self, path: &[u8], flags: usize, uid: u32, _gid: u32) -> Result<Option<usize>> {
if uid == 0 {
@@ -372,10 +347,7 @@ impl SchemeBlockMut for DiskScheme {
Handle::Disk(number, ref mut size) => {
let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?;
let block_size = disk.as_ref().block_size;
if let Some(count) = unsafe {
self.nvme
.namespace_read(disk.as_ref().id, (*size as u64) / block_size, buf)?
} {
if let Some(count) = futures::executor::block_on(self.nvme.namespace_read(disk.as_ref().id, (*size as u64) / block_size, buf))? {
*size += count;
Ok(Some(count))
} else {
@@ -400,9 +372,7 @@ impl SchemeBlockMut for DiskScheme {
let abs_block = part.start_lba + rel_block;
if let Some(count) =
unsafe { self.nvme.namespace_read(disk.as_ref().id, abs_block, buf)? }
{
if let Some(count) = futures::executor::block_on(self.nvme.namespace_read(disk.as_ref().id, abs_block, buf))? {
*offset += count;
Ok(Some(count))
} else {
@@ -419,8 +389,8 @@ impl SchemeBlockMut for DiskScheme {
let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?;
let block_size = disk.as_ref().block_size;
if let Some(count) = unsafe {
self.nvme
.namespace_write(disk.as_ref().id, (*size as u64) / block_size, buf)?
futures::executor::block_on(self.nvme
.namespace_write(disk.as_ref().id, (*size as u64) / block_size, buf))?
} {
*size += count;
Ok(Some(count))
@@ -446,10 +416,7 @@ impl SchemeBlockMut for DiskScheme {
let abs_block = part.start_lba + rel_block;
if let Some(count) = unsafe {
self.nvme
.namespace_write(disk.as_ref().id, abs_block, buf)?
} {
if let Some(count) = futures::executor::block_on(self.nvme.namespace_write(disk.as_ref().id, abs_block, buf))? {
*offset += count;
Ok(Some(count))
} else {