From 5e4b53fe929671fdb9cd4ce43f83bac44e446b5d Mon Sep 17 00:00:00 2001 From: Simon Ellmann Date: Tue, 30 Jul 2019 13:20:30 +0200 Subject: [PATCH] Speed up e1000 driver --- e1000d/src/device.rs | 214 +++++++++++++++++++++++++------------------ e1000d/src/main.rs | 149 +++++++++++++++++++----------- 2 files changed, 218 insertions(+), 145 deletions(-) diff --git a/e1000d/src/device.rs b/e1000d/src/device.rs index 72e066b063..bded679c9e 100644 --- a/e1000d/src/device.rs +++ b/e1000d/src/device.rs @@ -1,8 +1,8 @@ -use std::{cmp, mem, ptr, slice, thread}; use std::collections::BTreeMap; +use std::{cmp, mem, ptr, slice, thread}; use netutils::setcfg; -use syscall::error::{Error, EACCES, EBADF, EINVAL, EWOULDBLOCK, Result}; +use syscall::error::{Error, Result, EACCES, EBADF, EINVAL, EWOULDBLOCK}; use syscall::flag::O_NONBLOCK; use syscall::io::Dma; use syscall::scheme::SchemeBlockMut; @@ -98,12 +98,20 @@ pub struct Intel8254x { base: usize, receive_buffer: [Dma<[u8; 16384]>; 16], receive_ring: Dma<[Rd; 16]>, + receive_index: usize, transmit_buffer: [Dma<[u8; 16384]>; 16], transmit_ring: Dma<[Td; 16]>, + transmit_ring_free: usize, + transmit_index: usize, + transmit_clean_index: usize, next_id: usize, pub handles: BTreeMap, } +fn wrap_ring(index: usize, ring_size: usize) -> usize { + (index + 1) & (ring_size - 1) +} + impl SchemeBlockMut for Intel8254x { fn open(&mut self, _path: &[u8], flags: usize, uid: u32, _gid: u32) -> Result> { if uid == 0 { @@ -116,7 +124,7 @@ impl SchemeBlockMut for Intel8254x { } fn dup(&mut self, id: usize, buf: &[u8]) -> Result> { - if ! buf.is_empty() { + if !buf.is_empty() { return Err(Error::new(EINVAL)); } @@ -132,31 +140,20 @@ impl SchemeBlockMut for Intel8254x { fn read(&mut self, id: usize, buf: &mut [u8]) -> Result> { let flags = self.handles.get(&id).ok_or(Error::new(EBADF))?; - let head = unsafe { self.read_reg(RDH) }; - let mut tail = unsafe { self.read_reg(RDT) }; + let desc = unsafe { &mut *(self.receive_ring.as_ptr().add(self.receive_index) as *mut Rd) }; - tail += 1; - if tail >= self.receive_ring.len() as u32 { - tail = 0; - } + if desc.status & RD_DD == RD_DD { + desc.status = 0; - if tail != head { - let rd = unsafe { &mut * (self.receive_ring.as_ptr().offset(tail as isize) as *mut Rd) }; - if rd.status & RD_DD == RD_DD { - rd.status = 0; + let data = &self.receive_buffer[self.receive_index][..desc.length as usize]; - let data = &self.receive_buffer[tail as usize][.. rd.length as usize]; + let i = cmp::min(buf.len(), data.len()); + buf[..i].copy_from_slice(&data[..i]); - let mut i = 0; - while i < buf.len() && i < data.len() { - buf[i] = data[i]; - i += 1; - } + unsafe { self.write_reg(RDT, self.receive_index as u32) }; + self.receive_index = wrap_ring(self.receive_index, self.receive_ring.len()); - unsafe { self.write_reg(RDT, tail) }; - - return Ok(Some(i)); - } + return Ok(Some(i)); } if flags & O_NONBLOCK == O_NONBLOCK { @@ -169,44 +166,56 @@ impl SchemeBlockMut for Intel8254x { fn write(&mut self, id: usize, buf: &[u8]) -> Result> { let _flags = self.handles.get(&id).ok_or(Error::new(EBADF))?; - loop { - let head = unsafe { self.read_reg(TDH) }; - let mut tail = unsafe { self.read_reg(TDT) }; - let old_tail = tail; + if self.transmit_ring_free == 0 { + loop { + let desc = unsafe { + &*(self.transmit_ring.as_ptr().add(self.transmit_clean_index) as *const Td) + }; - tail += 1; - if tail >= self.transmit_ring.len() as u32 { - tail = 0; - } - - if tail != head { - let td = unsafe { &mut * (self.transmit_ring.as_ptr().offset(old_tail as isize) as *mut Td) }; - - td.cso = 0; - td.command = TD_CMD_EOP | TD_CMD_IFCS | TD_CMD_RS; - td.status = 0; - td.css = 0; - td.special = 0; - - td.length = (cmp::min(buf.len(), 0x3FFF)) as u16; - - let data = unsafe { slice::from_raw_parts_mut(self.transmit_buffer[old_tail as usize].as_ptr() as *mut u8, td.length as usize) }; - - let mut i = 0; - while i < buf.len() && i < data.len() { - data[i] = buf[i]; - i += 1; + if desc.status != 0 { + self.transmit_clean_index = + wrap_ring(self.transmit_clean_index, self.transmit_ring.len()); + self.transmit_ring_free += 1; + } else if self.transmit_ring_free > 0 { + break; } - unsafe { self.write_reg(TDT, tail) }; - - while td.status == 0 { - thread::yield_now(); + if self.transmit_ring_free >= self.transmit_ring.len() { + break; } - - return Ok(Some(i)); } } + + let desc = + unsafe { &mut *(self.transmit_ring.as_ptr().add(self.transmit_index) as *mut Td) }; + + let data = unsafe { + slice::from_raw_parts_mut( + self.transmit_buffer[self.transmit_index].as_ptr() as *mut u8, + cmp::min(buf.len(), self.transmit_buffer[self.transmit_index].len()) as usize, + ) + }; + + let i = cmp::min(buf.len(), data.len()); + data[..i].copy_from_slice(&buf[..i]); + + desc.cso = 0; + desc.command = TD_CMD_EOP | TD_CMD_IFCS | TD_CMD_RS; + desc.status = 0; + desc.css = 0; + desc.special = 0; + + desc.length = (cmp::min( + buf.len(), + self.transmit_buffer[self.transmit_index].len() - 1, + )) as u16; + + self.transmit_index = wrap_ring(self.transmit_index, self.transmit_ring.len()); + self.transmit_ring_free -= 1; + + unsafe { self.write_reg(TDT, self.transmit_index as u32) }; + + Ok(Some(i)) } fn fevent(&mut self, id: usize, _flags: usize) -> Result> { @@ -239,20 +248,29 @@ impl SchemeBlockMut for Intel8254x { impl Intel8254x { pub unsafe fn new(base: usize) -> Result { + #[rustfmt::skip] let mut module = Intel8254x { base: base, - receive_buffer: [Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, - Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, - Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, - Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?], + receive_buffer: [ + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + ], receive_ring: Dma::zeroed()?, - transmit_buffer: [Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, - Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, - Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, - Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?], + transmit_buffer: [ + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, Dma::zeroed()?, + ], + receive_index: 0, transmit_ring: Dma::zeroed()?, + transmit_ring_free: 16, + transmit_index: 0, + transmit_clean_index: 0, next_id: 0, - handles: BTreeMap::new() + handles: BTreeMap::new(), }; module.init(); @@ -266,19 +284,10 @@ impl Intel8254x { } pub fn next_read(&self) -> usize { - let head = unsafe { self.read_reg(RDH) }; - let mut tail = unsafe { self.read_reg(RDT) }; + let desc = unsafe { &*(self.receive_ring.as_ptr().add(self.receive_index) as *const Rd) }; - tail += 1; - if tail >= self.receive_ring.len() as u32 { - tail = 0; - } - - if tail != head { - let rd = unsafe { &* (self.receive_ring.as_ptr().offset(tail as isize) as *const Rd) }; - if rd.status & RD_DD == RD_DD { - return rd.length as usize; - } + if desc.status & RD_DD == RD_DD { + return desc.length as usize; } 0 @@ -324,14 +333,28 @@ impl Intel8254x { let mac_low = self.read_reg(RAL0); let mac_high = self.read_reg(RAH0); - let mac = [mac_low as u8, - (mac_low >> 8) as u8, - (mac_low >> 16) as u8, - (mac_low >> 24) as u8, - mac_high as u8, - (mac_high >> 8) as u8]; - print!("{}", format!(" - MAC: {:>02X}:{:>02X}:{:>02X}:{:>02X}:{:>02X}:{:>02X}\n", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5])); - let _ = setcfg("mac", &format!("{:>02X}-{:>02X}-{:>02X}-{:>02X}-{:>02X}-{:>02X}\n", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5])); + let mac = [ + mac_low as u8, + (mac_low >> 8) as u8, + (mac_low >> 16) as u8, + (mac_low >> 24) as u8, + mac_high as u8, + (mac_high >> 8) as u8, + ]; + print!( + "{}", + format!( + " - MAC: {:>02X}:{:>02X}:{:>02X}:{:>02X}:{:>02X}:{:>02X}\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] + ) + ); + let _ = setcfg( + "mac", + &format!( + "{:>02X}-{:>02X}-{:>02X}-{:>02X}-{:>02X}-{:>02X}\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] + ), + ); // // MTA => 0; @@ -344,7 +367,10 @@ impl Intel8254x { self.write_reg(RDBAH, (self.receive_ring.physical() >> 32) as u32); self.write_reg(RDBAL, self.receive_ring.physical() as u32); - self.write_reg(RDLEN, (self.receive_ring.len() * mem::size_of::()) as u32); + self.write_reg( + RDLEN, + (self.receive_ring.len() * mem::size_of::()) as u32, + ); self.write_reg(RDH, 0); self.write_reg(RDT, self.receive_ring.len() as u32 - 1); @@ -355,7 +381,10 @@ impl Intel8254x { self.write_reg(TDBAH, (self.transmit_ring.physical() >> 32) as u32); self.write_reg(TDBAL, self.transmit_ring.physical() as u32); - self.write_reg(TDLEN, (self.transmit_ring.len() * mem::size_of::()) as u32); + self.write_reg( + TDLEN, + (self.transmit_ring.len() * mem::size_of::()) as u32, + ); self.write_reg(TDH, 0); self.write_reg(TDT, 0); @@ -384,10 +413,13 @@ impl Intel8254x { while self.read_reg(STATUS) & 2 != 2 { print!(" - Waiting for link up: {:X}\n", self.read_reg(STATUS)); } - print!(" - Link is up with speed {}\n", match (self.read_reg(STATUS) >> 6) & 0b11 { - 0b00 => "10 Mb/s", - 0b01 => "100 Mb/s", - _ => "1000 Mb/s", - }); + print!( + " - Link is up with speed {}\n", + match (self.read_reg(STATUS) >> 6) & 0b11 { + 0b00 => "10 Mb/s", + 0b01 => "100 Mb/s", + _ => "1000 Mb/s", + } + ); } } diff --git a/e1000d/src/main.rs b/e1000d/src/main.rs index 1ca21f1969..fb87503ef4 100644 --- a/e1000d/src/main.rs +++ b/e1000d/src/main.rs @@ -7,17 +7,20 @@ extern crate syscall; use std::cell::RefCell; use std::env; use std::fs::File; -use std::io::{ErrorKind, Read, Write, Result}; +use std::io::{ErrorKind, Read, Result, Write}; use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::sync::Arc; use event::EventQueue; use syscall::{Packet, SchemeBlockMut, PHYSMAP_NO_CACHE, PHYSMAP_WRITE}; - pub mod device; -fn handle_update(socket: &mut File, device: &mut device::Intel8254x, todo: &mut Vec) -> Result { +fn handle_update( + socket: &mut File, + device: &mut device::Intel8254x, + todo: &mut Vec, +) -> Result { // Handle any blocked packets let mut i = 0; while i < todo.len() { @@ -36,10 +39,12 @@ fn handle_update(socket: &mut File, device: &mut device::Intel8254x, todo: &mut match socket.read(&mut packet) { Ok(0) => return Ok(true), Ok(_) => (), - Err(err) => if err.kind() == ErrorKind::WouldBlock { - break; - } else { - return Err(err); + Err(err) => { + if err.kind() == ErrorKind::WouldBlock { + break; + } else { + return Err(err); + } } } @@ -66,20 +71,36 @@ fn main() { let irq_str = args.next().expect("e1000d: no irq provided"); let irq = irq_str.parse::().expect("e1000d: failed to parse irq"); - print!("{}", format!(" + E1000 {} on: {:X} IRQ: {}\n", name, bar, irq)); + print!( + "{}", + format!(" + E1000 {} on: {:X} IRQ: {}\n", name, bar, irq) + ); // Daemonize if unsafe { syscall::clone(0).unwrap() } == 0 { - let socket_fd = syscall::open(":network", syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK).expect("e1000d: failed to create network scheme"); - let socket = Arc::new(RefCell::new(unsafe { File::from_raw_fd(socket_fd as RawFd) })); + let socket_fd = syscall::open( + ":network", + syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK, + ) + .expect("e1000d: failed to create network scheme"); + let socket = Arc::new(RefCell::new(unsafe { + File::from_raw_fd(socket_fd as RawFd) + })); - let mut irq_file = File::open(format!("irq:{}", irq)).expect("e1000d: failed to open IRQ file"); + let mut irq_file = + File::open(format!("irq:{}", irq)).expect("e1000d: failed to open IRQ file"); - let address = unsafe { syscall::physmap(bar, 128*1024, PHYSMAP_WRITE | PHYSMAP_NO_CACHE).expect("e1000d: failed to map address") }; + let address = unsafe { + syscall::physmap(bar, 128 * 1024, PHYSMAP_WRITE | PHYSMAP_NO_CACHE) + .expect("e1000d: failed to map address") + }; { - let device = Arc::new(RefCell::new(unsafe { device::Intel8254x::new(address).expect("e1000d: failed to allocate device") })); + let device = Arc::new(RefCell::new(unsafe { + device::Intel8254x::new(address).expect("e1000d: failed to allocate device") + })); - let mut event_queue = EventQueue::::new().expect("e1000d: failed to create event queue"); + let mut event_queue = + EventQueue::::new().expect("e1000d: failed to create event queue"); syscall::setrens(0, 0).expect("e1000d: failed to enter null namespace"); @@ -88,58 +109,76 @@ fn main() { let device_irq = device.clone(); let socket_irq = socket.clone(); let todo_irq = todo.clone(); - event_queue.add(irq_file.as_raw_fd(), move |_event| -> Result> { - let mut irq = [0; 8]; - irq_file.read(&mut irq)?; - if unsafe { device_irq.borrow().irq() } { - irq_file.write(&mut irq)?; + event_queue + .add( + irq_file.as_raw_fd(), + move |_event| -> Result> { + let mut irq = [0; 8]; + irq_file.read(&mut irq)?; + if unsafe { device_irq.borrow().irq() } { + irq_file.write(&mut irq)?; - if handle_update(&mut socket_irq.borrow_mut(), &mut device_irq.borrow_mut(), &mut todo_irq.borrow_mut())? { - return Ok(Some(0)) - } + if handle_update( + &mut socket_irq.borrow_mut(), + &mut device_irq.borrow_mut(), + &mut todo_irq.borrow_mut(), + )? { + return Ok(Some(0)); + } - let next_read = device_irq.borrow().next_read(); - if next_read > 0 { - return Ok(Some(next_read)); - } - } - Ok(None) - }).expect("e1000d: failed to catch events on IRQ file"); + let next_read = device_irq.borrow().next_read(); + if next_read > 0 { + return Ok(Some(next_read)); + } + } + Ok(None) + }, + ) + .expect("e1000d: failed to catch events on IRQ file"); let device_packet = device.clone(); let socket_packet = socket.clone(); - event_queue.add(socket_fd as RawFd, move |_event| -> Result> { - if handle_update(&mut socket_packet.borrow_mut(), &mut device_packet.borrow_mut(), &mut todo.borrow_mut())? { - return Ok(Some(0)); - } + event_queue + .add(socket_fd as RawFd, move |_event| -> Result> { + if handle_update( + &mut socket_packet.borrow_mut(), + &mut device_packet.borrow_mut(), + &mut todo.borrow_mut(), + )? { + return Ok(Some(0)); + } - let next_read = device_packet.borrow().next_read(); - if next_read > 0 { - return Ok(Some(next_read)); - } + let next_read = device_packet.borrow().next_read(); + if next_read > 0 { + return Ok(Some(next_read)); + } - Ok(None) - }).expect("e1000d: failed to catch events on scheme file"); + Ok(None) + }) + .expect("e1000d: failed to catch events on scheme file"); let send_events = |event_count| { for (handle_id, _handle) in device.borrow().handles.iter() { - socket.borrow_mut().write(&Packet { - id: 0, - pid: 0, - uid: 0, - gid: 0, - a: syscall::number::SYS_FEVENT, - b: *handle_id, - c: syscall::flag::EVENT_READ, - d: event_count - }).expect("e1000d: failed to write event"); + socket + .borrow_mut() + .write(&Packet { + id: 0, + pid: 0, + uid: 0, + gid: 0, + a: syscall::number::SYS_FEVENT, + b: *handle_id, + c: syscall::flag::EVENT_READ, + d: event_count, + }) + .expect("e1000d: failed to write event"); } }; - for event_count in event_queue.trigger_all(event::Event { - fd: 0, - flags: 0, - }).expect("e1000d: failed to trigger events") { + for event_count in event_queue + .trigger_all(event::Event { fd: 0, flags: 0 }) + .expect("e1000d: failed to trigger events") + { send_events(event_count); } @@ -152,6 +191,8 @@ fn main() { send_events(event_count); } } - unsafe { let _ = syscall::physunmap(address); } + unsafe { + let _ = syscall::physunmap(address); + } } }