ixgbed: DMA barriers on TDT and RDT doorbells

Same release/acquire fence pattern as the other PCI drivers.
The ixgbe (10 GbE) has the same hand-off semantics as e1000/rtl8168
(OWN/OWN-bit), so the same ordering is required: release fence
before TDT (TX doorbell) and RDT (RX doorbell), and an acquire
fence before reading the descriptor status.

The Linux ixgbe driver applies the same fences in the same
places; see afadba9..c17924b in the upstream tree. The original
Redox port from ixy.rs didn't carry them, which is a real bug
on aarch64 (or on any future cache-coherent extension).

drivers/net/ixgbed/src/device.rs

@@ -41,6 +41,9 @@ impl NetworkAdapter for Intel8259x {
             &mut *(self.receive_ring.as_ptr().add(self.receive_index) as *mut ixgbe_adv_rx_desc)
         };
 
+        // Acquire fence pairs the device's write of status /
+        // length with our subsequent reads.
+        core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire);
         let status = unsafe { desc.wb.upper.status_error };
 
         if (status & IXGBE_RXDADV_STAT_DD) != 0 {
@@ -63,6 +66,10 @@ impl NetworkAdapter for Intel8259x {
             desc.read.pkt_addr = self.receive_buffer[self.receive_index].physical() as u64;
             desc.read.hdr_addr = 0;
 
+            // Release fence ensures the buffer reads above are
+            // visible to the device before the RDT doorbell is
+            // observed.
+            core::sync::atomic::fence(core::sync::atomic::Ordering::Release);
             self.write_reg(IXGBE_RDT(0), self.receive_index as u32);
             self.receive_index = wrap_ring(self.receive_index, self.receive_ring.len());
 
@@ -120,6 +127,9 @@ impl NetworkAdapter for Intel8259x {
         self.transmit_index = wrap_ring(self.transmit_index, self.transmit_ring.len());
         self.transmit_ring_free -= 1;
 
+        // Release fence ensures the buffer writes above are visible
+        // to the device before the TDT doorbell is observed.
+        core::sync::atomic::fence(core::sync::atomic::Ordering::Release);
         self.write_reg(IXGBE_TDT(0), self.transmit_index as u32);
 
         Ok(i)