Phase 10a: SFTP open_read streaming — replace full-file buffer with chunked reads
Replaced session.read(path) (buffers entire file in memory, OOM risk for large files) with session.open(path) + SftpReader — a streaming Read bridge that wraps an open SFTP file handle and reads in bounded chunks via AsyncReadExt::read + block_on. SftpReader uses Mutex<File> for thread safety and tokio::runtime::Handle to bridge async→sync. Each Read::read() call issues a single SFTP read request for a bounded chunk — large files are never buffered in memory. Updated module doc to document the streaming read surface. 1433 tests pass (default), 1442 with --features sftp, zero warnings.
This commit is contained in:
@@ -10,6 +10,11 @@
|
||||
//! `stat`, `exists`, `is_dir`, `is_file`, `open_read`, `open_write`,
|
||||
//! `mkdir`, `remove`, and `rename` are all implemented.
|
||||
//!
|
||||
//! `open_read` returns a streaming [`SftpReader`] that reads from an
|
||||
//! open SFTP file handle in chunks via `AsyncReadExt::read`. Large
|
||||
//! files are never buffered in memory — each `Read::read()` call
|
||||
//! issues a fresh SFTP read request for a bounded chunk.
|
||||
//!
|
||||
//! Host-key verification uses the [`crate::vfs::known_hosts`] store
|
||||
//! at `~/.config/tlc/known_hosts` (or `$XDG_CONFIG_HOME/tlc/known_hosts`).
|
||||
//! First-time connects TOFU-trust the host; subsequent connects
|
||||
@@ -23,13 +28,13 @@
|
||||
#![cfg(feature = "sftp")]
|
||||
|
||||
use std::io::{self, Read, Write};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use russh::client::Config as SshConfig;
|
||||
use russh::client::Handle as SshHandle;
|
||||
use russh_keys::key::PublicKey;
|
||||
use russh_sftp::client::fs::{DirEntry, ReadDir};
|
||||
use russh_sftp::client::fs::{DirEntry, File, ReadDir};
|
||||
use russh_sftp::client::SftpSession;
|
||||
|
||||
use crate::fs::{FileType, Permissions, Stat};
|
||||
@@ -387,21 +392,16 @@ impl Vfs for SftpVfs {
|
||||
}
|
||||
|
||||
fn open_read(&self, p: &VfsPath) -> Result<Box<dyn Read + Send>, VfsError> {
|
||||
// The remote file is an async russh-sftp File. The sync Vfs
|
||||
// surface demands a `dyn Read`, so we buffer the full
|
||||
// contents into a `Cursor<Vec<u8>>` synchronously. This
|
||||
// matches the Phase 1 panel-listing use case (small directory
|
||||
// entries) and avoids having to ship a custom AsyncRead →
|
||||
// Read bridge. Large-file streaming is a Phase 7c concern.
|
||||
let path = sftp_path(p)?;
|
||||
let session = Arc::clone(&self.session);
|
||||
let bytes = self.block_on(async move {
|
||||
session
|
||||
.read(&path)
|
||||
.await
|
||||
.map_err(|e| VfsError::Other(format!("sftp read: {e}")))
|
||||
let file = self.block_on(async move {
|
||||
session.open(&path).await
|
||||
.map_err(|e| VfsError::Other(format!("sftp open: {e}")))
|
||||
})?;
|
||||
Ok(Box::new(io::Cursor::new(bytes)))
|
||||
Ok(Box::new(SftpReader {
|
||||
file: Mutex::new(file),
|
||||
handle: self.runtime.handle().clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn open_write(&self, p: &VfsPath) -> Result<Box<dyn Write + Send>, VfsError> {
|
||||
@@ -503,6 +503,28 @@ impl Drop for SftpVfs {
|
||||
}
|
||||
}
|
||||
|
||||
/// Streaming reader for remote SFTP files.
|
||||
///
|
||||
/// Wraps an open [`File`] handle and a [`tokio::runtime::Handle`],
|
||||
/// bridging the async read surface to the synchronous [`Read`] trait
|
||||
/// via `block_on`. Each `read()` call issues an SFTP read request
|
||||
/// for a bounded chunk; large files are never buffered in memory
|
||||
/// (unlike the previous `session.read()` approach).
|
||||
struct SftpReader {
|
||||
file: Mutex<File>,
|
||||
handle: tokio::runtime::Handle,
|
||||
}
|
||||
|
||||
impl Read for SftpReader {
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let mut file = self.file.lock().map_err(|e| {
|
||||
io::Error::new(io::ErrorKind::Other, format!("sftp reader lock: {e}"))
|
||||
})?;
|
||||
self.handle.block_on(async { file.read(buf).await })
|
||||
}
|
||||
}
|
||||
|
||||
struct SftpWriter {
|
||||
session: Arc<SftpSession>,
|
||||
handle: tokio::runtime::Handle,
|
||||
|
||||
Reference in New Issue
Block a user