From 2eefa1a16fba0086f0c267aa2a9d1fe898503525 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Fri, 19 Feb 2021 23:29:04 +0100 Subject: [PATCH 01/23] Initial Commit --- .gitignore | 2 + Cargo.toml | 14 ++ src/lib.rs | 8 + src/types.rs | 50 +++++++ utils/Cargo.toml | 17 +++ utils/src/redox_initfs_package.rs | 237 ++++++++++++++++++++++++++++++ 6 files changed, 328 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/lib.rs create mode 100644 src/types.rs create mode 100644 utils/Cargo.toml create mode 100644 utils/src/redox_initfs_package.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..ba263d9102 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "redox-initfs" +version = "0.1.0" +authors = ["4lDO2 <4lDO2@protonmail.com>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] + +[workspace] +members = [ + "utils", +] diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000000..341006cdfd --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,8 @@ +//! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be +//! read. + +pub mod types; + +pub struct InitFs<'a> { + base: &'a [u8], +} diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000000..290c538f92 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,50 @@ +pub const MAGIC_LEN: usize = 8; +pub const MAGIC: [u8; 8] = *b"RedoxFtw"; + +#[repr(transparent)] +#[derive(Clone, Copy, Debug)] +pub struct Magic(pub [u8; MAGIC_LEN]); + +#[repr(transparent)] +#[derive(Clone, Copy, Debug)] +pub struct Offset(pub u32); + +#[repr(transparent)] +#[derive(Clone, Copy, Debug)] +pub struct Length(pub u32); + +#[repr(transparent)] +#[derive(Clone, Copy, Debug)] +pub struct Inode(pub u16); + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct Timespec { + pub sec: u64, + pub nsec: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct Header { + pub magic: Magic, + pub inode_table_offset: Offset, + pub creation_time: Timespec, + pub inode_count: u16, +} + +#[repr(C)] +pub struct InodeHeader { + pub type_and_mode: u32, + pub length: u32, + pub offset: Offset, + pub uid: u32, + pub gid: u32, +} + +#[repr(C)] +pub struct DirEntry { + pub inode: Inode, + pub name_len: u16, + pub name_offset: Offset, +} diff --git a/utils/Cargo.toml b/utils/Cargo.toml new file mode 100644 index 0000000000..1246414d08 --- /dev/null +++ b/utils/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "utils" +version = "0.1.0" +authors = ["4lDO2 <4lDO2@protonmail.com>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +path = "src/redox_initfs_package.rs" +name = "redox-initfs-package" + +[dependencies] +anyhow = "1" +clap = "2.33" + +redox-initfs = { path = ".." } diff --git a/utils/src/redox_initfs_package.rs b/utils/src/redox_initfs_package.rs new file mode 100644 index 0000000000..53a4ed90f6 --- /dev/null +++ b/utils/src/redox_initfs_package.rs @@ -0,0 +1,237 @@ +use std::convert::{TryFrom, TryInto}; +use std::fs::{DirEntry, File, FileType, OpenOptions, ReadDir}; +use std::path::{Path, PathBuf}; + +use std::os::unix::ffi::OsStringExt; +use std::os::unix::fs::{FileExt, FileTypeExt}; + +use anyhow::{anyhow, Context, Result}; +use clap::{App, Arg}; + +use redox_initfs::types as initfs; + +const DEFAULT_MAX_SIZE: u64 = 8 * 1024 * 1024; + +enum Entry { + File(File), + Dir(Dir), +} +struct Child { + name: Vec, + entry: Entry, +} +struct Dir { + children: Vec, +} + +struct State { + file: File, + offset: u64, + max_size: u64, + inode_count: u16, +} + +fn read_directory(state: &mut State, path: &Path) -> Result { + let read_dir = path.read_dir().map_err(|error| { + anyhow!( + "failed to read directory `{}`: {}", + path.to_string_lossy(), + error + ) + })?; + + let children = read_dir + .map(|result| { + let entry = result.map_err(|error| { + anyhow!( + "failed to get a directory entry from `{}`: {}", + path.to_string_lossy(), + error + ) + })?; + + let file_type = entry.file_type().map_err(|error| { + anyhow!( + "failed to determine file type for `{}`: {}", + entry.path().to_string_lossy(), + error + ) + })?; + + let unsupported_type = |ty: &str, entry: &DirEntry| { + Err(anyhow!( + "failed to include {} at `{}`: not supported by redox-initfs", + ty, + entry.path().to_string_lossy() + )) + }; + let name = entry.path().into_os_string().into_vec(); + let entry = if file_type.is_symlink() { + return unsupported_type("symlink", &entry); + } else if file_type.is_socket() { + return unsupported_type("socket", &entry); + } else if file_type.is_fifo() { + return unsupported_type("FIFO", &entry); + } else if file_type.is_block_device() { + return unsupported_type("block device", &entry); + } else if file_type.is_char_device() { + return unsupported_type("character device", &entry); + } else if file_type.is_dir() { + Entry::File(File::open(&entry.path()).map_err(|error| { + anyhow!( + "failed to open file `{}`: {}", + entry.path().to_string_lossy(), + error + ) + })?) + } else if file_type.is_file() { + Entry::Dir(read_directory(state, &entry.path())?) + } else { + return Err(anyhow!( + "unknown file type at `{}`", + entry.path().to_string_lossy() + )); + }; + + // TODO: Allow the user to specify a lower limit than u16::MAX. + state.inode_count = state.inode_count.checked_add(1) + .ok_or_else(|| anyhow!("exceeded the maximum inode limit"))?; + + Ok(Child { entry, name }) + }) + .collect::>>()?; + + Ok(Dir { children }) +} + +fn bump_alloc(state: &mut State, size: u64) -> Result { + if state.offset + size <= state.max_size { + let offset = state.offset; + state.offset += size; + Ok(offset) + } else { + Err(anyhow!("Bump allocation failed: max limit reached")) + } +} + +fn main() -> Result<()> { + let matches = App::new("redox_initfs_package") + .help("Package a Redox initfs") + .arg( + Arg::with_name("MAX_SIZE") + .long("--max-size") + .short("-m") + .takes_value(true) + .required(false) + .help("Set the upper limit for how large the image can become (default 8 MiB)."), + ) + .arg( + Arg::with_name("SOURCE") + .takes_value(true) + .required(true) + .help("Specify the source directory to build the image from."), + ) + .arg( + Arg::with_name("OUTPUT") + .takes_value(true) + .required(true) + .long("--output") + .short("-o") + .help("Specify the path of the new image file."), + ) + .get_matches(); + + let max_size = if let Some(max_size_str) = matches.value_of("MAX_SIZE") { + max_size_str + .parse::() + .context("expected an integer for MAX_SIZE")? + } else { + DEFAULT_MAX_SIZE + }; + + let source = matches + .value_of("SOURCE") + .expect("expected the required arg SOURCE to exist"); + + let destination = matches + .value_of("OUTPUT") + .expect("expected the required arg OUTPUT to exist"); + + let destination_path = Path::new(destination); + + let previous_extension = destination_path.extension().map_or("", |ext| { + ext.to_str() + .expect("expected destination path to be valid UTF-8") + }); + + if !destination_path + .metadata() + .map_or(true, |metadata| metadata.is_file()) + { + return Err(anyhow!("Destination file must be a file")); + } + + let destination_temp_path = + destination_path.with_extension(format!("{}.partial", previous_extension)); + + let destination_temp_file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .create_new(false) + .open(destination_temp_path) + .context("failed to open destination file")?; + + let mut state = State { + file: destination_temp_file, + offset: 0, + max_size, + inode_count: 0, + }; + + let root = read_directory(&mut state, Path::new(source)).context("failed to read root")?; + + // NOTE: The header is always stored at offset zero. + let header_offset = bump_alloc( + &mut state, + std::mem::size_of::().try_into() + .expect("expected header size to fit"), + )?; + + let inode_table_length = { + let inode_entry_size: u64 = std::mem::size_of::() + .try_into() + .expect("expected table entry size to fit"); + + inode_entry_size + .checked_mul(u64::from(state.inode_count)) + .ok_or_else(|| anyhow!("inode table too large"))? + }; + + let inode_table_offset = bump_alloc( + &mut state, + inode_table_length, + )?; + let inode_table_offset = initfs::Offset( + u32::try_from(inode_table_offset) + .map_err(|_| anyhow!("inode table located too far away"))? + ); + + let current_system_time = std::time::SystemTime::now(); + + let time_since_epoch = current_system_time + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .context("could not calculate timestamp")?; + + let header = initfs::Header { + magic: initfs::Magic(initfs::MAGIC), + creation_time: initfs::Timespec { + sec: time_since_epoch.as_secs(), + nsec: time_since_epoch.subsec_nanos(), + }, + inode_count: state.inode_count, + inode_table_offset, + }; + + Ok(()) +} From 8b5474189debe415809cd7576c5988c1a6910911 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 11:51:33 +0100 Subject: [PATCH 02/23] WIP: (probably) complete basic initfs-ar util. --- Cargo.toml | 1 + src/types.rs | 89 +++++++-- utils/Cargo.toml | 1 + utils/src/redox_initfs_package.rs | 305 +++++++++++++++++++++++++++--- 4 files changed, 351 insertions(+), 45 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ba263d9102..ac56f5dadc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +plain = "0.2" [workspace] members = [ diff --git a/src/types.rs b/src/types.rs index 290c538f92..f1cbb22461 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,50 +1,109 @@ pub const MAGIC_LEN: usize = 8; pub const MAGIC: [u8; 8] = *b"RedoxFtw"; +macro_rules! primitive( + ($wrapper:ident, $bits:expr, $primitive:ident) => { + #[repr(transparent)] + #[derive(Clone, Copy, Debug, Default)] + pub struct $wrapper([u8; $bits / 8]); + + impl $wrapper { + #[inline] + pub fn get(&self) -> $primitive { + <$primitive>::from_le_bytes(self.0) + } + #[inline] + pub fn set(&mut self, new: $primitive) { + self.0 = <$primitive>::to_le_bytes(new); + } + #[inline] + pub fn new(primitive: $primitive) -> Self { + let mut this = Self::default(); + this.set(primitive); + this + } + } + impl From<$primitive> for $wrapper { + fn from(primitive: $primitive) -> Self { + Self::new(primitive) + } + } + impl From<$wrapper> for $primitive { + fn from(wrapper: $wrapper) -> Self { + wrapper.get() + } + } + } +); + +primitive!(U16, 16, u16); +primitive!(U32, 32, u32); +primitive!(U64, 64, u64); + #[repr(transparent)] #[derive(Clone, Copy, Debug)] pub struct Magic(pub [u8; MAGIC_LEN]); #[repr(transparent)] #[derive(Clone, Copy, Debug)] -pub struct Offset(pub u32); +pub struct Offset(pub U32); #[repr(transparent)] #[derive(Clone, Copy, Debug)] -pub struct Length(pub u32); +pub struct Length(pub U32); #[repr(transparent)] #[derive(Clone, Copy, Debug)] -pub struct Inode(pub u16); +pub struct Inode(pub U16); -#[repr(C)] +#[repr(packed)] #[derive(Clone, Copy, Debug)] pub struct Timespec { - pub sec: u64, - pub nsec: u32, + pub sec: U64, + pub nsec: U32, } -#[repr(C)] +#[repr(packed)] #[derive(Clone, Copy, Debug)] pub struct Header { pub magic: Magic, pub inode_table_offset: Offset, pub creation_time: Timespec, - pub inode_count: u16, + pub inode_count: U16, } -#[repr(C)] +#[repr(packed)] +#[derive(Clone, Copy, Debug)] pub struct InodeHeader { - pub type_and_mode: u32, - pub length: u32, + pub type_and_mode: U32, + pub length: U32, pub offset: Offset, - pub uid: u32, - pub gid: u32, + pub uid: U32, + pub gid: U32, } -#[repr(C)] +pub const MODE_MASK: u32 = 0xFFF; +pub const MODE_SHIFT: u8 = 0; + +pub const TYPE_SHIFT: u8 = 28; +pub const TYPE_MASK: u32 = 0xF000_0000; + +#[derive(Clone, Copy, Debug)] +#[repr(u8)] +pub enum InodeType { + RegularFile = 0x0, + Dir = 0x1, + // All other bit patterns are reserved... for now. TODO: Add symlinks? +} + +#[repr(packed)] +#[derive(Clone, Copy, Debug)] pub struct DirEntry { pub inode: Inode, - pub name_len: u16, + pub name_len: U16, pub name_offset: Offset, } + +unsafe impl plain::Plain for Header {} +unsafe impl plain::Plain for InodeHeader {} +unsafe impl plain::Plain for DirEntry {} diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 1246414d08..86f2970edf 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -13,5 +13,6 @@ name = "redox-initfs-package" [dependencies] anyhow = "1" clap = "2.33" +plain = "0.2" redox-initfs = { path = ".." } diff --git a/utils/src/redox_initfs_package.rs b/utils/src/redox_initfs_package.rs index 53a4ed90f6..9be62d7546 100644 --- a/utils/src/redox_initfs_package.rs +++ b/utils/src/redox_initfs_package.rs @@ -1,9 +1,10 @@ use std::convert::{TryFrom, TryInto}; -use std::fs::{DirEntry, File, FileType, OpenOptions, ReadDir}; +use std::fs::{DirEntry, File, FileType, Metadata, OpenOptions, ReadDir}; +use std::io::{prelude::*, SeekFrom}; use std::path::{Path, PathBuf}; use std::os::unix::ffi::OsStringExt; -use std::os::unix::fs::{FileExt, FileTypeExt}; +use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt}; use anyhow::{anyhow, Context, Result}; use clap::{App, Arg}; @@ -12,16 +13,18 @@ use redox_initfs::types as initfs; const DEFAULT_MAX_SIZE: u64 = 8 * 1024 * 1024; -enum Entry { +enum EntryKind { File(File), Dir(Dir), } -struct Child { + +struct Entry { name: Vec, - entry: Entry, + kind: EntryKind, + metadata: Metadata, } struct Dir { - children: Vec, + entries: Vec, } struct State { @@ -29,6 +32,7 @@ struct State { offset: u64, max_size: u64, inode_count: u16, + buffer: Box<[u8]>, } fn read_directory(state: &mut State, path: &Path) -> Result { @@ -40,7 +44,7 @@ fn read_directory(state: &mut State, path: &Path) -> Result { ) })?; - let children = read_dir + let entries = read_dir .map(|result| { let entry = result.map_err(|error| { anyhow!( @@ -50,13 +54,14 @@ fn read_directory(state: &mut State, path: &Path) -> Result { ) })?; - let file_type = entry.file_type().map_err(|error| { + let metadata = entry.metadata().map_err(|error| { anyhow!( - "failed to determine file type for `{}`: {}", + "failed to get metadata for `{}`: {}", entry.path().to_string_lossy(), error ) })?; + let file_type = metadata.file_type(); let unsupported_type = |ty: &str, entry: &DirEntry| { Err(anyhow!( @@ -66,7 +71,7 @@ fn read_directory(state: &mut State, path: &Path) -> Result { )) }; let name = entry.path().into_os_string().into_vec(); - let entry = if file_type.is_symlink() { + let entry_kind = if file_type.is_symlink() { return unsupported_type("symlink", &entry); } else if file_type.is_socket() { return unsupported_type("socket", &entry); @@ -77,7 +82,7 @@ fn read_directory(state: &mut State, path: &Path) -> Result { } else if file_type.is_char_device() { return unsupported_type("character device", &entry); } else if file_type.is_dir() { - Entry::File(File::open(&entry.path()).map_err(|error| { + EntryKind::File(File::open(&entry.path()).map_err(|error| { anyhow!( "failed to open file `{}`: {}", entry.path().to_string_lossy(), @@ -85,7 +90,7 @@ fn read_directory(state: &mut State, path: &Path) -> Result { ) })?) } else if file_type.is_file() { - Entry::Dir(read_directory(state, &entry.path())?) + EntryKind::Dir(read_directory(state, &entry.path())?) } else { return Err(anyhow!( "unknown file type at `{}`", @@ -94,14 +99,20 @@ fn read_directory(state: &mut State, path: &Path) -> Result { }; // TODO: Allow the user to specify a lower limit than u16::MAX. - state.inode_count = state.inode_count.checked_add(1) + state.inode_count = state + .inode_count + .checked_add(1) .ok_or_else(|| anyhow!("exceeded the maximum inode limit"))?; - Ok(Child { entry, name }) + Ok(Entry { + kind: entry_kind, + metadata, + name, + }) }) .collect::>>()?; - Ok(Dir { children }) + Ok(Dir { entries }) } fn bump_alloc(state: &mut State, size: u64) -> Result { @@ -113,6 +124,214 @@ fn bump_alloc(state: &mut State, size: u64) -> Result { Err(anyhow!("Bump allocation failed: max limit reached")) } } +struct WriteResult { + size: u32, + offset: u32, +} + +fn allocate_and_write_file(state: &mut State, mut file: &File) -> Result { + let size = file + .seek(SeekFrom::End(0)) + .context("failed to seek to end")?; + + let size: u32 = size.try_into().context("file too large")?; + + let offset: u32 = bump_alloc(state, size.into()) + .context("failed to allocate space for file")? + .try_into() + .context("file offset too high")?; + + let buffer_size: u32 = state.buffer.len().try_into().context("buffer too large")?; + + file.seek(SeekFrom::Start(0)) + .context("failed to seek to start")?; + + state + .file + .seek(SeekFrom::Start(offset.into())) + .context("failed to seek to offset for image")?; + + let mut relative_offset = 0; + + // TODO: If this would ever turn out to be a bottleneck, then perhaps we could use + // copy_file_range in `nix`. + + while relative_offset < size { + let allowed_length = std::cmp::min(buffer_size, size - relative_offset); + let allowed_length = + usize::try_from(allowed_length).expect("expected buffer size not to be outside usize"); + + file.read(&mut state.buffer[..allowed_length]) + .context("failed to read from source file")?; + + state + .file + .write(&state.buffer[..allowed_length]) + .context("failed to write source file into destination image")?; + + relative_offset += buffer_size; + } + + Ok(WriteResult { size, offset }) +} +fn write_inode( + state: &mut State, + ty: initfs::InodeType, + metadata: &Metadata, + write_result: WriteResult, + inode_table_offset: u32, + index: u16, +) -> Result<()> { + let inode_size: u32 = std::mem::size_of::() + .try_into() + .expect("inode header length cannot fit within u32"); + + let type_and_mode = ((ty as u32) << initfs::TYPE_SHIFT) | u32::from(metadata.mode() & 0xFFF); + + // TODO: Use main buffer and write in bulk. + let mut inode_buf = [0_u8; std::mem::size_of::()]; + + let inode = plain::from_mut_bytes::(&mut inode_buf) + .expect("expected inode struct to have alignment 1, and buffer size to match"); + + *inode = initfs::InodeHeader { + type_and_mode: type_and_mode.into(), + length: write_result.size.into(), + offset: initfs::Offset(write_result.offset.into()), + + gid: metadata.gid().into(), + uid: metadata.uid().into(), + }; + + state + .file + .write_all_at( + &inode_buf, + u64::from(inode_table_offset + u32::from(index) * inode_size), + ) + .context("failed to write inode struct to disk image") +} +fn allocate_and_write_dir( + state: &mut State, + dir: &Dir, + inode_table_offset: u32, + start_inode: &mut u16, +) -> Result { + let entry_size = + u16::try_from(std::mem::size_of::()).context("entry size too large")?; + let entry_count = u16::try_from(dir.entries.len()).context("too many subdirectories")?; + + let entry_table_length = u32::from(entry_count) + .checked_mul(u32::from(entry_size)) + .ok_or_else(|| anyhow!("entry table length too large when multiplying by size"))?; + + let entry_table_offset: u32 = bump_alloc(state, entry_table_length.into()) + .context("failed to allocate entry table")? + .try_into() + .context("directory entries offset too high")?; + + let this_start_inode = *start_inode; + + let inode_size: u32 = std::mem::size_of::() + .try_into() + .expect("inode header length cannot fit within u32"); + + *start_inode += entry_count; + + let inode_table_offset = + inode_table_offset + u32::from(this_start_inode) * u32::from(inode_size); + + for (index, entry) in dir.entries.iter().enumerate() { + let (write_result, ty) = match entry.kind { + EntryKind::Dir(ref subdir) => { + let write_result = + allocate_and_write_dir(state, subdir, inode_table_offset, start_inode) + .context("failed to copy directory entries into image")?; + + (write_result, initfs::InodeType::Dir) + } + + EntryKind::File(ref file) => { + let write_result = allocate_and_write_file(state, file) + .context("failed to copy file into image")?; + + (write_result, initfs::InodeType::RegularFile) + } + }; + + let index: u16 = index + .try_into() + .expect("expected dir entry count not to exceed u32"); + + write_inode( + state, + ty, + &entry.metadata, + write_result, + inode_table_offset, + index, + )?; + + let (name_offset, name_len) = { + let name_len: u16 = entry.name.len().try_into().context("file name too long")?; + + let offset: u32 = bump_alloc(state, u64::from(name_len)) + .context("failed to allocate space for file name")? + .try_into() + .context("file name offset too high up")?; + + (offset, name_len) + }; + { + let mut direntry_buf = [0_u8; std::mem::size_of::()]; + + let direntry = plain::from_mut_bytes::(&mut direntry_buf) + .expect("expected dir entry struct to have alignment 1, and buffer size to match"); + + let inode = this_start_inode + index; + + *direntry = initfs::DirEntry { + inode: initfs::Inode(inode.into()), + name_len: name_len.into(), + name_offset: initfs::Offset(name_offset.into()), + }; + + state + .file + .write_all_at( + &direntry_buf, + u64::from(entry_table_offset + u32::from(index) * u32::from(entry_size)), + ) + .context("failed to write dir entry struct to image")?; + } + } + + Ok(WriteResult { + size: entry_table_length, + offset: entry_table_offset, + }) +} +fn allocate_contents_and_write_inodes( + state: &mut State, + dir: &Dir, + inode_table_offset: u32, + root_metadata: Metadata, +) -> Result<()> { + let index = 0; + let mut start_inode = 1; + + let write_result = allocate_and_write_dir(state, dir, inode_table_offset, &mut start_inode) + .context("failed to allocate and write all directories and files")?; + + write_inode( + state, + initfs::InodeType::Dir, + &root_metadata, + write_result, + inode_table_offset, + index, + ) +} fn main() -> Result<()> { let matches = App::new("redox_initfs_package") @@ -182,19 +401,27 @@ fn main() -> Result<()> { .open(destination_temp_path) .context("failed to open destination file")?; + const BUFFER_SIZE: usize = 8192; + let mut state = State { file: destination_temp_file, offset: 0, max_size, inode_count: 0, + buffer: vec![0_u8; BUFFER_SIZE].into_boxed_slice(), }; - let root = read_directory(&mut state, Path::new(source)).context("failed to read root")?; + let root_path = Path::new(source); + let root_metadata = root_path + .metadata() + .context("failed to obtain metadata for root")?; + let root = read_directory(&mut state, root_path).context("failed to read root")?; // NOTE: The header is always stored at offset zero. let header_offset = bump_alloc( &mut state, - std::mem::size_of::().try_into() + std::mem::size_of::() + .try_into() .expect("expected header size to fit"), )?; @@ -208,30 +435,48 @@ fn main() -> Result<()> { .ok_or_else(|| anyhow!("inode table too large"))? }; - let inode_table_offset = bump_alloc( - &mut state, - inode_table_length, - )?; + let inode_table_offset = bump_alloc(&mut state, inode_table_length)?; + + // Finally, write the header to the disk image. + let inode_table_offset = initfs::Offset( u32::try_from(inode_table_offset) .map_err(|_| anyhow!("inode table located too far away"))? + .into(), ); + allocate_contents_and_write_inodes( + &mut state, + &root, + inode_table_offset.0.get(), + root_metadata, + )?; + let current_system_time = std::time::SystemTime::now(); let time_since_epoch = current_system_time .duration_since(std::time::SystemTime::UNIX_EPOCH) .context("could not calculate timestamp")?; - let header = initfs::Header { - magic: initfs::Magic(initfs::MAGIC), - creation_time: initfs::Timespec { - sec: time_since_epoch.as_secs(), - nsec: time_since_epoch.subsec_nanos(), - }, - inode_count: state.inode_count, - inode_table_offset, - }; + { + let mut header_bytes = [0_u8; std::mem::size_of::()]; + let header = plain::from_mut_bytes(&mut header_bytes) + .expect("expected header size to be sufficient and alignment to be 1"); + + *header = initfs::Header { + magic: initfs::Magic(initfs::MAGIC), + creation_time: initfs::Timespec { + sec: time_since_epoch.as_secs().into(), + nsec: time_since_epoch.subsec_nanos().into(), + }, + inode_count: state.inode_count.into(), + inode_table_offset, + }; + state + .file + .write_all_at(&header_bytes, header_offset) + .context("failed to write header")?; + } Ok(()) } From 1cdb1c7f198d9a31015dcbe6e1e3c155fc16cd0b Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 12:09:56 +0100 Subject: [PATCH 03/23] Increse default limit to 64 MiB. --- utils/src/redox_initfs_package.rs | 61 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/utils/src/redox_initfs_package.rs b/utils/src/redox_initfs_package.rs index 9be62d7546..d438c78e7f 100644 --- a/utils/src/redox_initfs_package.rs +++ b/utils/src/redox_initfs_package.rs @@ -1,9 +1,9 @@ use std::convert::{TryFrom, TryInto}; -use std::fs::{DirEntry, File, FileType, Metadata, OpenOptions, ReadDir}; +use std::fs::{DirEntry, File, Metadata, OpenOptions}; use std::io::{prelude::*, SeekFrom}; -use std::path::{Path, PathBuf}; +use std::path::Path; -use std::os::unix::ffi::OsStringExt; +use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt}; use anyhow::{anyhow, Context, Result}; @@ -11,7 +11,9 @@ use clap::{App, Arg}; use redox_initfs::types as initfs; -const DEFAULT_MAX_SIZE: u64 = 8 * 1024 * 1024; +const KIBIBYTE: u64 = 1024; +const MEBIBYTE: u64 = KIBIBYTE * 1024; +const DEFAULT_MAX_SIZE: u64 = 64 * MEBIBYTE; enum EntryKind { File(File), @@ -36,29 +38,23 @@ struct State { } fn read_directory(state: &mut State, path: &Path) -> Result { - let read_dir = path.read_dir().map_err(|error| { - anyhow!( - "failed to read directory `{}`: {}", - path.to_string_lossy(), - error - ) - })?; + let read_dir = path + .read_dir() + .with_context(|| anyhow!("failed to read directory `{}`", path.to_string_lossy(),))?; let entries = read_dir .map(|result| { - let entry = result.map_err(|error| { + let entry = result.with_context(|| { anyhow!( - "failed to get a directory entry from `{}`: {}", + "failed to get a directory entry from `{}`", path.to_string_lossy(), - error ) })?; - let metadata = entry.metadata().map_err(|error| { + let metadata = entry.metadata().with_context(|| { anyhow!( - "failed to get metadata for `{}`: {}", + "failed to get metadata for `{}`", entry.path().to_string_lossy(), - error ) })?; let file_type = metadata.file_type(); @@ -70,7 +66,13 @@ fn read_directory(state: &mut State, path: &Path) -> Result { entry.path().to_string_lossy() )) }; - let name = entry.path().into_os_string().into_vec(); + let name = entry + .path() + .file_name() + .context("expected path to have a valid filename")? + .as_bytes() + .to_owned(); + let entry_kind = if file_type.is_symlink() { return unsupported_type("symlink", &entry); } else if file_type.is_socket() { @@ -81,15 +83,11 @@ fn read_directory(state: &mut State, path: &Path) -> Result { return unsupported_type("block device", &entry); } else if file_type.is_char_device() { return unsupported_type("character device", &entry); - } else if file_type.is_dir() { - EntryKind::File(File::open(&entry.path()).map_err(|error| { - anyhow!( - "failed to open file `{}`: {}", - entry.path().to_string_lossy(), - error - ) - })?) } else if file_type.is_file() { + EntryKind::File(File::open(&entry.path()).with_context(|| { + anyhow!("failed to open file `{}`", entry.path().to_string_lossy(),) + })?) + } else if file_type.is_dir() { EntryKind::Dir(read_directory(state, &entry.path())?) } else { return Err(anyhow!( @@ -121,7 +119,7 @@ fn bump_alloc(state: &mut State, size: u64) -> Result { state.offset += size; Ok(offset) } else { - Err(anyhow!("Bump allocation failed: max limit reached")) + Err(anyhow!("bump allocation failed: max limit reached")) } } struct WriteResult { @@ -246,7 +244,12 @@ fn allocate_and_write_dir( EntryKind::Dir(ref subdir) => { let write_result = allocate_and_write_dir(state, subdir, inode_table_offset, start_inode) - .context("failed to copy directory entries into image")?; + .with_context(|| { + anyhow!( + "failed to copy directory entries from `{}` into image", + String::from_utf8_lossy(&entry.name) + ) + })?; (write_result, initfs::InodeType::Dir) } @@ -441,7 +444,7 @@ fn main() -> Result<()> { let inode_table_offset = initfs::Offset( u32::try_from(inode_table_offset) - .map_err(|_| anyhow!("inode table located too far away"))? + .with_context(|| "inode table located too far away")? .into(), ); From 672a92bb51d6757a195915d6a5728e6f311f1a6d Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 13:58:52 +0100 Subject: [PATCH 04/23] Create a skeleton for the initfs API for kernel. --- Cargo.toml | 8 ++ src/lib.rs | 207 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/types.rs | 14 ++-- 3 files changed, 219 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac56f5dadc..160d9d4358 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,14 @@ edition = "2018" [dependencies] plain = "0.2" +[features] +default = [] + +# Enables functionality to actually read the filesystem, which is only done in +# the kernel. (Writing to the filesystem is done in utils, and that crate only +# needs the type definitions). +kernel = [] + [workspace] members = [ "utils", diff --git a/src/lib.rs b/src/lib.rs index 341006cdfd..df56dba28a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,211 @@ //! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be //! read. +use core::convert::{TryFrom, TryInto}; + pub mod types; -pub struct InitFs<'a> { - base: &'a [u8], +use self::types::*; + +#[derive(Clone, Copy)] +pub struct InitFs<'initfs> { + base: &'initfs [u8], +} + +#[derive(Clone, Copy, Debug)] +pub struct Error; + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "invalid or corrupt initfs") + } +} + +type Result = core::result::Result; + +#[derive(Clone, Copy)] +pub struct InodeStruct<'initfs> { + initfs: InitFs<'initfs>, + inode: &'initfs InodeHeader, +} + +#[derive(Clone, Copy)] +pub struct InodeFile<'initfs> { + inner: InodeStruct<'initfs>, +} +impl<'initfs> InodeFile<'initfs> { + pub fn inode(self) -> InodeStruct<'initfs> { + self.inner + } + pub fn data(&self) -> Result<&'initfs [u8]> { + self.inner.data() + } +} + +#[derive(Clone, Copy)] +pub struct InodeDir<'initfs> { + inner: InodeStruct<'initfs>, +} +impl<'initfs> InodeDir<'initfs> { + pub fn inode(self) -> InodeStruct<'initfs> { + self.inner + } +} + +#[derive(Clone, Copy)] +pub enum InodeKind<'initfs> { + File(InodeFile<'initfs>), + Dir(InodeDir<'initfs>), + Unknown, +} + +impl<'initfs> InodeStruct<'initfs> { + fn data(&self) -> Result<&'initfs [u8]> { + let start: usize = self.inode.offset.0.get().try_into().map_err(|_| Error)?; + + let length: usize = self.inode.length.get().try_into().map_err(|_| Error)?; + + let end = start.checked_add(length).ok_or(Error)?; + + self.initfs.base.get(start..end).ok_or(Error) + } + pub fn mode(&self) -> u16 { + (self.inode.type_and_mode.get() & MODE_MASK) as u16 + } + pub fn uid(&self) -> u32 { + self.inode.uid.get() + } + pub fn gid(&self) -> u32 { + self.inode.gid.get() + } + fn ty(&self) -> Option { + let raw = (self.inode.type_and_mode.get() & TYPE_MASK) >> TYPE_SHIFT; + + Some(if raw == InodeType::Dir as u32 { + InodeType::Dir + } else if raw == InodeType::RegularFile as u32 { + InodeType::RegularFile + } else { + return None; + }) + } + pub fn kind(&self) -> InodeKind<'initfs> { + match self.ty() { + Some(InodeType::Dir) => InodeKind::Dir(InodeDir { inner: *self }), + Some(InodeType::RegularFile) => InodeKind::File(InodeFile { inner: *self }), + None => InodeKind::Unknown, + } + } +} + +impl<'initfs> InitFs<'initfs> { + pub fn new(base: &'initfs [u8]) -> Result { + let this = Self { base }; + + if base.len() < core::mem::size_of::
() { + return Err(Error); + } + if u32::try_from(base.len()).is_err() { + return Err(Error); + } + + let header = this.get_header_assume_valid(); + + if header.magic != Magic(MAGIC) { + return Err(Error); + } + + let inode_table_offset = header.inode_table_offset.0.get(); + + if inode_table_offset < u32::from(Self::header_len_8()) { + return Err(Error); + } + + let inode_table_size = this.inode_table_size(); + + let inode_table_end = inode_table_offset + .checked_add(inode_table_size) + .ok_or(Error)?; + + if inode_table_end > this.base_len_32() { + return Err(Error); + } + + // From now on, we can be completely sure that the header and inode tables offsets are + // valid, and thus continue based on that assumption. + + Ok(this) + } + pub fn image_creation_time(&self) -> Timespec { + self.get_header_assume_valid().creation_time + } + fn get_header_assume_valid(&self) -> &Header { + plain::from_bytes::
(&self.base[..core::mem::size_of::
()]) + .expect("expected header type to require no alignment, and size to be sufficient") + } + fn header_len_8() -> u8 { + core::mem::size_of::
() + .try_into() + .expect("expected header size to fit within u8") + } + fn inode_struct_len_8() -> u8 { + core::mem::size_of::() + .try_into() + .expect("expected inode struct size to fit within u8") + } + fn direntry_struct_len_8() -> u8 { + core::mem::size_of::() + .try_into() + .expect("expected dir entry struct size to fit within u8") + } + fn inode_table_offset_usize(&self) -> usize { + // NOTE: We have already validated that the inode table fits within the initfs slice, and + // that this length, which must fit within usize, also fits within u32. + self.get_header_assume_valid().inode_table_offset.0.get() as usize + } + fn inode_table_end_usize(&self) -> usize { + // NOTE: This follows the same reasoning as in inode_table_offset_usize(). The end offset + // has been checked against u32 and the initfs slice length. + self.inode_table_offset_usize() + .wrapping_add(self.inode_table_size() as usize) + } + fn inode_table_range(&self) -> core::ops::Range { + self.inode_table_offset_usize()..self.inode_table_end_usize() + } + fn base_len_32(&self) -> u32 { + // NOTE: We have already validated that the length is sufficient. + self.base.len() as u32 + } + fn inode_table_size(&self) -> u32 { + let count = self.get_header_assume_valid().inode_count.get(); + let struct_size = Self::inode_struct_len_8(); + + // NOTE: We know for a fact, that even the largest u8 (255) and the largest u16 (65536), + // can only be approximately 2^24 at max. + u32::wrapping_mul(u32::from(count), u32::from(struct_size)) + } + fn inode_table(&self) -> &'initfs [InodeHeader] { + let inode_table_bytes = &self.base[self.inode_table_range()]; + + plain::slice_from_bytes::(inode_table_bytes) + .expect("expected inode struct alignment to be 1") + } + pub const ROOT_INODE: Inode = Inode(U16::new(0)); + + pub fn inode_count(&self) -> u16 { + self.get_header_assume_valid().inode_count.get() + } + pub fn get_inode(&self, inode: Inode) -> Option> { + // NOTE: Even for 16-bit architectures (obviously edge-case, but some bootloaders may + // perhaps use this code), we have already checked that the inode table can fit within + // usize, and the table byte size is always larger than the count. + let inode_usize = inode.0.get() as usize; + + let inode = self.inode_table().get(inode_usize)?; + + Some(InodeStruct { + initfs: *self, + inode, + }) + } } diff --git a/src/types.rs b/src/types.rs index f1cbb22461..cbddb7ed48 100644 --- a/src/types.rs +++ b/src/types.rs @@ -9,18 +9,16 @@ macro_rules! primitive( impl $wrapper { #[inline] - pub fn get(&self) -> $primitive { + pub const fn get(&self) -> $primitive { <$primitive>::from_le_bytes(self.0) } #[inline] - pub fn set(&mut self, new: $primitive) { - self.0 = <$primitive>::to_le_bytes(new); + pub fn set(&mut self, primitive: $primitive) { + *self = Self::new(primitive); } #[inline] - pub fn new(primitive: $primitive) -> Self { - let mut this = Self::default(); - this.set(primitive); - this + pub const fn new(primitive: $primitive) -> Self { + Self(<$primitive>::to_le_bytes(primitive)) } } impl From<$primitive> for $wrapper { @@ -41,7 +39,7 @@ primitive!(U32, 32, u32); primitive!(U64, 64, u64); #[repr(transparent)] -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct Magic(pub [u8; MAGIC_LEN]); #[repr(transparent)] From 3b85a1d887900c83cb2557a05882984caaef65e1 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 14:15:27 +0100 Subject: [PATCH 05/23] Finish kernel-facing API. --- src/lib.rs | 65 +++++++++++++++++++++++++++---- src/types.rs | 6 +-- utils/src/redox_initfs_package.rs | 2 +- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index df56dba28a..fdc683541c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,9 @@ pub struct InitFs<'initfs> { base: &'initfs [u8], } +#[derive(Clone, Copy, Debug)] +pub struct Inode(u16); + #[derive(Clone, Copy, Debug)] pub struct Error; @@ -50,6 +53,59 @@ impl<'initfs> InodeDir<'initfs> { pub fn inode(self) -> InodeStruct<'initfs> { self.inner } + pub fn entry_count(&self) -> Result { + let len = self.entries()?.len(); + + // NOTE: Len is originally stored as a u32 in the struct, so it can never exceed u32 + // despite first being converted to usize. + let len = len as u32; + + Ok(len) + } + pub fn get_entry(&self, idx: u32) -> Result>> { + let idx = usize::try_from(idx).map_err(|_| Error)?; + + self.entries().map(|entries| { + let entry = entries.get(idx)?; + + Some(Entry { + entry, + initfs: self.inner.initfs, + }) + }) + } + fn entries(&self) -> Result<&'initfs [DirEntry]> { + let bytes = self.inner.data()?; + let entries = plain::slice_from_bytes::(bytes) + .expect("expected dir entry to have alignment 1"); + + Ok(entries) + } +} + +pub struct Entry<'initfs> { + initfs: InitFs<'initfs>, + entry: &'initfs DirEntry, +} + +impl<'initfs> Entry<'initfs> { + pub fn inode(&self) -> Inode { + Inode(self.entry.inode.get()) + } + pub fn name(&self) -> Result<&'initfs [u8]> { + let name_offset: usize = self + .entry + .name_offset + .0 + .get() + .try_into() + .map_err(|_| Error)?; + let name_length: usize = self.entry.name_len.get().try_into().map_err(|_| Error)?; + + let name_end = name_offset.checked_add(name_length).ok_or(Error)?; + + self.initfs.base.get(name_offset..name_end).ok_or(Error) + } } #[derive(Clone, Copy)] @@ -153,11 +209,6 @@ impl<'initfs> InitFs<'initfs> { .try_into() .expect("expected inode struct size to fit within u8") } - fn direntry_struct_len_8() -> u8 { - core::mem::size_of::() - .try_into() - .expect("expected dir entry struct size to fit within u8") - } fn inode_table_offset_usize(&self) -> usize { // NOTE: We have already validated that the inode table fits within the initfs slice, and // that this length, which must fit within usize, also fits within u32. @@ -190,7 +241,7 @@ impl<'initfs> InitFs<'initfs> { plain::slice_from_bytes::(inode_table_bytes) .expect("expected inode struct alignment to be 1") } - pub const ROOT_INODE: Inode = Inode(U16::new(0)); + pub const ROOT_INODE: Inode = Inode(0); pub fn inode_count(&self) -> u16 { self.get_header_assume_valid().inode_count.get() @@ -199,7 +250,7 @@ impl<'initfs> InitFs<'initfs> { // NOTE: Even for 16-bit architectures (obviously edge-case, but some bootloaders may // perhaps use this code), we have already checked that the inode table can fit within // usize, and the table byte size is always larger than the count. - let inode_usize = inode.0.get() as usize; + let inode_usize = inode.0 as usize; let inode = self.inode_table().get(inode_usize)?; diff --git a/src/types.rs b/src/types.rs index cbddb7ed48..405387f1e3 100644 --- a/src/types.rs +++ b/src/types.rs @@ -50,10 +50,6 @@ pub struct Offset(pub U32); #[derive(Clone, Copy, Debug)] pub struct Length(pub U32); -#[repr(transparent)] -#[derive(Clone, Copy, Debug)] -pub struct Inode(pub U16); - #[repr(packed)] #[derive(Clone, Copy, Debug)] pub struct Timespec { @@ -97,7 +93,7 @@ pub enum InodeType { #[repr(packed)] #[derive(Clone, Copy, Debug)] pub struct DirEntry { - pub inode: Inode, + pub inode: U16, pub name_len: U16, pub name_offset: Offset, } diff --git a/utils/src/redox_initfs_package.rs b/utils/src/redox_initfs_package.rs index d438c78e7f..447b73c947 100644 --- a/utils/src/redox_initfs_package.rs +++ b/utils/src/redox_initfs_package.rs @@ -294,7 +294,7 @@ fn allocate_and_write_dir( let inode = this_start_inode + index; *direntry = initfs::DirEntry { - inode: initfs::Inode(inode.into()), + inode: inode.into(), name_len: name_len.into(), name_offset: initfs::Offset(name_offset.into()), }; From 5e9e2e9ca8e2fcdefd4c73012b9b1802b192579c Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 14:33:37 +0100 Subject: [PATCH 06/23] Rename utils into redox-initfs-ar. --- Cargo.toml | 2 +- {utils => redox-initfs-ar}/Cargo.toml | 7 ++----- .../src/main.rs | 8 +++++--- 3 files changed, 8 insertions(+), 9 deletions(-) rename {utils => redox-initfs-ar}/Cargo.toml (75%) rename utils/src/redox_initfs_package.rs => redox-initfs-ar/src/main.rs (98%) diff --git a/Cargo.toml b/Cargo.toml index 160d9d4358..46c5f20404 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,5 +19,5 @@ kernel = [] [workspace] members = [ - "utils", + "redox-initfs-ar", ] diff --git a/utils/Cargo.toml b/redox-initfs-ar/Cargo.toml similarity index 75% rename from utils/Cargo.toml rename to redox-initfs-ar/Cargo.toml index 86f2970edf..5c149bfaf2 100644 --- a/utils/Cargo.toml +++ b/redox-initfs-ar/Cargo.toml @@ -1,15 +1,12 @@ [package] -name = "utils" +name = "redox-initfs-ar" version = "0.1.0" authors = ["4lDO2 <4lDO2@protonmail.com>"] edition = "2018" +description = "Archive a directory into a Redox initfs image" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[[bin]] -path = "src/redox_initfs_package.rs" -name = "redox-initfs-package" - [dependencies] anyhow = "1" clap = "2.33" diff --git a/utils/src/redox_initfs_package.rs b/redox-initfs-ar/src/main.rs similarity index 98% rename from utils/src/redox_initfs_package.rs rename to redox-initfs-ar/src/main.rs index 447b73c947..7a12caf930 100644 --- a/utils/src/redox_initfs_package.rs +++ b/redox-initfs-ar/src/main.rs @@ -337,15 +337,17 @@ fn allocate_contents_and_write_inodes( } fn main() -> Result<()> { - let matches = App::new("redox_initfs_package") - .help("Package a Redox initfs") + let matches = App::new(clap::crate_name!()) + .about(clap::crate_description!()) + .version(clap::crate_version!()) + .author(clap::crate_authors!()) .arg( Arg::with_name("MAX_SIZE") .long("--max-size") .short("-m") .takes_value(true) .required(false) - .help("Set the upper limit for how large the image can become (default 8 MiB)."), + .help("Set the upper limit for how large the image can become (default 64 MiB)."), ) .arg( Arg::with_name("SOURCE") From ef6c8805956a148801c333e36e1f04dc1b50975f Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 14:35:45 +0100 Subject: [PATCH 07/23] Add license. --- Cargo.toml | 1 + LICENSE | 21 +++++++++++++++++++++ redox-initfs-ar/Cargo.toml | 1 + 3 files changed, 23 insertions(+) create mode 100644 LICENSE diff --git a/Cargo.toml b/Cargo.toml index 46c5f20404..3360d5ccb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "redox-initfs" version = "0.1.0" authors = ["4lDO2 <4lDO2@protonmail.com>"] edition = "2018" +license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..bfcdf07f4e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 4lDO2 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/redox-initfs-ar/Cargo.toml b/redox-initfs-ar/Cargo.toml index 5c149bfaf2..8cc3496070 100644 --- a/redox-initfs-ar/Cargo.toml +++ b/redox-initfs-ar/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["4lDO2 <4lDO2@protonmail.com>"] edition = "2018" description = "Archive a directory into a Redox initfs image" +license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From ae65c19bad96a471266958791ebe6324bd0a1f30 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 20 Feb 2021 15:04:40 +0100 Subject: [PATCH 08/23] Put the output file handle behind a guard. This prevents the image from leaking, in case there is an error, as it removes the file when the guard is dropped, before the "ok" status is set. --- redox-initfs-ar/src/main.rs | 46 +++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/redox-initfs-ar/src/main.rs b/redox-initfs-ar/src/main.rs index 7a12caf930..707ef5e770 100644 --- a/redox-initfs-ar/src/main.rs +++ b/redox-initfs-ar/src/main.rs @@ -29,8 +29,8 @@ struct Dir { entries: Vec, } -struct State { - file: File, +struct State<'path> { + file: OutputImageGuard<'path>, offset: u64, max_size: u64, inode_count: u16, @@ -336,6 +336,33 @@ fn allocate_contents_and_write_inodes( ) } +struct OutputImageGuard<'a> { + file: File, + path: &'a Path, + ok: bool, +} + +impl std::ops::Deref for OutputImageGuard<'_> { + type Target = File; + + fn deref(&self) -> &Self::Target { + &self.file + } +} +impl std::ops::DerefMut for OutputImageGuard<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.file + } +} + +impl Drop for OutputImageGuard<'_> { + fn drop(&mut self) { + if !self.ok { + let _ = std::fs::remove_file(self.path); + } + } +} + fn main() -> Result<()> { let matches = App::new(clap::crate_name!()) .about(clap::crate_description!()) @@ -403,13 +430,19 @@ fn main() -> Result<()> { .write(true) .create(true) .create_new(false) - .open(destination_temp_path) + .open(&destination_temp_path) .context("failed to open destination file")?; + let guard = OutputImageGuard { + file: destination_temp_file, + path: &destination_temp_path, + ok: false, + }; + const BUFFER_SIZE: usize = 8192; let mut state = State { - file: destination_temp_file, + file: guard, offset: 0, max_size, inode_count: 0, @@ -483,5 +516,10 @@ fn main() -> Result<()> { .context("failed to write header")?; } + std::fs::rename(&destination_temp_path, destination_path) + .context("failed to rename output image")?; + + state.file.ok = true; + Ok(()) } From 811f58cc9aebe54dce7ea7b1d98aeefe1e04c61e Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Tue, 29 Jun 2021 21:09:43 +0200 Subject: [PATCH 09/23] Add #![no_std]. --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index fdc683541c..1da1594a97 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![no_std] //! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be //! read. From 0a41ea5b8f43b9a89e1d78adee281d1547deaa75 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Wed, 30 Jun 2021 12:13:19 +0200 Subject: [PATCH 10/23] Implement more traits for Inode. --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 1da1594a97..696d2a6540 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ pub struct InitFs<'initfs> { base: &'initfs [u8], } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct Inode(u16); #[derive(Clone, Copy, Debug)] From 312499d042f9c776cf693d0d038af5af7df9dc88 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Wed, 30 Jun 2021 12:33:28 +0200 Subject: [PATCH 11/23] Derive Clone and Copy for Entry. --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 696d2a6540..a84d9e831a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -84,6 +84,7 @@ impl<'initfs> InodeDir<'initfs> { } } +#[derive(Clone, Copy)] pub struct Entry<'initfs> { initfs: InitFs<'initfs>, entry: &'initfs DirEntry, From 08d18160ee7f277c35e7b5dee11b83272182f789 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Fri, 2 Jul 2021 14:22:35 +0200 Subject: [PATCH 12/23] Fix overwrite of image while writing inodes. --- .gitignore | 3 + Cargo.toml | 9 +- data/file.txt | 1 + redox-initfs-ar/Cargo.toml | 2 + redox-initfs-ar/src/archive.rs | 489 +++++++++++++++++++++++++++++++++ redox-initfs-ar/src/main.rs | 481 +------------------------------- src/lib.rs | 5 +- tests/archive_and_read.rs | 46 ++++ 8 files changed, 562 insertions(+), 474 deletions(-) create mode 100644 data/file.txt create mode 100644 redox-initfs-ar/src/archive.rs create mode 100644 tests/archive_and_read.rs diff --git a/.gitignore b/.gitignore index 96ef6c0b94..a13600d6ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /target Cargo.lock +# TODO: Maybe use an OUT_DIR environment variable, or mktemp, rather than +# writing directly to the git repository root? +/out.img diff --git a/Cargo.toml b/Cargo.toml index 3360d5ccb4..d27ae0e6da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,13 +11,20 @@ license = "MIT" plain = "0.2" [features] -default = [] +default = ["std"] + +std = [] # Enables functionality to actually read the filesystem, which is only done in # the kernel. (Writing to the filesystem is done in utils, and that crate only # needs the type definitions). kernel = [] +[dev-dependencies] +log = "0.4" +env_logger = "0.8" +anyhow = "1" + [workspace] members = [ "redox-initfs-ar", diff --git a/data/file.txt b/data/file.txt new file mode 100644 index 0000000000..98eb3df43f --- /dev/null +++ b/data/file.txt @@ -0,0 +1 @@ +This is a file meant to be used in a redox-initfs test. diff --git a/redox-initfs-ar/Cargo.toml b/redox-initfs-ar/Cargo.toml index 8cc3496070..7b711b9ff4 100644 --- a/redox-initfs-ar/Cargo.toml +++ b/redox-initfs-ar/Cargo.toml @@ -11,6 +11,8 @@ license = "MIT" [dependencies] anyhow = "1" clap = "2.33" +env_logger = "0.8" +log = "0.4" plain = "0.2" redox-initfs = { path = ".." } diff --git a/redox-initfs-ar/src/archive.rs b/redox-initfs-ar/src/archive.rs new file mode 100644 index 0000000000..b006fa6154 --- /dev/null +++ b/redox-initfs-ar/src/archive.rs @@ -0,0 +1,489 @@ +use std::convert::{TryFrom, TryInto}; +use std::fs::{DirEntry, File, Metadata, OpenOptions}; +use std::io::{prelude::*, SeekFrom}; +use std::path::Path; + +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt}; + +use anyhow::{anyhow, Context, Result}; + +use redox_initfs::types as initfs; + +pub const KIBIBYTE: u64 = 1024; +pub const MEBIBYTE: u64 = KIBIBYTE * 1024; +pub const DEFAULT_MAX_SIZE: u64 = 64 * MEBIBYTE; + +enum EntryKind { + File(File), + Dir(Dir), +} + +struct Entry { + name: Vec, + kind: EntryKind, + metadata: Metadata, +} +struct Dir { + entries: Vec, +} + +struct State<'path> { + file: OutputImageGuard<'path>, + offset: u64, + max_size: u64, + inode_count: u16, + buffer: Box<[u8]>, +} + +fn write_all_at(file: &File, buf: &[u8], offset: u64, r#where: &str) -> Result<()> { + file.write_all_at(buf, offset)?; + log::debug!("Wrote {}..{} within {}", offset, offset + buf.len() as u64, r#where); + Ok(()) +} + +fn read_directory(state: &mut State, path: &Path) -> Result { + let read_dir = path + .read_dir() + .with_context(|| anyhow!("failed to read directory `{}`", path.to_string_lossy(),))?; + + let entries = read_dir + .map(|result| { + let entry = result.with_context(|| { + anyhow!( + "failed to get a directory entry from `{}`", + path.to_string_lossy(), + ) + })?; + + let metadata = entry.metadata().with_context(|| { + anyhow!( + "failed to get metadata for `{}`", + entry.path().to_string_lossy(), + ) + })?; + let file_type = metadata.file_type(); + + let unsupported_type = |ty: &str, entry: &DirEntry| { + Err(anyhow!( + "failed to include {} at `{}`: not supported by redox-initfs", + ty, + entry.path().to_string_lossy() + )) + }; + let name = entry + .path() + .file_name() + .context("expected path to have a valid filename")? + .as_bytes() + .to_owned(); + + let entry_kind = if file_type.is_symlink() { + return unsupported_type("symlink", &entry); + } else if file_type.is_socket() { + return unsupported_type("socket", &entry); + } else if file_type.is_fifo() { + return unsupported_type("FIFO", &entry); + } else if file_type.is_block_device() { + return unsupported_type("block device", &entry); + } else if file_type.is_char_device() { + return unsupported_type("character device", &entry); + } else if file_type.is_file() { + EntryKind::File(File::open(&entry.path()).with_context(|| { + anyhow!("failed to open file `{}`", entry.path().to_string_lossy(),) + })?) + } else if file_type.is_dir() { + EntryKind::Dir(read_directory(state, &entry.path())?) + } else { + return Err(anyhow!( + "unknown file type at `{}`", + entry.path().to_string_lossy() + )); + }; + + // TODO: Allow the user to specify a lower limit than u16::MAX. + state.inode_count = state + .inode_count + .checked_add(1) + .ok_or_else(|| anyhow!("exceeded the maximum inode limit"))?; + + Ok(Entry { + kind: entry_kind, + metadata, + name, + }) + }) + .collect::>>()?; + + Ok(Dir { entries }) +} + +fn bump_alloc(state: &mut State, size: u64, why: &str) -> Result { + if state.offset + size <= state.max_size { + let offset = state.offset; + state.offset += size; + log::debug!("Allocating range {}..{} in {}", offset, state.offset, why); + Ok(offset) + } else { + Err(anyhow!("bump allocation failed: max limit reached")) + } +} +struct WriteResult { + size: u32, + offset: u32, +} + +fn allocate_and_write_file(state: &mut State, mut file: &File) -> Result { + let size = file + .seek(SeekFrom::End(0)) + .context("failed to seek to end")?; + + let size: u32 = size.try_into().context("file too large")?; + + let offset: u32 = bump_alloc(state, size.into(), "allocate space for file") + .context("failed to allocate space for file")? + .try_into() + .context("file offset too high")?; + + let buffer_size: u32 = state.buffer.len().try_into().context("buffer too large")?; + + file.seek(SeekFrom::Start(0)) + .context("failed to seek to start")?; + + let mut relative_offset = 0; + + // TODO: If this would ever turn out to be a bottleneck, then perhaps we could use + // copy_file_range in `nix`. + + while relative_offset < size { + let allowed_length = std::cmp::min(buffer_size, size - relative_offset); + let allowed_length = + usize::try_from(allowed_length).expect("expected buffer size not to be outside usize"); + + file.read(&mut state.buffer[..allowed_length]) + .context("failed to read from source file")?; + + write_all_at(&*state.file, &state.buffer[..allowed_length], u64::from(offset + relative_offset), "allocate_and_write_file buffer chunk") + .context("failed to write source file into destination image")?; + + relative_offset += buffer_size; + } + + Ok(WriteResult { size, offset }) +} +fn write_inode( + state: &mut State, + ty: initfs::InodeType, + metadata: &Metadata, + write_result: WriteResult, + inode_table_offset: u32, + index: u16, +) -> Result<()> { + let inode_size: u32 = std::mem::size_of::() + .try_into() + .expect("inode header length cannot fit within u32"); + + let type_and_mode = ((ty as u32) << initfs::TYPE_SHIFT) | u32::from(metadata.mode() & 0xFFF); + + // TODO: Use main buffer and write in bulk. + let mut inode_buf = [0_u8; std::mem::size_of::()]; + + let inode = plain::from_mut_bytes::(&mut inode_buf) + .expect("expected inode struct to have alignment 1, and buffer size to match"); + + *inode = initfs::InodeHeader { + type_and_mode: type_and_mode.into(), + length: write_result.size.into(), + offset: initfs::Offset(write_result.offset.into()), + + gid: metadata.gid().into(), + uid: metadata.uid().into(), + }; + + log::debug!("Writing inode index {} from offset {}", index, inode_table_offset); + write_all_at( + &*state.file, + &inode_buf, + u64::from(inode_table_offset + u32::from(index) * inode_size), + "write_inode", + ) + .context("failed to write inode struct to disk image") +} +fn allocate_and_write_dir( + state: &mut State, + dir: &Dir, + inode_table_offset: u32, + start_inode: &mut u16, +) -> Result { + let entry_size = + u16::try_from(std::mem::size_of::()).context("entry size too large")?; + let entry_count = u16::try_from(dir.entries.len()).context("too many subdirectories")?; + + let entry_table_length = u32::from(entry_count) + .checked_mul(u32::from(entry_size)) + .ok_or_else(|| anyhow!("entry table length too large when multiplying by size"))?; + + let entry_table_offset: u32 = bump_alloc(state, entry_table_length.into(), "allocate entry table") + .context("failed to allocate entry table")? + .try_into() + .context("directory entries offset too high")?; + + let this_start_inode = *start_inode; + + let inode_size: u32 = std::mem::size_of::() + .try_into() + .expect("inode header length cannot fit within u32"); + + *start_inode += entry_count; + + let inode_table_offset = + inode_table_offset + u32::from(this_start_inode) * u32::from(inode_size); + + for (index, entry) in dir.entries.iter().enumerate() { + let (write_result, ty) = match entry.kind { + EntryKind::Dir(ref subdir) => { + let write_result = + allocate_and_write_dir(state, subdir, inode_table_offset, start_inode) + .with_context(|| { + anyhow!( + "failed to copy directory entries from `{}` into image", + String::from_utf8_lossy(&entry.name) + ) + })?; + + (write_result, initfs::InodeType::Dir) + } + + EntryKind::File(ref file) => { + let write_result = allocate_and_write_file(state, file) + .context("failed to copy file into image")?; + + (write_result, initfs::InodeType::RegularFile) + } + }; + + let index: u16 = index + .try_into() + .expect("expected dir entry count not to exceed u32"); + + write_inode( + state, + ty, + &entry.metadata, + write_result, + inode_table_offset, + index, + )?; + + let (name_offset, name_len) = { + let name_len: u16 = entry.name.len().try_into().context("file name too long")?; + + let offset: u32 = bump_alloc(state, u64::from(name_len), "allocate file name") + .context("failed to allocate space for file name")? + .try_into() + .context("file name offset too high up")?; + + write_all_at(&*state.file, &entry.name, offset.into(), "writing file name").context("failed to write file name")?; + + (offset, name_len) + }; + { + let mut direntry_buf = [0_u8; std::mem::size_of::()]; + + let direntry = plain::from_mut_bytes::(&mut direntry_buf) + .expect("expected dir entry struct to have alignment 1, and buffer size to match"); + + let inode = this_start_inode + index; + + *direntry = initfs::DirEntry { + inode: inode.into(), + name_len: name_len.into(), + name_offset: initfs::Offset(name_offset.into()), + }; + + write_all_at( + &*state.file, + &direntry_buf, + u64::from(entry_table_offset + u32::from(index) * u32::from(entry_size)), + "allocate_and_write_dir entry", + ) + .context("failed to write dir entry struct to image")?; + } + } + + Ok(WriteResult { + size: entry_table_length, + offset: entry_table_offset, + }) +} +fn allocate_contents_and_write_inodes( + state: &mut State, + dir: &Dir, + inode_table_offset: u32, + root_metadata: Metadata, +) -> Result<()> { + let index = 0; + let mut start_inode = 1; + + let write_result = allocate_and_write_dir(state, dir, inode_table_offset, &mut start_inode) + .context("failed to allocate and write all directories and files")?; + + write_inode( + state, + initfs::InodeType::Dir, + &root_metadata, + write_result, + inode_table_offset, + index, + ) +} + +struct OutputImageGuard<'a> { + file: File, + path: &'a Path, + ok: bool, +} + +impl std::ops::Deref for OutputImageGuard<'_> { + type Target = File; + + fn deref(&self) -> &Self::Target { + &self.file + } +} +impl std::ops::DerefMut for OutputImageGuard<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.file + } +} + +impl Drop for OutputImageGuard<'_> { + fn drop(&mut self) { + if !self.ok { + let _ = std::fs::remove_file(self.path); + } + } +} + +pub struct Args<'a> { + pub destination_path: &'a Path, + pub max_size: u64, + pub source: &'a Path, +} +pub fn archive(&Args { destination_path, max_size, source }: &Args) -> Result<()> { + let previous_extension = destination_path.extension().map_or("", |ext| { + ext.to_str() + .expect("expected destination path to be valid UTF-8") + }); + + if !destination_path + .metadata() + .map_or(true, |metadata| metadata.is_file()) + { + return Err(anyhow!("Destination file must be a file")); + } + + let destination_temp_path = + destination_path.with_extension(format!("{}.partial", previous_extension)); + + let destination_temp_file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .create_new(false) + .open(&destination_temp_path) + .context("failed to open destination file")?; + + let guard = OutputImageGuard { + file: destination_temp_file, + path: &destination_temp_path, + ok: false, + }; + + const BUFFER_SIZE: usize = 8192; + + let mut state = State { + file: guard, + offset: 0, + max_size, + // Include root directory. + inode_count: 1, + buffer: vec![0_u8; BUFFER_SIZE].into_boxed_slice(), + }; + + let root_path = source; + let root_metadata = root_path + .metadata() + .context("failed to obtain metadata for root")?; + let root = read_directory(&mut state, root_path).context("failed to read root")?; + + log::debug!("there are {} inodes", state.inode_count); + + // NOTE: The header is always stored at offset zero. + let header_offset = bump_alloc( + &mut state, + std::mem::size_of::() + .try_into() + .expect("expected header size to fit"), + "allocate header", + )?; + assert_eq!(header_offset, 0); + + let inode_table_length = { + let inode_entry_size: u64 = std::mem::size_of::() + .try_into() + .expect("expected table entry size to fit"); + + inode_entry_size + .checked_mul(u64::from(state.inode_count)) + .ok_or_else(|| anyhow!("inode table too large"))? + }; + + let inode_table_offset = bump_alloc(&mut state, inode_table_length, "allocate inode table")?; + + // Finally, write the header to the disk image. + + let inode_table_offset = initfs::Offset( + u32::try_from(inode_table_offset) + .with_context(|| "inode table located too far away")? + .into(), + ); + + allocate_contents_and_write_inodes( + &mut state, + &root, + inode_table_offset.0.get(), + root_metadata, + )?; + + let current_system_time = std::time::SystemTime::now(); + + let time_since_epoch = current_system_time + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .context("could not calculate timestamp")?; + + { + let mut header_bytes = [0_u8; std::mem::size_of::()]; + let header = plain::from_mut_bytes(&mut header_bytes) + .expect("expected header size to be sufficient and alignment to be 1"); + + *header = initfs::Header { + magic: initfs::Magic(initfs::MAGIC), + creation_time: initfs::Timespec { + sec: time_since_epoch.as_secs().into(), + nsec: time_since_epoch.subsec_nanos().into(), + }, + inode_count: state.inode_count.into(), + inode_table_offset, + }; + write_all_at(&*state.file, &header_bytes, header_offset, "writing header") + .context("failed to write header")?; + } + + std::fs::rename(&destination_temp_path, destination_path) + .context("failed to rename output image")?; + + state.file.ok = true; + + Ok(()) +} diff --git a/redox-initfs-ar/src/main.rs b/redox-initfs-ar/src/main.rs index 707ef5e770..0f3349edf0 100644 --- a/redox-initfs-ar/src/main.rs +++ b/redox-initfs-ar/src/main.rs @@ -1,367 +1,10 @@ -use std::convert::{TryFrom, TryInto}; -use std::fs::{DirEntry, File, Metadata, OpenOptions}; -use std::io::{prelude::*, SeekFrom}; use std::path::Path; -use std::os::unix::ffi::OsStrExt; -use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt}; - -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use clap::{App, Arg}; -use redox_initfs::types as initfs; - -const KIBIBYTE: u64 = 1024; -const MEBIBYTE: u64 = KIBIBYTE * 1024; -const DEFAULT_MAX_SIZE: u64 = 64 * MEBIBYTE; - -enum EntryKind { - File(File), - Dir(Dir), -} - -struct Entry { - name: Vec, - kind: EntryKind, - metadata: Metadata, -} -struct Dir { - entries: Vec, -} - -struct State<'path> { - file: OutputImageGuard<'path>, - offset: u64, - max_size: u64, - inode_count: u16, - buffer: Box<[u8]>, -} - -fn read_directory(state: &mut State, path: &Path) -> Result { - let read_dir = path - .read_dir() - .with_context(|| anyhow!("failed to read directory `{}`", path.to_string_lossy(),))?; - - let entries = read_dir - .map(|result| { - let entry = result.with_context(|| { - anyhow!( - "failed to get a directory entry from `{}`", - path.to_string_lossy(), - ) - })?; - - let metadata = entry.metadata().with_context(|| { - anyhow!( - "failed to get metadata for `{}`", - entry.path().to_string_lossy(), - ) - })?; - let file_type = metadata.file_type(); - - let unsupported_type = |ty: &str, entry: &DirEntry| { - Err(anyhow!( - "failed to include {} at `{}`: not supported by redox-initfs", - ty, - entry.path().to_string_lossy() - )) - }; - let name = entry - .path() - .file_name() - .context("expected path to have a valid filename")? - .as_bytes() - .to_owned(); - - let entry_kind = if file_type.is_symlink() { - return unsupported_type("symlink", &entry); - } else if file_type.is_socket() { - return unsupported_type("socket", &entry); - } else if file_type.is_fifo() { - return unsupported_type("FIFO", &entry); - } else if file_type.is_block_device() { - return unsupported_type("block device", &entry); - } else if file_type.is_char_device() { - return unsupported_type("character device", &entry); - } else if file_type.is_file() { - EntryKind::File(File::open(&entry.path()).with_context(|| { - anyhow!("failed to open file `{}`", entry.path().to_string_lossy(),) - })?) - } else if file_type.is_dir() { - EntryKind::Dir(read_directory(state, &entry.path())?) - } else { - return Err(anyhow!( - "unknown file type at `{}`", - entry.path().to_string_lossy() - )); - }; - - // TODO: Allow the user to specify a lower limit than u16::MAX. - state.inode_count = state - .inode_count - .checked_add(1) - .ok_or_else(|| anyhow!("exceeded the maximum inode limit"))?; - - Ok(Entry { - kind: entry_kind, - metadata, - name, - }) - }) - .collect::>>()?; - - Ok(Dir { entries }) -} - -fn bump_alloc(state: &mut State, size: u64) -> Result { - if state.offset + size <= state.max_size { - let offset = state.offset; - state.offset += size; - Ok(offset) - } else { - Err(anyhow!("bump allocation failed: max limit reached")) - } -} -struct WriteResult { - size: u32, - offset: u32, -} - -fn allocate_and_write_file(state: &mut State, mut file: &File) -> Result { - let size = file - .seek(SeekFrom::End(0)) - .context("failed to seek to end")?; - - let size: u32 = size.try_into().context("file too large")?; - - let offset: u32 = bump_alloc(state, size.into()) - .context("failed to allocate space for file")? - .try_into() - .context("file offset too high")?; - - let buffer_size: u32 = state.buffer.len().try_into().context("buffer too large")?; - - file.seek(SeekFrom::Start(0)) - .context("failed to seek to start")?; - - state - .file - .seek(SeekFrom::Start(offset.into())) - .context("failed to seek to offset for image")?; - - let mut relative_offset = 0; - - // TODO: If this would ever turn out to be a bottleneck, then perhaps we could use - // copy_file_range in `nix`. - - while relative_offset < size { - let allowed_length = std::cmp::min(buffer_size, size - relative_offset); - let allowed_length = - usize::try_from(allowed_length).expect("expected buffer size not to be outside usize"); - - file.read(&mut state.buffer[..allowed_length]) - .context("failed to read from source file")?; - - state - .file - .write(&state.buffer[..allowed_length]) - .context("failed to write source file into destination image")?; - - relative_offset += buffer_size; - } - - Ok(WriteResult { size, offset }) -} -fn write_inode( - state: &mut State, - ty: initfs::InodeType, - metadata: &Metadata, - write_result: WriteResult, - inode_table_offset: u32, - index: u16, -) -> Result<()> { - let inode_size: u32 = std::mem::size_of::() - .try_into() - .expect("inode header length cannot fit within u32"); - - let type_and_mode = ((ty as u32) << initfs::TYPE_SHIFT) | u32::from(metadata.mode() & 0xFFF); - - // TODO: Use main buffer and write in bulk. - let mut inode_buf = [0_u8; std::mem::size_of::()]; - - let inode = plain::from_mut_bytes::(&mut inode_buf) - .expect("expected inode struct to have alignment 1, and buffer size to match"); - - *inode = initfs::InodeHeader { - type_and_mode: type_and_mode.into(), - length: write_result.size.into(), - offset: initfs::Offset(write_result.offset.into()), - - gid: metadata.gid().into(), - uid: metadata.uid().into(), - }; - - state - .file - .write_all_at( - &inode_buf, - u64::from(inode_table_offset + u32::from(index) * inode_size), - ) - .context("failed to write inode struct to disk image") -} -fn allocate_and_write_dir( - state: &mut State, - dir: &Dir, - inode_table_offset: u32, - start_inode: &mut u16, -) -> Result { - let entry_size = - u16::try_from(std::mem::size_of::()).context("entry size too large")?; - let entry_count = u16::try_from(dir.entries.len()).context("too many subdirectories")?; - - let entry_table_length = u32::from(entry_count) - .checked_mul(u32::from(entry_size)) - .ok_or_else(|| anyhow!("entry table length too large when multiplying by size"))?; - - let entry_table_offset: u32 = bump_alloc(state, entry_table_length.into()) - .context("failed to allocate entry table")? - .try_into() - .context("directory entries offset too high")?; - - let this_start_inode = *start_inode; - - let inode_size: u32 = std::mem::size_of::() - .try_into() - .expect("inode header length cannot fit within u32"); - - *start_inode += entry_count; - - let inode_table_offset = - inode_table_offset + u32::from(this_start_inode) * u32::from(inode_size); - - for (index, entry) in dir.entries.iter().enumerate() { - let (write_result, ty) = match entry.kind { - EntryKind::Dir(ref subdir) => { - let write_result = - allocate_and_write_dir(state, subdir, inode_table_offset, start_inode) - .with_context(|| { - anyhow!( - "failed to copy directory entries from `{}` into image", - String::from_utf8_lossy(&entry.name) - ) - })?; - - (write_result, initfs::InodeType::Dir) - } - - EntryKind::File(ref file) => { - let write_result = allocate_and_write_file(state, file) - .context("failed to copy file into image")?; - - (write_result, initfs::InodeType::RegularFile) - } - }; - - let index: u16 = index - .try_into() - .expect("expected dir entry count not to exceed u32"); - - write_inode( - state, - ty, - &entry.metadata, - write_result, - inode_table_offset, - index, - )?; - - let (name_offset, name_len) = { - let name_len: u16 = entry.name.len().try_into().context("file name too long")?; - - let offset: u32 = bump_alloc(state, u64::from(name_len)) - .context("failed to allocate space for file name")? - .try_into() - .context("file name offset too high up")?; - - (offset, name_len) - }; - { - let mut direntry_buf = [0_u8; std::mem::size_of::()]; - - let direntry = plain::from_mut_bytes::(&mut direntry_buf) - .expect("expected dir entry struct to have alignment 1, and buffer size to match"); - - let inode = this_start_inode + index; - - *direntry = initfs::DirEntry { - inode: inode.into(), - name_len: name_len.into(), - name_offset: initfs::Offset(name_offset.into()), - }; - - state - .file - .write_all_at( - &direntry_buf, - u64::from(entry_table_offset + u32::from(index) * u32::from(entry_size)), - ) - .context("failed to write dir entry struct to image")?; - } - } - - Ok(WriteResult { - size: entry_table_length, - offset: entry_table_offset, - }) -} -fn allocate_contents_and_write_inodes( - state: &mut State, - dir: &Dir, - inode_table_offset: u32, - root_metadata: Metadata, -) -> Result<()> { - let index = 0; - let mut start_inode = 1; - - let write_result = allocate_and_write_dir(state, dir, inode_table_offset, &mut start_inode) - .context("failed to allocate and write all directories and files")?; - - write_inode( - state, - initfs::InodeType::Dir, - &root_metadata, - write_result, - inode_table_offset, - index, - ) -} - -struct OutputImageGuard<'a> { - file: File, - path: &'a Path, - ok: bool, -} - -impl std::ops::Deref for OutputImageGuard<'_> { - type Target = File; - - fn deref(&self) -> &Self::Target { - &self.file - } -} -impl std::ops::DerefMut for OutputImageGuard<'_> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.file - } -} - -impl Drop for OutputImageGuard<'_> { - fn drop(&mut self) { - if !self.ok { - let _ = std::fs::remove_file(self.path); - } - } -} +mod archive; +use self::archive::*; fn main() -> Result<()> { let matches = App::new(clap::crate_name!()) @@ -392,6 +35,8 @@ fn main() -> Result<()> { ) .get_matches(); + env_logger::init(); + let max_size = if let Some(max_size_str) = matches.value_of("MAX_SIZE") { max_size_str .parse::() @@ -408,118 +53,10 @@ fn main() -> Result<()> { .value_of("OUTPUT") .expect("expected the required arg OUTPUT to exist"); - let destination_path = Path::new(destination); - - let previous_extension = destination_path.extension().map_or("", |ext| { - ext.to_str() - .expect("expected destination path to be valid UTF-8") - }); - - if !destination_path - .metadata() - .map_or(true, |metadata| metadata.is_file()) - { - return Err(anyhow!("Destination file must be a file")); - } - - let destination_temp_path = - destination_path.with_extension(format!("{}.partial", previous_extension)); - - let destination_temp_file = OpenOptions::new() - .read(false) - .write(true) - .create(true) - .create_new(false) - .open(&destination_temp_path) - .context("failed to open destination file")?; - - let guard = OutputImageGuard { - file: destination_temp_file, - path: &destination_temp_path, - ok: false, - }; - - const BUFFER_SIZE: usize = 8192; - - let mut state = State { - file: guard, - offset: 0, + let args = Args { + source: Path::new(source), + destination_path: Path::new(destination), max_size, - inode_count: 0, - buffer: vec![0_u8; BUFFER_SIZE].into_boxed_slice(), }; - - let root_path = Path::new(source); - let root_metadata = root_path - .metadata() - .context("failed to obtain metadata for root")?; - let root = read_directory(&mut state, root_path).context("failed to read root")?; - - // NOTE: The header is always stored at offset zero. - let header_offset = bump_alloc( - &mut state, - std::mem::size_of::() - .try_into() - .expect("expected header size to fit"), - )?; - - let inode_table_length = { - let inode_entry_size: u64 = std::mem::size_of::() - .try_into() - .expect("expected table entry size to fit"); - - inode_entry_size - .checked_mul(u64::from(state.inode_count)) - .ok_or_else(|| anyhow!("inode table too large"))? - }; - - let inode_table_offset = bump_alloc(&mut state, inode_table_length)?; - - // Finally, write the header to the disk image. - - let inode_table_offset = initfs::Offset( - u32::try_from(inode_table_offset) - .with_context(|| "inode table located too far away")? - .into(), - ); - - allocate_contents_and_write_inodes( - &mut state, - &root, - inode_table_offset.0.get(), - root_metadata, - )?; - - let current_system_time = std::time::SystemTime::now(); - - let time_since_epoch = current_system_time - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .context("could not calculate timestamp")?; - - { - let mut header_bytes = [0_u8; std::mem::size_of::()]; - let header = plain::from_mut_bytes(&mut header_bytes) - .expect("expected header size to be sufficient and alignment to be 1"); - - *header = initfs::Header { - magic: initfs::Magic(initfs::MAGIC), - creation_time: initfs::Timespec { - sec: time_since_epoch.as_secs().into(), - nsec: time_since_epoch.subsec_nanos().into(), - }, - inode_count: state.inode_count.into(), - inode_table_offset, - }; - state - .file - .write_all_at(&header_bytes, header_offset) - .context("failed to write header")?; - } - - std::fs::rename(&destination_temp_path, destination_path) - .context("failed to rename output image")?; - - state.file.ok = true; - - Ok(()) + self::archive::archive(&args) } diff --git a/src/lib.rs b/src/lib.rs index a84d9e831a..9a1682b800 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![no_std] +#![cfg_attr(not(any(test, feature = "std")), no_std)] //! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be //! read. @@ -25,6 +25,9 @@ impl core::fmt::Display for Error { } } +#[cfg(any(test, feature = "std"))] +impl std::error::Error for Error {} + type Result = core::result::Result; #[derive(Clone, Copy)] diff --git a/tests/archive_and_read.rs b/tests/archive_and_read.rs new file mode 100644 index 0000000000..d7bd1e2d3c --- /dev/null +++ b/tests/archive_and_read.rs @@ -0,0 +1,46 @@ +#![feature(array_methods)] + +use std::path::Path; + +use anyhow::{anyhow, bail, Context, Result}; + +#[path = "../redox-initfs-ar/src/archive.rs"] +mod archive; + +#[test] +fn archive_and_read() -> Result<()> { + env_logger::init(); + + let args = self::archive::Args { + destination_path: Path::new("out.img"), + source: Path::new("data"), + max_size: self::archive::DEFAULT_MAX_SIZE, + }; + self::archive::archive(&args).context("failed to archive")?; + + let data = std::fs::read(args.destination_path).context("failed to read new archive")?; + let filesystem = redox_initfs::InitFs::new(&data).context("failed to parse archive header")?; + let root_inode = filesystem.get_inode(redox_initfs::InitFs::ROOT_INODE).ok_or_else(|| anyhow!("Failed to get root inode"))?; + + let dir = match root_inode.kind() { + redox_initfs::InodeKind::Dir(dir) => dir, + _ => bail!("root inode was not a directory"), + }; + + for idx in 0..dir.entry_count().context("failed to get inode entry count")? { + let entry = dir.get_entry(idx).context("failed to get entry for index")?.ok_or_else(|| anyhow!("no entry found"))?; + + if entry.name().context("failed to get entry name")? == b"file.txt".as_slice() { + let inode = filesystem.get_inode(entry.inode()).context("failed to load file inode")?; + + let file = match inode.kind() { + redox_initfs::InodeKind::File(file) => file, + _ => bail!("file.txt was a directory"), + }; + let data = file.data().context("failed to get file.txt data")?; + assert_eq!(data, std::fs::read("data/file.txt").context("failed to read the real file.txt")?); + } + } + + Ok(()) +} From 89b8fb8984cf96c418880b7dcd9ce3d6afc3f71c Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sat, 26 Mar 2022 18:15:49 +0100 Subject: [PATCH 13/23] Refactor into a separate library and tools crate. --- Cargo.toml | 2 +- src/lib.rs | 7 ++ src/types.rs | 7 +- tests/archive_and_read.rs | 2 +- {redox-initfs-ar => tools}/Cargo.toml | 9 ++ .../archive.rs => tools/src/archive_common.rs | 56 +++++------ .../src/main.rs => tools/src/bin/archive.rs | 11 ++- tools/src/bin/dump.rs | 96 +++++++++++++++++++ 8 files changed, 148 insertions(+), 42 deletions(-) rename {redox-initfs-ar => tools}/Cargo.toml (74%) rename redox-initfs-ar/src/archive.rs => tools/src/archive_common.rs (91%) rename redox-initfs-ar/src/main.rs => tools/src/bin/archive.rs (86%) create mode 100644 tools/src/bin/dump.rs diff --git a/Cargo.toml b/Cargo.toml index d27ae0e6da..5de53b06c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,5 +27,5 @@ anyhow = "1" [workspace] members = [ - "redox-initfs-ar", + "tools", ] diff --git a/src/lib.rs b/src/lib.rs index 9a1682b800..50c63e892b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -204,6 +204,9 @@ impl<'initfs> InitFs<'initfs> { plain::from_bytes::
(&self.base[..core::mem::size_of::
()]) .expect("expected header type to require no alignment, and size to be sufficient") } + pub fn header(&self) -> &Header { + self.get_header_assume_valid() + } fn header_len_8() -> u8 { core::mem::size_of::
() .try_into() @@ -248,6 +251,10 @@ impl<'initfs> InitFs<'initfs> { } pub const ROOT_INODE: Inode = Inode(0); + pub fn all_inodes(&self) -> impl Iterator { + (0..self.inode_count()).map(Inode) + } + pub fn inode_count(&self) -> u16 { self.get_header_assume_valid().inode_count.get() } diff --git a/src/types.rs b/src/types.rs index 405387f1e3..24508fcfbc 100644 --- a/src/types.rs +++ b/src/types.rs @@ -4,7 +4,7 @@ pub const MAGIC: [u8; 8] = *b"RedoxFtw"; macro_rules! primitive( ($wrapper:ident, $bits:expr, $primitive:ident) => { #[repr(transparent)] - #[derive(Clone, Copy, Debug, Default)] + #[derive(Clone, Copy, Default)] pub struct $wrapper([u8; $bits / 8]); impl $wrapper { @@ -31,6 +31,11 @@ macro_rules! primitive( wrapper.get() } } + impl core::fmt::Debug for $wrapper { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{:#0width$x}", self.get(), width = 2 * core::mem::size_of::<$primitive>()) + } + } } ); diff --git a/tests/archive_and_read.rs b/tests/archive_and_read.rs index d7bd1e2d3c..d26df6c71a 100644 --- a/tests/archive_and_read.rs +++ b/tests/archive_and_read.rs @@ -4,7 +4,7 @@ use std::path::Path; use anyhow::{anyhow, bail, Context, Result}; -#[path = "../redox-initfs-ar/src/archive.rs"] +#[path = "../tools/src/archive_common.rs"] mod archive; #[test] diff --git a/redox-initfs-ar/Cargo.toml b/tools/Cargo.toml similarity index 74% rename from redox-initfs-ar/Cargo.toml rename to tools/Cargo.toml index 7b711b9ff4..82330525c9 100644 --- a/redox-initfs-ar/Cargo.toml +++ b/tools/Cargo.toml @@ -8,11 +8,20 @@ license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +path = "src/bin/archive.rs" +name = "redox-initfs-ar" + +[[bin]] +path = "src/bin/dump.rs" +name = "redox-initfs-dump" + [dependencies] anyhow = "1" clap = "2.33" env_logger = "0.8" log = "0.4" plain = "0.2" +twox-hash = "1.6" redox-initfs = { path = ".." } diff --git a/redox-initfs-ar/src/archive.rs b/tools/src/archive_common.rs similarity index 91% rename from redox-initfs-ar/src/archive.rs rename to tools/src/archive_common.rs index b006fa6154..a340aeecad 100644 --- a/redox-initfs-ar/src/archive.rs +++ b/tools/src/archive_common.rs @@ -34,11 +34,12 @@ struct State<'path> { max_size: u64, inode_count: u16, buffer: Box<[u8]>, + inode_table_offset: u32, } fn write_all_at(file: &File, buf: &[u8], offset: u64, r#where: &str) -> Result<()> { file.write_all_at(buf, offset)?; - log::debug!("Wrote {}..{} within {}", offset, offset + buf.len() as u64, r#where); + log::trace!("Wrote {}..{} within {}", offset, offset + buf.len() as u64, r#where); Ok(()) } @@ -176,8 +177,7 @@ fn write_inode( ty: initfs::InodeType, metadata: &Metadata, write_result: WriteResult, - inode_table_offset: u32, - index: u16, + inode: u16, ) -> Result<()> { let inode_size: u32 = std::mem::size_of::() .try_into() @@ -188,23 +188,23 @@ fn write_inode( // TODO: Use main buffer and write in bulk. let mut inode_buf = [0_u8; std::mem::size_of::()]; - let inode = plain::from_mut_bytes::(&mut inode_buf) + let inode_hdr = plain::from_mut_bytes::(&mut inode_buf) .expect("expected inode struct to have alignment 1, and buffer size to match"); - *inode = initfs::InodeHeader { + *inode_hdr = initfs::InodeHeader { type_and_mode: type_and_mode.into(), length: write_result.size.into(), offset: initfs::Offset(write_result.offset.into()), - gid: metadata.gid().into(), - uid: metadata.uid().into(), + gid: 0.into(),//metadata.gid().into(), + uid: 0.into(),//metadata.uid().into(), }; - log::debug!("Writing inode index {} from offset {}", index, inode_table_offset); + log::debug!("Writing inode index {} from offset {}", inode, state.inode_table_offset); write_all_at( &*state.file, &inode_buf, - u64::from(inode_table_offset + u32::from(index) * inode_size), + u64::from(state.inode_table_offset + u32::from(inode) * inode_size), "write_inode", ) .context("failed to write inode struct to disk image") @@ -212,8 +212,7 @@ fn write_inode( fn allocate_and_write_dir( state: &mut State, dir: &Dir, - inode_table_offset: u32, - start_inode: &mut u16, + current_inode: &mut u16, ) -> Result { let entry_size = u16::try_from(std::mem::size_of::()).context("entry size too large")?; @@ -228,22 +227,11 @@ fn allocate_and_write_dir( .try_into() .context("directory entries offset too high")?; - let this_start_inode = *start_inode; - - let inode_size: u32 = std::mem::size_of::() - .try_into() - .expect("inode header length cannot fit within u32"); - - *start_inode += entry_count; - - let inode_table_offset = - inode_table_offset + u32::from(this_start_inode) * u32::from(inode_size); - for (index, entry) in dir.entries.iter().enumerate() { let (write_result, ty) = match entry.kind { EntryKind::Dir(ref subdir) => { let write_result = - allocate_and_write_dir(state, subdir, inode_table_offset, start_inode) + allocate_and_write_dir(state, subdir, current_inode) .with_context(|| { anyhow!( "failed to copy directory entries from `{}` into image", @@ -266,13 +254,13 @@ fn allocate_and_write_dir( .try_into() .expect("expected dir entry count not to exceed u32"); + *current_inode += 1; write_inode( state, ty, &entry.metadata, write_result, - inode_table_offset, - index, + *current_inode, )?; let (name_offset, name_len) = { @@ -293,10 +281,10 @@ fn allocate_and_write_dir( let direntry = plain::from_mut_bytes::(&mut direntry_buf) .expect("expected dir entry struct to have alignment 1, and buffer size to match"); - let inode = this_start_inode + index; + log::debug!("Linking inode {} into dir entry index {}, file name `{}`", current_inode, index, String::from_utf8_lossy(&entry.name)); *direntry = initfs::DirEntry { - inode: inode.into(), + inode: (*current_inode).into(), name_len: name_len.into(), name_offset: initfs::Offset(name_offset.into()), }; @@ -319,13 +307,12 @@ fn allocate_and_write_dir( fn allocate_contents_and_write_inodes( state: &mut State, dir: &Dir, - inode_table_offset: u32, root_metadata: Metadata, ) -> Result<()> { - let index = 0; - let mut start_inode = 1; + let start_inode = 0; + let mut current_inode = start_inode; - let write_result = allocate_and_write_dir(state, dir, inode_table_offset, &mut start_inode) + let write_result = allocate_and_write_dir(state, dir, &mut current_inode) .context("failed to allocate and write all directories and files")?; write_inode( @@ -333,8 +320,7 @@ fn allocate_contents_and_write_inodes( initfs::InodeType::Dir, &root_metadata, write_result, - inode_table_offset, - index, + start_inode, ) } @@ -409,6 +395,7 @@ pub fn archive(&Args { destination_path, max_size, source }: &Args) -> Result<() // Include root directory. inode_count: 1, buffer: vec![0_u8; BUFFER_SIZE].into_boxed_slice(), + inode_table_offset: 0, }; let root_path = source; @@ -449,10 +436,11 @@ pub fn archive(&Args { destination_path, max_size, source }: &Args) -> Result<() .into(), ); + state.inode_table_offset = inode_table_offset.0.get(); + allocate_contents_and_write_inodes( &mut state, &root, - inode_table_offset.0.get(), root_metadata, )?; diff --git a/redox-initfs-ar/src/main.rs b/tools/src/bin/archive.rs similarity index 86% rename from redox-initfs-ar/src/main.rs rename to tools/src/bin/archive.rs index 0f3349edf0..ac14f11bcd 100644 --- a/redox-initfs-ar/src/main.rs +++ b/tools/src/bin/archive.rs @@ -3,12 +3,13 @@ use std::path::Path; use anyhow::{Context, Result}; use clap::{App, Arg}; -mod archive; -use self::archive::*; +#[path = "../archive_common.rs"] +mod archive_common; +use self::archive_common::{self as archive, Args, DEFAULT_MAX_SIZE}; fn main() -> Result<()> { - let matches = App::new(clap::crate_name!()) - .about(clap::crate_description!()) + let matches = App::new("redox-initfs-ar") + .about("create an initfs image from a directory") .version(clap::crate_version!()) .author(clap::crate_authors!()) .arg( @@ -58,5 +59,5 @@ fn main() -> Result<()> { destination_path: Path::new(destination), max_size, }; - self::archive::archive(&args) + archive::archive(&args) } diff --git a/tools/src/bin/dump.rs b/tools/src/bin/dump.rs new file mode 100644 index 0000000000..8f7c20af75 --- /dev/null +++ b/tools/src/bin/dump.rs @@ -0,0 +1,96 @@ +use anyhow::{Context, Result}; +use clap::{App, Arg}; + +use redox_initfs::InitFs; + +fn main() -> Result<()> { + let matches = App::new("redox-initfs-dump") + .about("dump initfs metadata") + .version(clap::crate_version!()) + .author(clap::crate_authors!()) + .arg( + Arg::with_name("IMAGE") + .takes_value(true) + .required(true) + .help("specify the image to dump") + ) + .get_matches(); + + let source = matches + .value_of_os("IMAGE") + .expect("expected the required arg IMAGE to exist"); + + let bytes = std::fs::read(source).context("failed to read image into memory")?; + let initfs = InitFs::new(&bytes).context("failed to parse initfs header")?; + + dbg!(initfs.header()); + + for inode in initfs.all_inodes() { + print!("{:?}: ", inode); + + let inode_struct = match initfs.get_inode(inode) { + Some(s) => s, + None => { + println!("failed to obtain."); + continue; + } + }; + print!("mode={:#0o}, uid={}, gid={}, kind=", inode_struct.mode(), inode_struct.uid(), inode_struct.gid()); + + use redox_initfs::InodeKind; + + match inode_struct.kind() { + InodeKind::Unknown => println!("(unknown)"), + InodeKind::Dir(dir) => { + print!("dir{{"); + let ec = match dir.entry_count().ok() { + Some(c) => c, + None => { + println!("(failed to get entry count)}}"); + continue; + } + }; + println!("entries=["); + + for entry in 0..ec { + let entry = match dir.get_entry(entry).ok().flatten() { + Some(e) => e, + None => { + println!("\t(unknown),"); + continue; + } + }; + let name = match entry.name().ok() { + Some(name) => name, + None => { + println!("\t(unknown name),"); + continue; + } + }; + println!("\t`{}` => {:?},", String::from_utf8_lossy(name), entry.inode()); + } + + println!("]}}"); + } + InodeKind::File(file) => { + print!("file{{"); + + match file.data().ok() { + Some(d) => { + use std::hash::Hasher; + let mut hasher = twox_hash::XxHash64::with_seed(0); + hasher.write(d); + print!("len={}, hash={:#0x}", d.len(), hasher.finish()); + } + None => { + print!("(failed to get data)"); + } + } + + println!("}}"); + } + } + } + + Ok(()) +} From 56642539bc168e997350203eaf2aac8f69690500 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:25:23 +0100 Subject: [PATCH 14/23] Update clap to v4 --- tools/Cargo.toml | 2 +- tools/src/bin/archive.rs | 27 ++++++++++++--------------- tools/src/bin/dump.rs | 26 ++++++++++++++++++-------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/tools/Cargo.toml b/tools/Cargo.toml index 82330525c9..fb9acfae24 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -18,7 +18,7 @@ name = "redox-initfs-dump" [dependencies] anyhow = "1" -clap = "2.33" +clap = { version = "4", features = ["cargo"] } env_logger = "0.8" log = "0.4" plain = "0.2" diff --git a/tools/src/bin/archive.rs b/tools/src/bin/archive.rs index ac14f11bcd..6fc48c66b4 100644 --- a/tools/src/bin/archive.rs +++ b/tools/src/bin/archive.rs @@ -1,44 +1,41 @@ use std::path::Path; use anyhow::{Context, Result}; -use clap::{App, Arg}; +use clap::{Arg, Command}; #[path = "../archive_common.rs"] mod archive_common; use self::archive_common::{self as archive, Args, DEFAULT_MAX_SIZE}; fn main() -> Result<()> { - let matches = App::new("redox-initfs-ar") + let matches = Command::new("redox-initfs-ar") .about("create an initfs image from a directory") .version(clap::crate_version!()) .author(clap::crate_authors!()) .arg( - Arg::with_name("MAX_SIZE") - .long("--max-size") - .short("-m") - .takes_value(true) + Arg::new("MAX_SIZE") + .long("max-size") + .short('m') .required(false) .help("Set the upper limit for how large the image can become (default 64 MiB)."), ) .arg( - Arg::with_name("SOURCE") - .takes_value(true) + Arg::new("SOURCE") .required(true) .help("Specify the source directory to build the image from."), ) .arg( - Arg::with_name("OUTPUT") - .takes_value(true) + Arg::new("OUTPUT") .required(true) - .long("--output") - .short("-o") + .long("output") + .short('o') .help("Specify the path of the new image file."), ) .get_matches(); env_logger::init(); - let max_size = if let Some(max_size_str) = matches.value_of("MAX_SIZE") { + let max_size = if let Some(max_size_str) = matches.get_one::("MAX_SIZE") { max_size_str .parse::() .context("expected an integer for MAX_SIZE")? @@ -47,11 +44,11 @@ fn main() -> Result<()> { }; let source = matches - .value_of("SOURCE") + .get_one::("SOURCE") .expect("expected the required arg SOURCE to exist"); let destination = matches - .value_of("OUTPUT") + .get_one::("OUTPUT") .expect("expected the required arg OUTPUT to exist"); let args = Args { diff --git a/tools/src/bin/dump.rs b/tools/src/bin/dump.rs index 8f7c20af75..1c0a06e330 100644 --- a/tools/src/bin/dump.rs +++ b/tools/src/bin/dump.rs @@ -1,23 +1,24 @@ +use std::ffi::OsString; + use anyhow::{Context, Result}; -use clap::{App, Arg}; +use clap::{Arg, Command}; use redox_initfs::InitFs; fn main() -> Result<()> { - let matches = App::new("redox-initfs-dump") + let matches = Command::new("redox-initfs-dump") .about("dump initfs metadata") .version(clap::crate_version!()) .author(clap::crate_authors!()) .arg( - Arg::with_name("IMAGE") - .takes_value(true) + Arg::new("IMAGE") .required(true) - .help("specify the image to dump") + .help("specify the image to dump"), ) .get_matches(); let source = matches - .value_of_os("IMAGE") + .get_one::("IMAGE") .expect("expected the required arg IMAGE to exist"); let bytes = std::fs::read(source).context("failed to read image into memory")?; @@ -35,7 +36,12 @@ fn main() -> Result<()> { continue; } }; - print!("mode={:#0o}, uid={}, gid={}, kind=", inode_struct.mode(), inode_struct.uid(), inode_struct.gid()); + print!( + "mode={:#0o}, uid={}, gid={}, kind=", + inode_struct.mode(), + inode_struct.uid(), + inode_struct.gid() + ); use redox_initfs::InodeKind; @@ -67,7 +73,11 @@ fn main() -> Result<()> { continue; } }; - println!("\t`{}` => {:?},", String::from_utf8_lossy(name), entry.inode()); + println!( + "\t`{}` => {:?},", + String::from_utf8_lossy(name), + entry.inode() + ); } println!("]}}"); From 30568c5ee06bd789075cde033b3460e626eb3303 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:26:35 +0100 Subject: [PATCH 15/23] Unconditionally use no_std And use `extern crate std;` when libstd is necessary. --- src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 50c63e892b..ecc1cc9980 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,10 @@ -#![cfg_attr(not(any(test, feature = "std")), no_std)] +#![no_std] //! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be //! read. +#[cfg(any(test, feature = "std"))] +extern crate std; + use core::convert::{TryFrom, TryInto}; pub mod types; From b9e856a4abd6c111a3e92ab62faf00b6dabb02b8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:27:26 +0100 Subject: [PATCH 16/23] Remove unused cargo feature --- Cargo.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5de53b06c9..28d85d25a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,12 +15,8 @@ default = ["std"] std = [] -# Enables functionality to actually read the filesystem, which is only done in -# the kernel. (Writing to the filesystem is done in utils, and that crate only -# needs the type definitions). -kernel = [] - [dev-dependencies] +# FIXME remove loggers log = "0.4" env_logger = "0.8" anyhow = "1" From ada6cfc973f195cd37119d3804666a0fc574b229 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 10 Mar 2024 19:28:09 +0100 Subject: [PATCH 17/23] Allow embedding bootstrap code into the initfs blob This may in the future allow booting using non-redox specific bootloaders. --- tests/archive_and_read.rs | 1 + tools/src/archive_common.rs | 30 ++++++++++++++++++++++-------- tools/src/bin/archive.rs | 8 ++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/archive_and_read.rs b/tests/archive_and_read.rs index d26df6c71a..ac4f87caaf 100644 --- a/tests/archive_and_read.rs +++ b/tests/archive_and_read.rs @@ -14,6 +14,7 @@ fn archive_and_read() -> Result<()> { let args = self::archive::Args { destination_path: Path::new("out.img"), source: Path::new("data"), + bootstrap_code: None, max_size: self::archive::DEFAULT_MAX_SIZE, }; self::archive::archive(&args).context("failed to archive")?; diff --git a/tools/src/archive_common.rs b/tools/src/archive_common.rs index a340aeecad..5030396169 100644 --- a/tools/src/archive_common.rs +++ b/tools/src/archive_common.rs @@ -355,8 +355,16 @@ pub struct Args<'a> { pub destination_path: &'a Path, pub max_size: u64, pub source: &'a Path, + pub bootstrap_code: Option<&'a Path>, } -pub fn archive(&Args { destination_path, max_size, source }: &Args) -> Result<()> { +pub fn archive( + &Args { + destination_path, + max_size, + source, + bootstrap_code, + }: &Args, +) -> Result<()> { let previous_extension = destination_path.extension().map_or("", |ext| { ext.to_str() .expect("expected destination path to be valid UTF-8") @@ -407,15 +415,21 @@ pub fn archive(&Args { destination_path, max_size, source }: &Args) -> Result<() log::debug!("there are {} inodes", state.inode_count); // NOTE: The header is always stored at offset zero. - let header_offset = bump_alloc( - &mut state, - std::mem::size_of::() - .try_into() - .expect("expected header size to fit"), - "allocate header", - )?; + let header_offset = bump_alloc(&mut state, 4096, "allocate header")?; assert_eq!(header_offset, 0); + if let Some(bootstrap_code) = bootstrap_code { + allocate_and_write_file( + &mut state, + &File::open(bootstrap_code).with_context(|| { + anyhow!( + "failed to open bootstrap code file `{}`", + bootstrap_code.to_string_lossy(), + ) + })?, + )?; + }; + let inode_table_length = { let inode_entry_size: u64 = std::mem::size_of::() .try_into() diff --git a/tools/src/bin/archive.rs b/tools/src/bin/archive.rs index 6fc48c66b4..5258f75ee3 100644 --- a/tools/src/bin/archive.rs +++ b/tools/src/bin/archive.rs @@ -24,6 +24,11 @@ fn main() -> Result<()> { .required(true) .help("Specify the source directory to build the image from."), ) + .arg( + Arg::new("BOOTSTRAP_CODE") + .required(false) + .help("Specify the bootstrap ELF file to include in the image.") + ) .arg( Arg::new("OUTPUT") .required(true) @@ -47,12 +52,15 @@ fn main() -> Result<()> { .get_one::("SOURCE") .expect("expected the required arg SOURCE to exist"); + let bootstrap_code = matches.get_one::("BOOTSTRAP_CODE"); + let destination = matches .get_one::("OUTPUT") .expect("expected the required arg OUTPUT to exist"); let args = Args { source: Path::new(source), + bootstrap_code: bootstrap_code.map(|bootstrap_code| Path::new(bootstrap_code)), destination_path: Path::new(destination), max_size, }; From 7978d794f7617cb6a59ee750ec14c50bda5a0043 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:16:33 +0100 Subject: [PATCH 18/23] Encode bootstrap code entrypoint in the initfs header --- src/lib.rs | 1 + src/types.rs | 8 ++++++++ tools/src/archive_common.rs | 37 ++++++++++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ecc1cc9980..a250ebb6d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![no_std] +#![feature(offset_of)] //! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be //! read. diff --git a/src/types.rs b/src/types.rs index 24508fcfbc..b041ac8875 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,3 +1,5 @@ +use core::mem::offset_of; + pub const MAGIC_LEN: usize = 8; pub const MAGIC: [u8; 8] = *b"RedoxFtw"; @@ -69,8 +71,14 @@ pub struct Header { pub inode_table_offset: Offset, pub creation_time: Timespec, pub inode_count: U16, + pub bootstrap_entry: U64, } +const _: () = { + // Ensure the offsets of field used by the bootloader stay stable. + assert!(offset_of!(Header, bootstrap_entry) == 0x1a); +}; + #[repr(packed)] #[derive(Clone, Copy, Debug)] pub struct InodeHeader { diff --git a/tools/src/archive_common.rs b/tools/src/archive_common.rs index 5030396169..a968bdf1e7 100644 --- a/tools/src/archive_common.rs +++ b/tools/src/archive_common.rs @@ -418,7 +418,7 @@ pub fn archive( let header_offset = bump_alloc(&mut state, 4096, "allocate header")?; assert_eq!(header_offset, 0); - if let Some(bootstrap_code) = bootstrap_code { + let bootstrap_entry = if let Some(bootstrap_code) = bootstrap_code { allocate_and_write_file( &mut state, &File::open(bootstrap_code).with_context(|| { @@ -428,6 +428,15 @@ pub fn archive( ) })?, )?; + let bootstrap_data = std::fs::read(bootstrap_code).with_context(|| { + anyhow!( + "failed to read bootstrap code file `{}`", + bootstrap_code.to_string_lossy(), + ) + })?; + elf_entry(&bootstrap_data) + } else { + u64::MAX }; let inode_table_length = { @@ -477,6 +486,7 @@ pub fn archive( }, inode_count: state.inode_count.into(), inode_table_offset, + bootstrap_entry: bootstrap_entry.into(), }; write_all_at(&*state.file, &header_bytes, header_offset, "writing header") .context("failed to write header")?; @@ -489,3 +499,28 @@ pub fn archive( Ok(()) } + +fn elf_entry(data: &[u8]) -> u64 { + assert!(&data[..4] == b"\x7FELF"); + match (data[4], data[5]) { + // 32-bit, little endian + (1, 1) => u32::from_le_bytes( + <[u8; 4]>::try_from(&data[0x18..0x18 + 4]).expect("conversion cannot fail"), + ) as u64, + // 32-bit, big endian + (1, 2) => u32::from_be_bytes( + <[u8; 4]>::try_from(&data[0x18..0x18 + 4]).expect("conversion cannot fail"), + ) as u64, + // 64-bit, little endian + (2, 1) => u64::from_le_bytes( + <[u8; 8]>::try_from(&data[0x18..0x18 + 8]).expect("conversion cannot fail"), + ), + // 64-bit, big endian + (2, 2) => u64::from_be_bytes( + <[u8; 8]>::try_from(&data[0x18..0x18 + 8]).expect("conversion cannot fail"), + ), + (ei_class, ei_data) => { + panic!("Unsupported ELF EI_CLASS {} EI_DATA {}", ei_class, ei_data); + } + } +} From 12ed3151f3df859775f327250244b483ea699edb Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:13:49 +0100 Subject: [PATCH 19/23] Add size to the initfs header --- src/types.rs | 1 + tools/src/archive_common.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/types.rs b/src/types.rs index b041ac8875..6faf321db5 100644 --- a/src/types.rs +++ b/src/types.rs @@ -72,6 +72,7 @@ pub struct Header { pub creation_time: Timespec, pub inode_count: U16, pub bootstrap_entry: U64, + pub initfs_size: U64, } const _: () = { diff --git a/tools/src/archive_common.rs b/tools/src/archive_common.rs index a968bdf1e7..b7a0b250b9 100644 --- a/tools/src/archive_common.rs +++ b/tools/src/archive_common.rs @@ -487,6 +487,7 @@ pub fn archive( inode_count: state.inode_count.into(), inode_table_offset, bootstrap_entry: bootstrap_entry.into(), + initfs_size: state.file.metadata().context("failed to get initfs size")?.len().into(), }; write_all_at(&*state.file, &header_bytes, header_offset, "writing header") .context("failed to write header")?; From 65cf406b7d4c5befc38b79696cdeb317bf06ed62 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Sun, 8 Sep 2024 21:09:22 +0200 Subject: [PATCH 20/23] Future-proof: repr(packed) => repr(C, packed). --- src/types.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/types.rs b/src/types.rs index 6faf321db5..c08458adf5 100644 --- a/src/types.rs +++ b/src/types.rs @@ -57,14 +57,14 @@ pub struct Offset(pub U32); #[derive(Clone, Copy, Debug)] pub struct Length(pub U32); -#[repr(packed)] +#[repr(C, packed)] #[derive(Clone, Copy, Debug)] pub struct Timespec { pub sec: U64, pub nsec: U32, } -#[repr(packed)] +#[repr(C, packed)] #[derive(Clone, Copy, Debug)] pub struct Header { pub magic: Magic, @@ -80,7 +80,7 @@ const _: () = { assert!(offset_of!(Header, bootstrap_entry) == 0x1a); }; -#[repr(packed)] +#[repr(C, packed)] #[derive(Clone, Copy, Debug)] pub struct InodeHeader { pub type_and_mode: U32, @@ -104,7 +104,7 @@ pub enum InodeType { // All other bit patterns are reserved... for now. TODO: Add symlinks? } -#[repr(packed)] +#[repr(C, packed)] #[derive(Clone, Copy, Debug)] pub struct DirEntry { pub inode: U16, From f0dec135b958d82f741db34f51381a065f0d1f11 Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Sat, 14 Sep 2024 12:01:16 +0200 Subject: [PATCH 21/23] Add symlink support * Implement symlinks * Initfs only has links with absolute paths. * Relative links are assumed to be relative to link location. * Absolute paths must point into the archived location. E.g. if we archive /home/foo/data and wish the resultant link to point to /scheme/rand, then the absolute link must be /home/foo/data/scheme/rand. * Move archive_common to a crate within the workspace -- Rust doesn't seem to accept #[path(..)] attributes with `..` segments anymore so tests wouldn't build on my machine. * Improve the test to assert complete filesystem structure. * Bump to edition 2021 * cargo fmt & clippy --- Cargo.toml | 13 +- archive-common/Cargo.toml | 12 ++ .../src/lib.rs | 158 +++++++++++++----- data/foo/file-link.txt | 1 + data/{ => foo}/file.txt | 0 src/lib.rs | 26 ++- src/types.rs | 3 +- tests/archive_and_read.rs | 112 ++++++++++--- tools/Cargo.toml | 17 +- tools/src/bin/archive.rs | 8 +- tools/src/bin/dump.rs | 17 +- 11 files changed, 270 insertions(+), 97 deletions(-) create mode 100644 archive-common/Cargo.toml rename tools/src/archive_common.rs => archive-common/src/lib.rs (76%) create mode 120000 data/foo/file-link.txt rename data/{ => foo}/file.txt (100%) diff --git a/Cargo.toml b/Cargo.toml index 28d85d25a7..a8ec1682e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "redox-initfs" -version = "0.1.0" -authors = ["4lDO2 <4lDO2@protonmail.com>"] -edition = "2018" +version = "0.2.0" +authors = ["4lDO2 <4lDO2@protonmail.com>", "Kamil Koczurek "] +edition = "2021" license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +pathdiff = "0.2.1" plain = "0.2" [features] @@ -17,11 +18,13 @@ std = [] [dev-dependencies] # FIXME remove loggers -log = "0.4" -env_logger = "0.8" anyhow = "1" +archive-common = {path = "archive-common"} +env_logger = "0.8" +log = "0.4" [workspace] members = [ + "archive-common", "tools", ] diff --git a/archive-common/Cargo.toml b/archive-common/Cargo.toml new file mode 100644 index 0000000000..ba5d1e963e --- /dev/null +++ b/archive-common/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "archive-common" +version = "0.1.0" +authors = ["4lDO2 <4lDO2@protonmail.com>", "Kamil Koczurek "] +edition = "2021" + +[dependencies] +anyhow = "1" +log = "0.4" +pathdiff = "*" +plain = "0.2" +redox-initfs = {path = ".."} diff --git a/tools/src/archive_common.rs b/archive-common/src/lib.rs similarity index 76% rename from tools/src/archive_common.rs rename to archive-common/src/lib.rs index b7a0b250b9..25fd8f4e79 100644 --- a/tools/src/archive_common.rs +++ b/archive-common/src/lib.rs @@ -1,12 +1,12 @@ use std::convert::{TryFrom, TryInto}; use std::fs::{DirEntry, File, Metadata, OpenOptions}; use std::io::{prelude::*, SeekFrom}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use redox_initfs::types as initfs; @@ -17,6 +17,7 @@ pub const DEFAULT_MAX_SIZE: u64 = 64 * MEBIBYTE; enum EntryKind { File(File), Dir(Dir), + Link(PathBuf), } struct Entry { @@ -39,11 +40,16 @@ struct State<'path> { fn write_all_at(file: &File, buf: &[u8], offset: u64, r#where: &str) -> Result<()> { file.write_all_at(buf, offset)?; - log::trace!("Wrote {}..{} within {}", offset, offset + buf.len() as u64, r#where); + log::trace!( + "Wrote {}..{} within {}", + offset, + offset + buf.len() as u64, + r#where + ); Ok(()) } -fn read_directory(state: &mut State, path: &Path) -> Result { +fn read_directory(state: &mut State, path: &Path, root_path: &Path) -> Result { let read_dir = path .read_dir() .with_context(|| anyhow!("failed to read directory `{}`", path.to_string_lossy(),))?; @@ -79,9 +85,7 @@ fn read_directory(state: &mut State, path: &Path) -> Result { .as_bytes() .to_owned(); - let entry_kind = if file_type.is_symlink() { - return unsupported_type("symlink", &entry); - } else if file_type.is_socket() { + let entry_kind = if file_type.is_socket() { return unsupported_type("socket", &entry); } else if file_type.is_fifo() { return unsupported_type("FIFO", &entry); @@ -90,11 +94,35 @@ fn read_directory(state: &mut State, path: &Path) -> Result { } else if file_type.is_char_device() { return unsupported_type("character device", &entry); } else if file_type.is_file() { - EntryKind::File(File::open(&entry.path()).with_context(|| { + EntryKind::File(File::open(entry.path()).with_context(|| { anyhow!("failed to open file `{}`", entry.path().to_string_lossy(),) })?) } else if file_type.is_dir() { - EntryKind::Dir(read_directory(state, &entry.path())?) + EntryKind::Dir(read_directory(state, &entry.path(), root_path)?) + } else if file_type.is_symlink() { + let link_file_path = entry.path(); + + let link_path = std::fs::read_link(&link_file_path)?; + let cannonical = if link_path.is_absolute() { + link_path.clone() + } else { + let Some(link_parent) = link_file_path.parent() else { + bail!("Link at `{}` has no parent", link_file_path.display()) + }; + link_parent.canonicalize()?.join(link_path.clone()) + }; + + let root_path = root_path + .canonicalize() + .context("Failed to cannonicalize root path")?; + let path = pathdiff::diff_paths(cannonical, &root_path).ok_or_else(|| { + anyhow!( + "Failed to diff symlink path [{}] to root path [{}]", + link_path.display(), + root_path.display() + ) + })?; + EntryKind::Link(path) } else { return Err(anyhow!( "unknown file type at `{}`", @@ -164,14 +192,40 @@ fn allocate_and_write_file(state: &mut State, mut file: &File) -> Result Result { + let data = link.as_os_str().as_bytes(); + let size: u32 = data.len().try_into().unwrap(); + + let offset: u32 = bump_alloc(state, size.into(), "allocate space for file") + .context("failed to allocate space for file")? + .try_into() + .context("file offset too high")?; + + write_all_at( + &state.file, + data, + u64::from(offset), + "allocate_and_write_link target path", + ) + .context("failed to write source file into destination image")?; + + Ok(WriteResult { size, offset }) +} + fn write_inode( state: &mut State, ty: initfs::InodeType, @@ -183,7 +237,7 @@ fn write_inode( .try_into() .expect("inode header length cannot fit within u32"); - let type_and_mode = ((ty as u32) << initfs::TYPE_SHIFT) | u32::from(metadata.mode() & 0xFFF); + let type_and_mode = ((ty as u32) << initfs::TYPE_SHIFT) | (metadata.mode() & 0xFFF); // TODO: Use main buffer and write in bulk. let mut inode_buf = [0_u8; std::mem::size_of::()]; @@ -196,13 +250,17 @@ fn write_inode( length: write_result.size.into(), offset: initfs::Offset(write_result.offset.into()), - gid: 0.into(),//metadata.gid().into(), - uid: 0.into(),//metadata.uid().into(), + gid: 0.into(), //metadata.gid().into(), + uid: 0.into(), //metadata.uid().into(), }; - log::debug!("Writing inode index {} from offset {}", inode, state.inode_table_offset); + log::debug!( + "Writing inode index {} from offset {}", + inode, + state.inode_table_offset + ); write_all_at( - &*state.file, + &state.file, &inode_buf, u64::from(state.inode_table_offset + u32::from(inode) * inode_size), "write_inode", @@ -222,22 +280,22 @@ fn allocate_and_write_dir( .checked_mul(u32::from(entry_size)) .ok_or_else(|| anyhow!("entry table length too large when multiplying by size"))?; - let entry_table_offset: u32 = bump_alloc(state, entry_table_length.into(), "allocate entry table") - .context("failed to allocate entry table")? - .try_into() - .context("directory entries offset too high")?; + let entry_table_offset: u32 = + bump_alloc(state, entry_table_length.into(), "allocate entry table") + .context("failed to allocate entry table")? + .try_into() + .context("directory entries offset too high")?; for (index, entry) in dir.entries.iter().enumerate() { let (write_result, ty) = match entry.kind { EntryKind::Dir(ref subdir) => { - let write_result = - allocate_and_write_dir(state, subdir, current_inode) - .with_context(|| { - anyhow!( - "failed to copy directory entries from `{}` into image", - String::from_utf8_lossy(&entry.name) - ) - })?; + let write_result = allocate_and_write_dir(state, subdir, current_inode) + .with_context(|| { + anyhow!( + "failed to copy directory entries from `{}` into image", + String::from_utf8_lossy(&entry.name) + ) + })?; (write_result, initfs::InodeType::Dir) } @@ -248,6 +306,12 @@ fn allocate_and_write_dir( (write_result, initfs::InodeType::RegularFile) } + + EntryKind::Link(ref path) => { + let write_result = allocate_and_write_link(state, path) + .context("failed to copy symbolic link into image")?; + (write_result, initfs::InodeType::Link) + } }; let index: u16 = index @@ -255,13 +319,7 @@ fn allocate_and_write_dir( .expect("expected dir entry count not to exceed u32"); *current_inode += 1; - write_inode( - state, - ty, - &entry.metadata, - write_result, - *current_inode, - )?; + write_inode(state, ty, &entry.metadata, write_result, *current_inode)?; let (name_offset, name_len) = { let name_len: u16 = entry.name.len().try_into().context("file name too long")?; @@ -271,7 +329,8 @@ fn allocate_and_write_dir( .try_into() .context("file name offset too high up")?; - write_all_at(&*state.file, &entry.name, offset.into(), "writing file name").context("failed to write file name")?; + write_all_at(&state.file, &entry.name, offset.into(), "writing file name") + .context("failed to write file name")?; (offset, name_len) }; @@ -281,7 +340,12 @@ fn allocate_and_write_dir( let direntry = plain::from_mut_bytes::(&mut direntry_buf) .expect("expected dir entry struct to have alignment 1, and buffer size to match"); - log::debug!("Linking inode {} into dir entry index {}, file name `{}`", current_inode, index, String::from_utf8_lossy(&entry.name)); + log::debug!( + "Linking inode {} into dir entry index {}, file name `{}`", + current_inode, + index, + String::from_utf8_lossy(&entry.name) + ); *direntry = initfs::DirEntry { inode: (*current_inode).into(), @@ -290,7 +354,7 @@ fn allocate_and_write_dir( }; write_all_at( - &*state.file, + &state.file, &direntry_buf, u64::from(entry_table_offset + u32::from(index) * u32::from(entry_size)), "allocate_and_write_dir entry", @@ -384,6 +448,7 @@ pub fn archive( .read(false) .write(true) .create(true) + .truncate(true) .create_new(false) .open(&destination_temp_path) .context("failed to open destination file")?; @@ -410,7 +475,7 @@ pub fn archive( let root_metadata = root_path .metadata() .context("failed to obtain metadata for root")?; - let root = read_directory(&mut state, root_path).context("failed to read root")?; + let root = read_directory(&mut state, root_path, root_path).context("failed to read root")?; log::debug!("there are {} inodes", state.inode_count); @@ -461,11 +526,7 @@ pub fn archive( state.inode_table_offset = inode_table_offset.0.get(); - allocate_contents_and_write_inodes( - &mut state, - &root, - root_metadata, - )?; + allocate_contents_and_write_inodes(&mut state, &root, root_metadata)?; let current_system_time = std::time::SystemTime::now(); @@ -487,9 +548,14 @@ pub fn archive( inode_count: state.inode_count.into(), inode_table_offset, bootstrap_entry: bootstrap_entry.into(), - initfs_size: state.file.metadata().context("failed to get initfs size")?.len().into(), + initfs_size: state + .file + .metadata() + .context("failed to get initfs size")? + .len() + .into(), }; - write_all_at(&*state.file, &header_bytes, header_offset, "writing header") + write_all_at(&state.file, &header_bytes, header_offset, "writing header") .context("failed to write header")?; } diff --git a/data/foo/file-link.txt b/data/foo/file-link.txt new file mode 120000 index 0000000000..4c330738cc --- /dev/null +++ b/data/foo/file-link.txt @@ -0,0 +1 @@ +file.txt \ No newline at end of file diff --git a/data/file.txt b/data/foo/file.txt similarity index 100% rename from data/file.txt rename to data/foo/file.txt diff --git a/src/lib.rs b/src/lib.rs index a250ebb6d7..0e77a0dcc7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,4 @@ #![no_std] -#![feature(offset_of)] //! A super simple initfs, only meant to be loaded into RAM by the bootloader, and then directly be //! read. @@ -91,6 +90,20 @@ impl<'initfs> InodeDir<'initfs> { } } +#[derive(Clone, Copy)] +pub struct InodeLink<'initfs> { + inner: InodeStruct<'initfs>, +} + +impl<'initfs> InodeLink<'initfs> { + pub fn inode(self) -> InodeStruct<'initfs> { + self.inner + } + pub fn data(&self) -> Result<&'initfs [u8]> { + self.inner.data() + } +} + #[derive(Clone, Copy)] pub struct Entry<'initfs> { initfs: InitFs<'initfs>, @@ -109,7 +122,7 @@ impl<'initfs> Entry<'initfs> { .get() .try_into() .map_err(|_| Error)?; - let name_length: usize = self.entry.name_len.get().try_into().map_err(|_| Error)?; + let name_length: usize = self.entry.name_len.get().into(); let name_end = name_offset.checked_add(name_length).ok_or(Error)?; @@ -121,6 +134,7 @@ impl<'initfs> Entry<'initfs> { pub enum InodeKind<'initfs> { File(InodeFile<'initfs>), Dir(InodeDir<'initfs>), + Link(InodeLink<'initfs>), Unknown, } @@ -150,14 +164,18 @@ impl<'initfs> InodeStruct<'initfs> { InodeType::Dir } else if raw == InodeType::RegularFile as u32 { InodeType::RegularFile + } else if raw == InodeType::Link as u32 { + InodeType::Link } else { return None; }) } pub fn kind(&self) -> InodeKind<'initfs> { + let inner = *self; match self.ty() { - Some(InodeType::Dir) => InodeKind::Dir(InodeDir { inner: *self }), - Some(InodeType::RegularFile) => InodeKind::File(InodeFile { inner: *self }), + Some(InodeType::Dir) => InodeKind::Dir(InodeDir { inner }), + Some(InodeType::RegularFile) => InodeKind::File(InodeFile { inner }), + Some(InodeType::Link) => InodeKind::Link(InodeLink { inner }), None => InodeKind::Unknown, } } diff --git a/src/types.rs b/src/types.rs index c08458adf5..6951bd86dc 100644 --- a/src/types.rs +++ b/src/types.rs @@ -101,7 +101,8 @@ pub const TYPE_MASK: u32 = 0xF000_0000; pub enum InodeType { RegularFile = 0x0, Dir = 0x1, - // All other bit patterns are reserved... for now. TODO: Add symlinks? + Link = 0x2, + // All other bit patterns are reserved... for now. } #[repr(C, packed)] diff --git a/tests/archive_and_read.rs b/tests/archive_and_read.rs index ac4f87caaf..294d9defe0 100644 --- a/tests/archive_and_read.rs +++ b/tests/archive_and_read.rs @@ -1,47 +1,107 @@ -#![feature(array_methods)] +use std::{collections::HashMap, path::Path}; -use std::path::Path; +use anyhow::{anyhow, Context, Result}; +use redox_initfs::{InitFs, InodeKind, InodeStruct}; -use anyhow::{anyhow, bail, Context, Result}; +#[derive(Debug, Clone, PartialEq)] +enum Node { + Link { to: Vec }, + File { data: Vec }, + Dir(HashMap, Node>), + Unknown, +} -#[path = "../tools/src/archive_common.rs"] -mod archive; +impl Node { + fn link(to: impl Into>) -> Self { + Node::Link { to: to.into() } + } + + fn file(data: impl Into>) -> Self { + Node::File { data: data.into() } + } + + fn dir(entries: impl IntoIterator>, Node)>) -> Self { + Self::Dir( + entries + .into_iter() + .map(|(name, node)| (name.into(), node)) + .collect(), + ) + } +} + +fn build_tree<'a>(fs: InitFs<'a>, inode: InodeStruct<'a>) -> anyhow::Result { + use InodeKind::*; + let node = match inode.kind() { + File(file) => { + let data = file.data().context("failed to get file data")?.to_owned(); + Node::File { data } + } + Link(link) => { + let data = link.data().context("failed to get link data")?.to_owned(); + Node::Link { to: data } + } + Dir(dir) => { + let mut entries = HashMap::new(); + for idx in 0..dir + .entry_count() + .context("failed to get inode entry count")? + { + let entry = dir + .get_entry(idx) + .context("failed to get entry for index")? + .ok_or_else(|| anyhow!("no entry found"))?; + + let entry_name = entry.name().context("failed to get entry name")?; + let inode = fs + .get_inode(entry.inode()) + .context("failed to load file inode")?; + + let entry_node = build_tree(fs, inode)?; + + entries.insert(entry_name.to_owned(), entry_node); + } + + Node::Dir(entries) + } + Unknown => Node::Unknown, + }; + + Ok(node) +} #[test] fn archive_and_read() -> Result<()> { env_logger::init(); - let args = self::archive::Args { + let args = archive_common::Args { destination_path: Path::new("out.img"), source: Path::new("data"), bootstrap_code: None, - max_size: self::archive::DEFAULT_MAX_SIZE, + max_size: archive_common::DEFAULT_MAX_SIZE, }; - self::archive::archive(&args).context("failed to archive")?; + archive_common::archive(&args).context("failed to archive")?; let data = std::fs::read(args.destination_path).context("failed to read new archive")?; let filesystem = redox_initfs::InitFs::new(&data).context("failed to parse archive header")?; - let root_inode = filesystem.get_inode(redox_initfs::InitFs::ROOT_INODE).ok_or_else(|| anyhow!("Failed to get root inode"))?; + let inode = filesystem + .get_inode(redox_initfs::InitFs::ROOT_INODE) + .ok_or_else(|| anyhow!("Failed to get root inode"))?; - let dir = match root_inode.kind() { - redox_initfs::InodeKind::Dir(dir) => dir, - _ => bail!("root inode was not a directory"), - }; + let tree = build_tree(filesystem, inode)?; - for idx in 0..dir.entry_count().context("failed to get inode entry count")? { - let entry = dir.get_entry(idx).context("failed to get entry for index")?.ok_or_else(|| anyhow!("no entry found"))?; + let reference_tree = Node::dir([( + b"foo", + Node::dir([ + ( + b"file.txt".as_slice(), + Node::file(b"This is a file meant to be used in a redox-initfs test.\n"), + ), + (b"file-link.txt".as_slice(), Node::link(b"foo/file.txt")), + ]), + )]); - if entry.name().context("failed to get entry name")? == b"file.txt".as_slice() { - let inode = filesystem.get_inode(entry.inode()).context("failed to load file inode")?; - - let file = match inode.kind() { - redox_initfs::InodeKind::File(file) => file, - _ => bail!("file.txt was a directory"), - }; - let data = file.data().context("failed to get file.txt data")?; - assert_eq!(data, std::fs::read("data/file.txt").context("failed to read the real file.txt")?); - } - } + assert_eq!(tree, reference_tree); Ok(()) } diff --git a/tools/Cargo.toml b/tools/Cargo.toml index fb9acfae24..e201f04c1a 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -1,27 +1,28 @@ [package] -name = "redox-initfs-ar" -version = "0.1.0" -authors = ["4lDO2 <4lDO2@protonmail.com>"] -edition = "2018" +name = "redox-initfs-tools" +version = "0.2.0" +authors = ["4lDO2 <4lDO2@protonmail.com>", "Kamil Koczurek "] +edition = "2021" description = "Archive a directory into a Redox initfs image" license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [[bin]] -path = "src/bin/archive.rs" name = "redox-initfs-ar" +path = "src/bin/archive.rs" [[bin]] -path = "src/bin/dump.rs" name = "redox-initfs-dump" +path = "src/bin/dump.rs" [dependencies] anyhow = "1" -clap = { version = "4", features = ["cargo"] } +clap = {version = "4", features = ["cargo"]} env_logger = "0.8" log = "0.4" plain = "0.2" twox-hash = "1.6" -redox-initfs = { path = ".." } +archive-common = {path = "../archive-common"} +redox-initfs = {path = ".."} diff --git a/tools/src/bin/archive.rs b/tools/src/bin/archive.rs index 5258f75ee3..b5ab1db9a3 100644 --- a/tools/src/bin/archive.rs +++ b/tools/src/bin/archive.rs @@ -3,9 +3,7 @@ use std::path::Path; use anyhow::{Context, Result}; use clap::{Arg, Command}; -#[path = "../archive_common.rs"] -mod archive_common; -use self::archive_common::{self as archive, Args, DEFAULT_MAX_SIZE}; +use archive_common::{self as archive, Args, DEFAULT_MAX_SIZE}; fn main() -> Result<()> { let matches = Command::new("redox-initfs-ar") @@ -27,7 +25,7 @@ fn main() -> Result<()> { .arg( Arg::new("BOOTSTRAP_CODE") .required(false) - .help("Specify the bootstrap ELF file to include in the image.") + .help("Specify the bootstrap ELF file to include in the image."), ) .arg( Arg::new("OUTPUT") @@ -60,7 +58,7 @@ fn main() -> Result<()> { let args = Args { source: Path::new(source), - bootstrap_code: bootstrap_code.map(|bootstrap_code| Path::new(bootstrap_code)), + bootstrap_code: bootstrap_code.map(Path::new), destination_path: Path::new(destination), max_size, }; diff --git a/tools/src/bin/dump.rs b/tools/src/bin/dump.rs index 1c0a06e330..fb97bc9cd2 100644 --- a/tools/src/bin/dump.rs +++ b/tools/src/bin/dump.rs @@ -1,4 +1,4 @@ -use std::ffi::OsString; +use std::{ffi::OsStr, path::Path}; use anyhow::{Context, Result}; use clap::{Arg, Command}; @@ -18,7 +18,7 @@ fn main() -> Result<()> { .get_matches(); let source = matches - .get_one::("IMAGE") + .get_one::("IMAGE") .expect("expected the required arg IMAGE to exist"); let bytes = std::fs::read(source).context("failed to read image into memory")?; @@ -99,6 +99,19 @@ fn main() -> Result<()> { println!("}}"); } + InodeKind::Link(link) => { + print!("link{{"); + match link.data().ok() { + Some(d) => { + use std::os::unix::ffi::OsStrExt; + print!("dst={}", Path::new(OsStr::from_bytes(d)).display()); + } + None => { + print!("(failed to get data)"); + } + } + println!("}}"); + } } } From 7005f5ee65b8505fd688309a5c3ff3b07d0cf859 Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Sat, 14 Sep 2024 12:29:55 +0200 Subject: [PATCH 22/23] Specify pathdiff version --- Cargo.toml | 1 - archive-common/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a8ec1682e4..3c944a2d54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -pathdiff = "0.2.1" plain = "0.2" [features] diff --git a/archive-common/Cargo.toml b/archive-common/Cargo.toml index ba5d1e963e..cb40b568b6 100644 --- a/archive-common/Cargo.toml +++ b/archive-common/Cargo.toml @@ -7,6 +7,6 @@ edition = "2021" [dependencies] anyhow = "1" log = "0.4" -pathdiff = "*" +pathdiff = "0.2.1" plain = "0.2" redox-initfs = {path = ".."} From 873062c1c254880ba47fed40e50a8a47a67eac1a Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Sat, 14 Sep 2024 22:41:58 +0200 Subject: [PATCH 23/23] Document missing support for non-utf8 paths --- tools/src/bin/archive.rs | 1 + tools/src/bin/dump.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/src/bin/archive.rs b/tools/src/bin/archive.rs index b5ab1db9a3..5e6291417a 100644 --- a/tools/src/bin/archive.rs +++ b/tools/src/bin/archive.rs @@ -17,6 +17,7 @@ fn main() -> Result<()> { .required(false) .help("Set the upper limit for how large the image can become (default 64 MiB)."), ) + // TODO: support non-utf8 paths (applies to other paths as well) .arg( Arg::new("SOURCE") .required(true) diff --git a/tools/src/bin/dump.rs b/tools/src/bin/dump.rs index fb97bc9cd2..414d58be0d 100644 --- a/tools/src/bin/dump.rs +++ b/tools/src/bin/dump.rs @@ -17,6 +17,7 @@ fn main() -> Result<()> { ) .get_matches(); + // TODO: support non-utf8 paths let source = matches .get_one::("IMAGE") .expect("expected the required arg IMAGE to exist");