diff --git a/src/cook.rs b/src/cook.rs index 9c989215c3..0e5735366c 100644 --- a/src/cook.rs +++ b/src/cook.rs @@ -1,4 +1,5 @@ // avoid confusion with build.rs +pub mod cargo_path_deps; pub mod cook_build; pub mod fetch; pub mod fetch_repo; diff --git a/src/cook/cargo_path_deps.rs b/src/cook/cargo_path_deps.rs new file mode 100644 index 0000000000..93286b4a18 --- /dev/null +++ b/src/cook/cargo_path_deps.rs @@ -0,0 +1,324 @@ +//! Cargo path-dependency source tracking for content-hash caching. +//! +//! The recipe dep-hash cache (`DepHashes`) covers recipe-level PKGAR +//! dependencies, and `SourceContentHash` covers the recipe's own source +//! tree. Neither sees Cargo `path =` dependencies inside a crate — the +//! exact pattern the local fork model mandates (`redox_syscall`, +//! `libredox`, `redox-driver-*`, `pcid_interface` are all consumed this +//! way). Without this module, editing a path-dep's source leaves the +//! dependent recipe cached and stale (observed: `cook driver-manager - +//! cached` while `redox-driver-core` sources had changed). +//! +//! The resolver below walks each recipe source's `Cargo.toml` +//! (dependencies, dev/build dependencies, target-specific deps, +//! `[patch.*]` tables, and workspace members) recursively, and hashes +//! every resolved path-dep source tree into the stored dep-hash record. +//! A content change anywhere in a path-dep tree invalidates the recipe. + +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, +}; + +/// Compute the cargo path-dep tree hashes for a recipe source dir. +/// Keys are canonical dep paths; values are tree content hashes. +/// An empty map means "no Cargo manifest or no path deps" — never an error. +pub fn compute_all(source_dir: &Path) -> BTreeMap { + let mut visited = BTreeSet::new(); + let mut out = BTreeMap::new(); + let manifest = source_dir.join("Cargo.toml"); + if manifest.is_file() { + resolve_recursive(&manifest, &mut visited, &mut out); + } + out +} + +fn resolve_recursive( + manifest_path: &Path, + visited: &mut BTreeSet, + out: &mut BTreeMap, +) { + let Ok(manifest_path) = manifest_path.canonicalize() else { + return; + }; + if !visited.insert(manifest_path.clone()) { + return; + } + let Ok(content) = fs::read_to_string(&manifest_path) else { + return; + }; + let Ok(doc) = content.parse::() else { + return; + }; + let Some(manifest_dir) = manifest_path.parent() else { + return; + }; + + let mut dep_paths = Vec::new(); + collect_dependency_paths(&doc, &mut dep_paths); + collect_workspace_member_paths(&doc, &mut dep_paths); + + for rel in dep_paths { + let candidate = manifest_dir.join(&rel); + let Ok(dep_dir) = candidate.canonicalize() else { + continue; + }; + if !dep_dir.is_dir() { + continue; + } + let key = dep_dir.to_string_lossy().to_string(); + if !out.contains_key(&key) { + let hash = hash_tree(&dep_dir).unwrap_or_else(|_| "".to_string()); + out.insert(key, hash); + } + let dep_manifest = dep_dir.join("Cargo.toml"); + if dep_manifest.is_file() { + resolve_recursive(&dep_manifest, visited, out); + } + } +} + +/// Extract every `path = "..."` value from dependency tables: +/// `[dependencies]`, `[dev-dependencies]`, `[build-dependencies]`, +/// `[target..dependencies*]`, and `[patch.]`. +fn collect_dependency_paths(doc: &toml::Value, out: &mut Vec) { + const TABLES: &[&str] = &["dependencies", "dev-dependencies", "build-dependencies"]; + + if let Some(root) = doc.as_table() { + for (key, value) in root { + if TABLES.contains(&key.as_str()) { + collect_from_dep_table(value, out); + } else if key == "patch" { + if let Some(patch_tables) = value.as_table() { + for (_registry, table) in patch_tables { + collect_from_dep_table(table, out); + } + } + } else if key == "target" { + if let Some(targets) = value.as_table() { + for (_triple, target_table) in targets { + if let Some(t) = target_table.as_table() { + for (tkey, tval) in t { + if TABLES.contains(&tkey.as_str()) { + collect_from_dep_table(tval, out); + } + } + } + } + } + } + } + } +} + +fn collect_from_dep_table(table: &toml::Value, out: &mut Vec) { + let Some(entries) = table.as_table() else { + return; + }; + for (_name, spec) in entries { + match spec { + toml::Value::String(_) => {} + toml::Value::Table(spec_table) => { + if let Some(toml::Value::String(path)) = spec_table.get("path") { + out.push(PathBuf::from(path)); + } + } + _ => {} + } + } +} + +/// Extract workspace member manifests: `[workspace] members = [...]`. +fn collect_workspace_member_paths(doc: &toml::Value, out: &mut Vec) { + let Some(workspace) = doc.get("workspace") else { + return; + }; + let Some(members) = workspace.get("members").and_then(|m| m.as_array()) else { + return; + }; + for member in members { + if let Some(toml::Value::String(path)) = Some(member) { + out.push(PathBuf::from(path)); + } + } +} + +/// Content-hash a path-dep source tree: sorted (relative path, content) +/// pairs, excluding `.git/`, `target/`, and editor artifacts. Matches the +/// exclusion policy of `SourceContentHash::compute` so a file that is +/// invisible there is also invisible here. +fn hash_tree(dir: &Path) -> std::io::Result { + let mut paths = Vec::new(); + collect_files(dir, dir, &mut paths)?; + paths.sort(); + + let mut hasher = blake3::Hasher::new(); + for rel in &paths { + let rel_str = rel.to_string_lossy(); + if rel_str.starts_with(".git/") + || rel_str.starts_with("target/") + || rel_str.contains("/target/") + || rel_str.ends_with(".swp") + || rel_str.ends_with(".tmp") + { + continue; + } + hasher.update(rel_str.as_bytes()); + hasher.update(b"\0"); + let content = fs::read(dir.join(rel)).unwrap_or_else(|_| b"".to_vec()); + hasher.update(&content); + hasher.update(b"\0"); + } + Ok(hasher.finalize().to_hex().to_string()) +} + +fn collect_files(dir: &Path, root: &Path, out: &mut Vec) -> std::io::Result<()> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let Ok(rel) = path.strip_prefix(root).map(|r| r.to_path_buf()) else { + continue; + }; + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_dir() { + collect_files(&path, root, out)?; + } else if file_type.is_file() || file_type.is_symlink() { + out.push(rel); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + fn serialized(body: impl FnOnce() -> R) -> R { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + body() + } + + fn temp_dir(label: &str) -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("rb-cargo-deps-{label}-{}-{nanos}", std::process::id())) + } + + fn write(path: &Path, body: &str) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, body).unwrap(); + } + + fn make_dep(root: &Path, name: &str, lib_body: &str) -> PathBuf { + let dep = root.join(name); + write(&dep.join("Cargo.toml"), &format!("[package]\nname = \"{name}\"\nversion = \"0.1.0\"\n")); + write(&dep.join("src/lib.rs"), lib_body); + dep + } + + #[test] + fn resolves_table_and_patch_path_deps_recursively() { + serialized(|| { + let root = temp_dir("resolve"); + let dep_a = make_dep(&root, "dep-a", "pub fn a() {}"); + let dep_b = make_dep(&root, "dep-b", "pub fn b() {}"); + write( + &dep_a.join("src/lib.rs"), + "pub fn a() { dep_b::b(); }", + ); + write( + &dep_a.join("Cargo.toml"), + "[package]\nname = \"dep-a\"\nversion = \"0.1.0\"\n[dependencies]\ndep-b = { path = \"../dep-b\" }\n", + ); + write( + &root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n\ + [dependencies]\ndep-a = { path = \"dep-a\" }\nplain = \"1.0\"\n\ + [patch.crates-io]\npatched = { path = \"dep-b\" }\n", + ); + + let map = compute_all(&root); + let keys: Vec = map.keys().cloned().collect(); + assert_eq!(keys.len(), 2, "expected dep-a and dep-b, got {keys:?}"); + assert!(keys.iter().any(|k| k.ends_with("dep-a"))); + assert!(keys.iter().any(|k| k.ends_with("dep-b"))); + let _ = fs::remove_dir_all(&root); + }); + } + + #[test] + fn resolves_workspace_members() { + serialized(|| { + let root = temp_dir("workspace"); + make_dep(&root, "member-one", "pub fn one() {}"); + write( + &root.join("Cargo.toml"), + "[workspace]\nmembers = [\"member-one\"]\n", + ); + let map = compute_all(&root); + assert_eq!(map.len(), 1); + let _ = fs::remove_dir_all(&root); + }); + } + + #[test] + fn hash_changes_with_content_and_file_list() { + serialized(|| { + let root = temp_dir("hash"); + let dep = make_dep(&root, "dep", "pub fn v() -> u32 { 1 }"); + let first = hash_tree(&dep).unwrap(); + let second = hash_tree(&dep).unwrap(); + assert_eq!(first, second, "identical tree must hash identically"); + + write(&dep.join("src/lib.rs"), "pub fn v() -> u32 { 2 }"); + let third = hash_tree(&dep).unwrap(); + assert_ne!(first, third, "content edit must change the hash"); + + write(&dep.join("src/new.rs"), "pub fn added() {}"); + let fourth = hash_tree(&dep).unwrap(); + assert_ne!(third, fourth, "new file must change the hash"); + let _ = fs::remove_dir_all(&root); + }); + } + + #[test] + fn ignores_target_dir_and_missing_manifest() { + serialized(|| { + let root = temp_dir("ignore"); + let dep = make_dep(&root, "dep", "pub fn x() {}"); + write(&dep.join("target/debug/artifact.o"), "binary junk"); + let with_target = hash_tree(&dep).unwrap(); + let _ = fs::remove_dir_all(dep.join("target")); + let without_target = hash_tree(&dep).unwrap(); + assert_eq!(with_target, without_target, "target/ must be excluded"); + + let no_manifest = temp_dir("no-manifest"); + assert!(compute_all(&no_manifest).is_empty()); + let _ = fs::remove_dir_all(&root); + let _ = fs::remove_dir_all(&no_manifest); + }); + } + + #[test] + fn recursive_cycles_terminate() { + serialized(|| { + let root = temp_dir("cycle"); + make_dep(&root, "dep", "pub fn c() {}"); + write( + &root.join("Cargo.toml"), + "[package]\nname = \"app\"\nversion = \"0.1.0\"\n[dependencies]\nselfish = { path = \".\" }\ndep = { path = \"dep\" }\n", + ); + let map = compute_all(&root); + assert!(map.keys().any(|k| k.ends_with("dep"))); + let _ = fs::remove_dir_all(&root); + }); + } +} diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index f959a5edd2..ff97af773c 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -27,6 +27,11 @@ use crate::{is_redox, log_to_pty, wrap_io_err}; pub struct DepHashes { #[serde(flatten)] pub hashes: BTreeMap, + /// Content hashes of Cargo `path =` dependency source trees (keyed by + /// canonical path). Recipe PKGAR hashes above cannot see these — see + /// `crate::cook::cargo_path_deps`. + #[serde(default)] + pub cargo_path_deps: BTreeMap, } impl DepHashes { @@ -762,14 +767,19 @@ pub fn build( let mut all_current = current; all_current.extend(current_host); let changed = dep_hashes_changed(&stored_hashes.hashes, &all_current); + let cargo_deps_current = crate::cook::cargo_path_deps::compute_all(source_dir); + let cargo_changed = stored_hashes.cargo_path_deps != cargo_deps_current; if cli_verbose { if changed { log_to_pty!(logger, "DEBUG: dependency blake3 hashes changed"); } else { log_to_pty!(logger, "DEBUG: dependency blake3 hashes unchanged (content-based cache)"); } + if cargo_changed { + log_to_pty!(logger, "DEBUG: cargo path-dep source hashes changed"); + } } - changed + changed || cargo_changed } Ok(None) => { if cli_verbose { @@ -1036,7 +1046,10 @@ pub fn build( let auto_deps = make_auto_deps!(false)?; - let dep_hashes = DepHashes { hashes: current_dep_hashes }; + let dep_hashes = DepHashes { + hashes: current_dep_hashes, + cargo_path_deps: crate::cook::cargo_path_deps::compute_all(source_dir), + }; if let Err(e) = dep_hashes.write(&dep_hashes_file) { log_to_pty!(logger, "WARN: failed to write dep_hashes.toml: {e}"); } @@ -1354,6 +1367,7 @@ mod tests { m.insert("relibc".to_string(), "def456".to_string()); m }, + cargo_path_deps: BTreeMap::new(), }; original.write(&path).expect("write should succeed");