cookbook: track Cargo path-dep source trees in the dep-hash cache

The content-hash cache covered recipe-level PKGARs and the recipe's own
source tree, but not Cargo 'path =' dependency sources inside a crate —
the exact pattern the local fork model mandates (redox_syscall, libredox,
redox-driver-*, pcid_interface). Editing a path-dep's source left the
dependent recipe cached and stale. Observed live during the
driver-manager QEMU gate: 'cook driver-manager - cached' while
redox-driver-core sources had changed; the markers never reached the ISO.

Fix:
- New cook::cargo_path_deps module: resolves path deps from Cargo.toml
  ([dependencies], dev/build deps, target-specific deps, [patch.*]
  tables, workspace members) recursively with cycle protection, and
  hashes each resolved source tree (sorted relpath+content walk,
  excluding .git/target — same policy as SourceContentHash).
- DepHashes gains a cargo_path_deps table (serde default for backward
  compatibility; a missing field invalidates once, then re-syncs).
- The rebuild decision now ORs recipe-PKGAR changes with cargo path-dep
  tree changes; the post-build write records the new map.

Acceptance: cook -> cached; content edit in redox-driver-core ->
'DEBUG: cargo path-dep source hashes changed' + rebuild; revert ->
rebuild once -> cached. 48 tests pass (43 cookbook lib + 5 new).
This commit is contained in:
2026-07-23 23:54:41 +09:00
parent 92b3d93de7
commit 6ce94d1c9e
3 changed files with 341 additions and 2 deletions
+1
View File
@@ -1,4 +1,5 @@
// avoid confusion with build.rs
pub mod cargo_path_deps;
pub mod cook_build;
pub mod fetch;
pub mod fetch_repo;
+324
View File
@@ -0,0 +1,324 @@
//! Cargo path-dependency source tracking for content-hash caching.
//!
//! The recipe dep-hash cache (`DepHashes`) covers recipe-level PKGAR
//! dependencies, and `SourceContentHash` covers the recipe's own source
//! tree. Neither sees Cargo `path =` dependencies inside a crate — the
//! exact pattern the local fork model mandates (`redox_syscall`,
//! `libredox`, `redox-driver-*`, `pcid_interface` are all consumed this
//! way). Without this module, editing a path-dep's source leaves the
//! dependent recipe cached and stale (observed: `cook driver-manager -
//! cached` while `redox-driver-core` sources had changed).
//!
//! The resolver below walks each recipe source's `Cargo.toml`
//! (dependencies, dev/build dependencies, target-specific deps,
//! `[patch.*]` tables, and workspace members) recursively, and hashes
//! every resolved path-dep source tree into the stored dep-hash record.
//! A content change anywhere in a path-dep tree invalidates the recipe.
use std::{
collections::{BTreeMap, BTreeSet},
fs,
path::{Path, PathBuf},
};
/// Compute the cargo path-dep tree hashes for a recipe source dir.
/// Keys are canonical dep paths; values are tree content hashes.
/// An empty map means "no Cargo manifest or no path deps" — never an error.
pub fn compute_all(source_dir: &Path) -> BTreeMap<String, String> {
let mut visited = BTreeSet::new();
let mut out = BTreeMap::new();
let manifest = source_dir.join("Cargo.toml");
if manifest.is_file() {
resolve_recursive(&manifest, &mut visited, &mut out);
}
out
}
fn resolve_recursive(
manifest_path: &Path,
visited: &mut BTreeSet<PathBuf>,
out: &mut BTreeMap<String, String>,
) {
let Ok(manifest_path) = manifest_path.canonicalize() else {
return;
};
if !visited.insert(manifest_path.clone()) {
return;
}
let Ok(content) = fs::read_to_string(&manifest_path) else {
return;
};
let Ok(doc) = content.parse::<toml::Value>() else {
return;
};
let Some(manifest_dir) = manifest_path.parent() else {
return;
};
let mut dep_paths = Vec::new();
collect_dependency_paths(&doc, &mut dep_paths);
collect_workspace_member_paths(&doc, &mut dep_paths);
for rel in dep_paths {
let candidate = manifest_dir.join(&rel);
let Ok(dep_dir) = candidate.canonicalize() else {
continue;
};
if !dep_dir.is_dir() {
continue;
}
let key = dep_dir.to_string_lossy().to_string();
if !out.contains_key(&key) {
let hash = hash_tree(&dep_dir).unwrap_or_else(|_| "<unreadable>".to_string());
out.insert(key, hash);
}
let dep_manifest = dep_dir.join("Cargo.toml");
if dep_manifest.is_file() {
resolve_recursive(&dep_manifest, visited, out);
}
}
}
/// Extract every `path = "..."` value from dependency tables:
/// `[dependencies]`, `[dev-dependencies]`, `[build-dependencies]`,
/// `[target.<triple>.dependencies*]`, and `[patch.<registry>]`.
fn collect_dependency_paths(doc: &toml::Value, out: &mut Vec<PathBuf>) {
const TABLES: &[&str] = &["dependencies", "dev-dependencies", "build-dependencies"];
if let Some(root) = doc.as_table() {
for (key, value) in root {
if TABLES.contains(&key.as_str()) {
collect_from_dep_table(value, out);
} else if key == "patch" {
if let Some(patch_tables) = value.as_table() {
for (_registry, table) in patch_tables {
collect_from_dep_table(table, out);
}
}
} else if key == "target" {
if let Some(targets) = value.as_table() {
for (_triple, target_table) in targets {
if let Some(t) = target_table.as_table() {
for (tkey, tval) in t {
if TABLES.contains(&tkey.as_str()) {
collect_from_dep_table(tval, out);
}
}
}
}
}
}
}
}
}
fn collect_from_dep_table(table: &toml::Value, out: &mut Vec<PathBuf>) {
let Some(entries) = table.as_table() else {
return;
};
for (_name, spec) in entries {
match spec {
toml::Value::String(_) => {}
toml::Value::Table(spec_table) => {
if let Some(toml::Value::String(path)) = spec_table.get("path") {
out.push(PathBuf::from(path));
}
}
_ => {}
}
}
}
/// Extract workspace member manifests: `[workspace] members = [...]`.
fn collect_workspace_member_paths(doc: &toml::Value, out: &mut Vec<PathBuf>) {
let Some(workspace) = doc.get("workspace") else {
return;
};
let Some(members) = workspace.get("members").and_then(|m| m.as_array()) else {
return;
};
for member in members {
if let Some(toml::Value::String(path)) = Some(member) {
out.push(PathBuf::from(path));
}
}
}
/// Content-hash a path-dep source tree: sorted (relative path, content)
/// pairs, excluding `.git/`, `target/`, and editor artifacts. Matches the
/// exclusion policy of `SourceContentHash::compute` so a file that is
/// invisible there is also invisible here.
fn hash_tree(dir: &Path) -> std::io::Result<String> {
let mut paths = Vec::new();
collect_files(dir, dir, &mut paths)?;
paths.sort();
let mut hasher = blake3::Hasher::new();
for rel in &paths {
let rel_str = rel.to_string_lossy();
if rel_str.starts_with(".git/")
|| rel_str.starts_with("target/")
|| rel_str.contains("/target/")
|| rel_str.ends_with(".swp")
|| rel_str.ends_with(".tmp")
{
continue;
}
hasher.update(rel_str.as_bytes());
hasher.update(b"\0");
let content = fs::read(dir.join(rel)).unwrap_or_else(|_| b"<unreadable>".to_vec());
hasher.update(&content);
hasher.update(b"\0");
}
Ok(hasher.finalize().to_hex().to_string())
}
fn collect_files(dir: &Path, root: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
for entry in fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
let Ok(rel) = path.strip_prefix(root).map(|r| r.to_path_buf()) else {
continue;
};
let Ok(file_type) = entry.file_type() else {
continue;
};
if file_type.is_dir() {
collect_files(&path, root, out)?;
} else if file_type.is_file() || file_type.is_symlink() {
out.push(rel);
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
static ENV_LOCK: Mutex<()> = Mutex::new(());
fn serialized<R>(body: impl FnOnce() -> R) -> R {
let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
body()
}
fn temp_dir(label: &str) -> PathBuf {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
std::env::temp_dir().join(format!("rb-cargo-deps-{label}-{}-{nanos}", std::process::id()))
}
fn write(path: &Path, body: &str) {
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, body).unwrap();
}
fn make_dep(root: &Path, name: &str, lib_body: &str) -> PathBuf {
let dep = root.join(name);
write(&dep.join("Cargo.toml"), &format!("[package]\nname = \"{name}\"\nversion = \"0.1.0\"\n"));
write(&dep.join("src/lib.rs"), lib_body);
dep
}
#[test]
fn resolves_table_and_patch_path_deps_recursively() {
serialized(|| {
let root = temp_dir("resolve");
let dep_a = make_dep(&root, "dep-a", "pub fn a() {}");
let dep_b = make_dep(&root, "dep-b", "pub fn b() {}");
write(
&dep_a.join("src/lib.rs"),
"pub fn a() { dep_b::b(); }",
);
write(
&dep_a.join("Cargo.toml"),
"[package]\nname = \"dep-a\"\nversion = \"0.1.0\"\n[dependencies]\ndep-b = { path = \"../dep-b\" }\n",
);
write(
&root.join("Cargo.toml"),
"[package]\nname = \"app\"\nversion = \"0.1.0\"\n\
[dependencies]\ndep-a = { path = \"dep-a\" }\nplain = \"1.0\"\n\
[patch.crates-io]\npatched = { path = \"dep-b\" }\n",
);
let map = compute_all(&root);
let keys: Vec<String> = map.keys().cloned().collect();
assert_eq!(keys.len(), 2, "expected dep-a and dep-b, got {keys:?}");
assert!(keys.iter().any(|k| k.ends_with("dep-a")));
assert!(keys.iter().any(|k| k.ends_with("dep-b")));
let _ = fs::remove_dir_all(&root);
});
}
#[test]
fn resolves_workspace_members() {
serialized(|| {
let root = temp_dir("workspace");
make_dep(&root, "member-one", "pub fn one() {}");
write(
&root.join("Cargo.toml"),
"[workspace]\nmembers = [\"member-one\"]\n",
);
let map = compute_all(&root);
assert_eq!(map.len(), 1);
let _ = fs::remove_dir_all(&root);
});
}
#[test]
fn hash_changes_with_content_and_file_list() {
serialized(|| {
let root = temp_dir("hash");
let dep = make_dep(&root, "dep", "pub fn v() -> u32 { 1 }");
let first = hash_tree(&dep).unwrap();
let second = hash_tree(&dep).unwrap();
assert_eq!(first, second, "identical tree must hash identically");
write(&dep.join("src/lib.rs"), "pub fn v() -> u32 { 2 }");
let third = hash_tree(&dep).unwrap();
assert_ne!(first, third, "content edit must change the hash");
write(&dep.join("src/new.rs"), "pub fn added() {}");
let fourth = hash_tree(&dep).unwrap();
assert_ne!(third, fourth, "new file must change the hash");
let _ = fs::remove_dir_all(&root);
});
}
#[test]
fn ignores_target_dir_and_missing_manifest() {
serialized(|| {
let root = temp_dir("ignore");
let dep = make_dep(&root, "dep", "pub fn x() {}");
write(&dep.join("target/debug/artifact.o"), "binary junk");
let with_target = hash_tree(&dep).unwrap();
let _ = fs::remove_dir_all(dep.join("target"));
let without_target = hash_tree(&dep).unwrap();
assert_eq!(with_target, without_target, "target/ must be excluded");
let no_manifest = temp_dir("no-manifest");
assert!(compute_all(&no_manifest).is_empty());
let _ = fs::remove_dir_all(&root);
let _ = fs::remove_dir_all(&no_manifest);
});
}
#[test]
fn recursive_cycles_terminate() {
serialized(|| {
let root = temp_dir("cycle");
make_dep(&root, "dep", "pub fn c() {}");
write(
&root.join("Cargo.toml"),
"[package]\nname = \"app\"\nversion = \"0.1.0\"\n[dependencies]\nselfish = { path = \".\" }\ndep = { path = \"dep\" }\n",
);
let map = compute_all(&root);
assert!(map.keys().any(|k| k.ends_with("dep")));
let _ = fs::remove_dir_all(&root);
});
}
}
+16 -2
View File
@@ -27,6 +27,11 @@ use crate::{is_redox, log_to_pty, wrap_io_err};
pub struct DepHashes {
#[serde(flatten)]
pub hashes: BTreeMap<String, String>,
/// Content hashes of Cargo `path =` dependency source trees (keyed by
/// canonical path). Recipe PKGAR hashes above cannot see these — see
/// `crate::cook::cargo_path_deps`.
#[serde(default)]
pub cargo_path_deps: BTreeMap<String, String>,
}
impl DepHashes {
@@ -762,14 +767,19 @@ pub fn build(
let mut all_current = current;
all_current.extend(current_host);
let changed = dep_hashes_changed(&stored_hashes.hashes, &all_current);
let cargo_deps_current = crate::cook::cargo_path_deps::compute_all(source_dir);
let cargo_changed = stored_hashes.cargo_path_deps != cargo_deps_current;
if cli_verbose {
if changed {
log_to_pty!(logger, "DEBUG: dependency blake3 hashes changed");
} else {
log_to_pty!(logger, "DEBUG: dependency blake3 hashes unchanged (content-based cache)");
}
if cargo_changed {
log_to_pty!(logger, "DEBUG: cargo path-dep source hashes changed");
}
}
changed
changed || cargo_changed
}
Ok(None) => {
if cli_verbose {
@@ -1036,7 +1046,10 @@ pub fn build(
let auto_deps = make_auto_deps!(false)?;
let dep_hashes = DepHashes { hashes: current_dep_hashes };
let dep_hashes = DepHashes {
hashes: current_dep_hashes,
cargo_path_deps: crate::cook::cargo_path_deps::compute_all(source_dir),
};
if let Err(e) = dep_hashes.write(&dep_hashes_file) {
log_to_pty!(logger, "WARN: failed to write dep_hashes.toml: {e}");
}
@@ -1354,6 +1367,7 @@ mod tests {
m.insert("relibc".to_string(), "def456".to_string());
m
},
cargo_path_deps: BTreeMap::new(),
};
original.write(&path).expect("write should succeed");