cookbook: track Cargo path-dep source trees in the dep-hash cache
The content-hash cache covered recipe-level PKGARs and the recipe's own source tree, but not Cargo 'path =' dependency sources inside a crate — the exact pattern the local fork model mandates (redox_syscall, libredox, redox-driver-*, pcid_interface). Editing a path-dep's source left the dependent recipe cached and stale. Observed live during the driver-manager QEMU gate: 'cook driver-manager - cached' while redox-driver-core sources had changed; the markers never reached the ISO. Fix: - New cook::cargo_path_deps module: resolves path deps from Cargo.toml ([dependencies], dev/build deps, target-specific deps, [patch.*] tables, workspace members) recursively with cycle protection, and hashes each resolved source tree (sorted relpath+content walk, excluding .git/target — same policy as SourceContentHash). - DepHashes gains a cargo_path_deps table (serde default for backward compatibility; a missing field invalidates once, then re-syncs). - The rebuild decision now ORs recipe-PKGAR changes with cargo path-dep tree changes; the post-build write records the new map. Acceptance: cook -> cached; content edit in redox-driver-core -> 'DEBUG: cargo path-dep source hashes changed' + rebuild; revert -> rebuild once -> cached. 48 tests pass (43 cookbook lib + 5 new).
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
// avoid confusion with build.rs
|
||||
pub mod cargo_path_deps;
|
||||
pub mod cook_build;
|
||||
pub mod fetch;
|
||||
pub mod fetch_repo;
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
//! Cargo path-dependency source tracking for content-hash caching.
|
||||
//!
|
||||
//! The recipe dep-hash cache (`DepHashes`) covers recipe-level PKGAR
|
||||
//! dependencies, and `SourceContentHash` covers the recipe's own source
|
||||
//! tree. Neither sees Cargo `path =` dependencies inside a crate — the
|
||||
//! exact pattern the local fork model mandates (`redox_syscall`,
|
||||
//! `libredox`, `redox-driver-*`, `pcid_interface` are all consumed this
|
||||
//! way). Without this module, editing a path-dep's source leaves the
|
||||
//! dependent recipe cached and stale (observed: `cook driver-manager -
|
||||
//! cached` while `redox-driver-core` sources had changed).
|
||||
//!
|
||||
//! The resolver below walks each recipe source's `Cargo.toml`
|
||||
//! (dependencies, dev/build dependencies, target-specific deps,
|
||||
//! `[patch.*]` tables, and workspace members) recursively, and hashes
|
||||
//! every resolved path-dep source tree into the stored dep-hash record.
|
||||
//! A content change anywhere in a path-dep tree invalidates the recipe.
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// Compute the cargo path-dep tree hashes for a recipe source dir.
|
||||
/// Keys are canonical dep paths; values are tree content hashes.
|
||||
/// An empty map means "no Cargo manifest or no path deps" — never an error.
|
||||
pub fn compute_all(source_dir: &Path) -> BTreeMap<String, String> {
|
||||
let mut visited = BTreeSet::new();
|
||||
let mut out = BTreeMap::new();
|
||||
let manifest = source_dir.join("Cargo.toml");
|
||||
if manifest.is_file() {
|
||||
resolve_recursive(&manifest, &mut visited, &mut out);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn resolve_recursive(
|
||||
manifest_path: &Path,
|
||||
visited: &mut BTreeSet<PathBuf>,
|
||||
out: &mut BTreeMap<String, String>,
|
||||
) {
|
||||
let Ok(manifest_path) = manifest_path.canonicalize() else {
|
||||
return;
|
||||
};
|
||||
if !visited.insert(manifest_path.clone()) {
|
||||
return;
|
||||
}
|
||||
let Ok(content) = fs::read_to_string(&manifest_path) else {
|
||||
return;
|
||||
};
|
||||
let Ok(doc) = content.parse::<toml::Value>() else {
|
||||
return;
|
||||
};
|
||||
let Some(manifest_dir) = manifest_path.parent() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut dep_paths = Vec::new();
|
||||
collect_dependency_paths(&doc, &mut dep_paths);
|
||||
collect_workspace_member_paths(&doc, &mut dep_paths);
|
||||
|
||||
for rel in dep_paths {
|
||||
let candidate = manifest_dir.join(&rel);
|
||||
let Ok(dep_dir) = candidate.canonicalize() else {
|
||||
continue;
|
||||
};
|
||||
if !dep_dir.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let key = dep_dir.to_string_lossy().to_string();
|
||||
if !out.contains_key(&key) {
|
||||
let hash = hash_tree(&dep_dir).unwrap_or_else(|_| "<unreadable>".to_string());
|
||||
out.insert(key, hash);
|
||||
}
|
||||
let dep_manifest = dep_dir.join("Cargo.toml");
|
||||
if dep_manifest.is_file() {
|
||||
resolve_recursive(&dep_manifest, visited, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract every `path = "..."` value from dependency tables:
|
||||
/// `[dependencies]`, `[dev-dependencies]`, `[build-dependencies]`,
|
||||
/// `[target.<triple>.dependencies*]`, and `[patch.<registry>]`.
|
||||
fn collect_dependency_paths(doc: &toml::Value, out: &mut Vec<PathBuf>) {
|
||||
const TABLES: &[&str] = &["dependencies", "dev-dependencies", "build-dependencies"];
|
||||
|
||||
if let Some(root) = doc.as_table() {
|
||||
for (key, value) in root {
|
||||
if TABLES.contains(&key.as_str()) {
|
||||
collect_from_dep_table(value, out);
|
||||
} else if key == "patch" {
|
||||
if let Some(patch_tables) = value.as_table() {
|
||||
for (_registry, table) in patch_tables {
|
||||
collect_from_dep_table(table, out);
|
||||
}
|
||||
}
|
||||
} else if key == "target" {
|
||||
if let Some(targets) = value.as_table() {
|
||||
for (_triple, target_table) in targets {
|
||||
if let Some(t) = target_table.as_table() {
|
||||
for (tkey, tval) in t {
|
||||
if TABLES.contains(&tkey.as_str()) {
|
||||
collect_from_dep_table(tval, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_from_dep_table(table: &toml::Value, out: &mut Vec<PathBuf>) {
|
||||
let Some(entries) = table.as_table() else {
|
||||
return;
|
||||
};
|
||||
for (_name, spec) in entries {
|
||||
match spec {
|
||||
toml::Value::String(_) => {}
|
||||
toml::Value::Table(spec_table) => {
|
||||
if let Some(toml::Value::String(path)) = spec_table.get("path") {
|
||||
out.push(PathBuf::from(path));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract workspace member manifests: `[workspace] members = [...]`.
|
||||
fn collect_workspace_member_paths(doc: &toml::Value, out: &mut Vec<PathBuf>) {
|
||||
let Some(workspace) = doc.get("workspace") else {
|
||||
return;
|
||||
};
|
||||
let Some(members) = workspace.get("members").and_then(|m| m.as_array()) else {
|
||||
return;
|
||||
};
|
||||
for member in members {
|
||||
if let Some(toml::Value::String(path)) = Some(member) {
|
||||
out.push(PathBuf::from(path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Content-hash a path-dep source tree: sorted (relative path, content)
|
||||
/// pairs, excluding `.git/`, `target/`, and editor artifacts. Matches the
|
||||
/// exclusion policy of `SourceContentHash::compute` so a file that is
|
||||
/// invisible there is also invisible here.
|
||||
fn hash_tree(dir: &Path) -> std::io::Result<String> {
|
||||
let mut paths = Vec::new();
|
||||
collect_files(dir, dir, &mut paths)?;
|
||||
paths.sort();
|
||||
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
for rel in &paths {
|
||||
let rel_str = rel.to_string_lossy();
|
||||
if rel_str.starts_with(".git/")
|
||||
|| rel_str.starts_with("target/")
|
||||
|| rel_str.contains("/target/")
|
||||
|| rel_str.ends_with(".swp")
|
||||
|| rel_str.ends_with(".tmp")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
hasher.update(rel_str.as_bytes());
|
||||
hasher.update(b"\0");
|
||||
let content = fs::read(dir.join(rel)).unwrap_or_else(|_| b"<unreadable>".to_vec());
|
||||
hasher.update(&content);
|
||||
hasher.update(b"\0");
|
||||
}
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
|
||||
fn collect_files(dir: &Path, root: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
|
||||
for entry in fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
let Ok(rel) = path.strip_prefix(root).map(|r| r.to_path_buf()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry.file_type() else {
|
||||
continue;
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
collect_files(&path, root, out)?;
|
||||
} else if file_type.is_file() || file_type.is_symlink() {
|
||||
out.push(rel);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn serialized<R>(body: impl FnOnce() -> R) -> R {
|
||||
let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
body()
|
||||
}
|
||||
|
||||
fn temp_dir(label: &str) -> PathBuf {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
std::env::temp_dir().join(format!("rb-cargo-deps-{label}-{}-{nanos}", std::process::id()))
|
||||
}
|
||||
|
||||
fn write(path: &Path, body: &str) {
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, body).unwrap();
|
||||
}
|
||||
|
||||
fn make_dep(root: &Path, name: &str, lib_body: &str) -> PathBuf {
|
||||
let dep = root.join(name);
|
||||
write(&dep.join("Cargo.toml"), &format!("[package]\nname = \"{name}\"\nversion = \"0.1.0\"\n"));
|
||||
write(&dep.join("src/lib.rs"), lib_body);
|
||||
dep
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_table_and_patch_path_deps_recursively() {
|
||||
serialized(|| {
|
||||
let root = temp_dir("resolve");
|
||||
let dep_a = make_dep(&root, "dep-a", "pub fn a() {}");
|
||||
let dep_b = make_dep(&root, "dep-b", "pub fn b() {}");
|
||||
write(
|
||||
&dep_a.join("src/lib.rs"),
|
||||
"pub fn a() { dep_b::b(); }",
|
||||
);
|
||||
write(
|
||||
&dep_a.join("Cargo.toml"),
|
||||
"[package]\nname = \"dep-a\"\nversion = \"0.1.0\"\n[dependencies]\ndep-b = { path = \"../dep-b\" }\n",
|
||||
);
|
||||
write(
|
||||
&root.join("Cargo.toml"),
|
||||
"[package]\nname = \"app\"\nversion = \"0.1.0\"\n\
|
||||
[dependencies]\ndep-a = { path = \"dep-a\" }\nplain = \"1.0\"\n\
|
||||
[patch.crates-io]\npatched = { path = \"dep-b\" }\n",
|
||||
);
|
||||
|
||||
let map = compute_all(&root);
|
||||
let keys: Vec<String> = map.keys().cloned().collect();
|
||||
assert_eq!(keys.len(), 2, "expected dep-a and dep-b, got {keys:?}");
|
||||
assert!(keys.iter().any(|k| k.ends_with("dep-a")));
|
||||
assert!(keys.iter().any(|k| k.ends_with("dep-b")));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_workspace_members() {
|
||||
serialized(|| {
|
||||
let root = temp_dir("workspace");
|
||||
make_dep(&root, "member-one", "pub fn one() {}");
|
||||
write(
|
||||
&root.join("Cargo.toml"),
|
||||
"[workspace]\nmembers = [\"member-one\"]\n",
|
||||
);
|
||||
let map = compute_all(&root);
|
||||
assert_eq!(map.len(), 1);
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_changes_with_content_and_file_list() {
|
||||
serialized(|| {
|
||||
let root = temp_dir("hash");
|
||||
let dep = make_dep(&root, "dep", "pub fn v() -> u32 { 1 }");
|
||||
let first = hash_tree(&dep).unwrap();
|
||||
let second = hash_tree(&dep).unwrap();
|
||||
assert_eq!(first, second, "identical tree must hash identically");
|
||||
|
||||
write(&dep.join("src/lib.rs"), "pub fn v() -> u32 { 2 }");
|
||||
let third = hash_tree(&dep).unwrap();
|
||||
assert_ne!(first, third, "content edit must change the hash");
|
||||
|
||||
write(&dep.join("src/new.rs"), "pub fn added() {}");
|
||||
let fourth = hash_tree(&dep).unwrap();
|
||||
assert_ne!(third, fourth, "new file must change the hash");
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_target_dir_and_missing_manifest() {
|
||||
serialized(|| {
|
||||
let root = temp_dir("ignore");
|
||||
let dep = make_dep(&root, "dep", "pub fn x() {}");
|
||||
write(&dep.join("target/debug/artifact.o"), "binary junk");
|
||||
let with_target = hash_tree(&dep).unwrap();
|
||||
let _ = fs::remove_dir_all(dep.join("target"));
|
||||
let without_target = hash_tree(&dep).unwrap();
|
||||
assert_eq!(with_target, without_target, "target/ must be excluded");
|
||||
|
||||
let no_manifest = temp_dir("no-manifest");
|
||||
assert!(compute_all(&no_manifest).is_empty());
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
let _ = fs::remove_dir_all(&no_manifest);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_cycles_terminate() {
|
||||
serialized(|| {
|
||||
let root = temp_dir("cycle");
|
||||
make_dep(&root, "dep", "pub fn c() {}");
|
||||
write(
|
||||
&root.join("Cargo.toml"),
|
||||
"[package]\nname = \"app\"\nversion = \"0.1.0\"\n[dependencies]\nselfish = { path = \".\" }\ndep = { path = \"dep\" }\n",
|
||||
);
|
||||
let map = compute_all(&root);
|
||||
assert!(map.keys().any(|k| k.ends_with("dep")));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
});
|
||||
}
|
||||
}
|
||||
+16
-2
@@ -27,6 +27,11 @@ use crate::{is_redox, log_to_pty, wrap_io_err};
|
||||
pub struct DepHashes {
|
||||
#[serde(flatten)]
|
||||
pub hashes: BTreeMap<String, String>,
|
||||
/// Content hashes of Cargo `path =` dependency source trees (keyed by
|
||||
/// canonical path). Recipe PKGAR hashes above cannot see these — see
|
||||
/// `crate::cook::cargo_path_deps`.
|
||||
#[serde(default)]
|
||||
pub cargo_path_deps: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl DepHashes {
|
||||
@@ -762,14 +767,19 @@ pub fn build(
|
||||
let mut all_current = current;
|
||||
all_current.extend(current_host);
|
||||
let changed = dep_hashes_changed(&stored_hashes.hashes, &all_current);
|
||||
let cargo_deps_current = crate::cook::cargo_path_deps::compute_all(source_dir);
|
||||
let cargo_changed = stored_hashes.cargo_path_deps != cargo_deps_current;
|
||||
if cli_verbose {
|
||||
if changed {
|
||||
log_to_pty!(logger, "DEBUG: dependency blake3 hashes changed");
|
||||
} else {
|
||||
log_to_pty!(logger, "DEBUG: dependency blake3 hashes unchanged (content-based cache)");
|
||||
}
|
||||
if cargo_changed {
|
||||
log_to_pty!(logger, "DEBUG: cargo path-dep source hashes changed");
|
||||
}
|
||||
}
|
||||
changed
|
||||
changed || cargo_changed
|
||||
}
|
||||
Ok(None) => {
|
||||
if cli_verbose {
|
||||
@@ -1036,7 +1046,10 @@ pub fn build(
|
||||
|
||||
let auto_deps = make_auto_deps!(false)?;
|
||||
|
||||
let dep_hashes = DepHashes { hashes: current_dep_hashes };
|
||||
let dep_hashes = DepHashes {
|
||||
hashes: current_dep_hashes,
|
||||
cargo_path_deps: crate::cook::cargo_path_deps::compute_all(source_dir),
|
||||
};
|
||||
if let Err(e) = dep_hashes.write(&dep_hashes_file) {
|
||||
log_to_pty!(logger, "WARN: failed to write dep_hashes.toml: {e}");
|
||||
}
|
||||
@@ -1354,6 +1367,7 @@ mod tests {
|
||||
m.insert("relibc".to_string(), "def456".to_string());
|
||||
m
|
||||
},
|
||||
cargo_path_deps: BTreeMap::new(),
|
||||
};
|
||||
|
||||
original.write(&path).expect("write should succeed");
|
||||
|
||||
Reference in New Issue
Block a user