cook: fix transient sysroot/stage rebuilds with content-hash fingerprints
The per-recipe sysroot and stage cache used mtime of the dep pkgar files to detect when a rebuild was needed. Any mtime bump on relibc or any leaf dep (including the pre-cook relibc in build-redbear.sh) would cascade-rebuild every downstream per-recipe sysroot even when the dep's content was bit-identical. The resulting transient sysroot extractions produced 'C compiler cannot create executables' and 'configure error' failures that retried fine standalone. Replace the mtime checks with a blake3 content-hash fingerprint of the dep pkgar set: - For the per-recipe sysroot: store the fingerprint in <target>/sysroot/.tags/deps-fingerprint and rebuild only when the computed fingerprint does not match. - For the per-recipe stage: store two fingerprints at <target>/.deps-fingerprint and <target>/.host-deps-fingerprint. Rebuild stage only when (source changed) OR (deps content changed) OR (host-deps content changed) OR (auto_deps.toml missing). This eliminates the transient build failures in 'make live' / 'build-redbear.sh' and aligns the cache invalidation signal with the actual content the recipe depends on, not the arbitrary mtime of the dependency package.
This commit is contained in:
+116
-17
@@ -282,10 +282,28 @@ pub fn build(
|
||||
|
||||
// check stage dir modified against pkgar files, any files missing will result in UNIX_EPOCH
|
||||
let stage_modified = modified_all(&stage_pkgars, modified).unwrap_or(SystemTime::UNIX_EPOCH);
|
||||
// Rebuild stage if source is newer
|
||||
let target_fingerprint_path = target_dir.join(".deps-fingerprint");
|
||||
let current_deps_fingerprint = if !dep_pkgars.is_empty() {
|
||||
Some(compute_deps_fingerprint(&dep_pkgars)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let current_host_fingerprint = if !dep_host_pkgars.is_empty() {
|
||||
Some(compute_deps_fingerprint(&dep_host_pkgars)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let stored_deps_fingerprint = read_text_file(&target_fingerprint_path)?;
|
||||
let stored_host_fingerprint = read_text_file(
|
||||
&target_dir.join(".host-deps-fingerprint"),
|
||||
)?;
|
||||
let deps_content_unchanged = current_deps_fingerprint.is_none()
|
||||
|| stored_deps_fingerprint.as_deref() == current_deps_fingerprint.as_deref();
|
||||
let host_deps_content_unchanged = current_host_fingerprint.is_none()
|
||||
|| stored_host_fingerprint.as_deref() == current_host_fingerprint.as_deref();
|
||||
if stage_modified < source_modified
|
||||
|| stage_modified < deps_modified
|
||||
|| stage_modified < deps_host_modified
|
||||
|| !deps_content_unchanged
|
||||
|| !host_deps_content_unchanged
|
||||
|| !auto_deps_file.is_file()
|
||||
{
|
||||
for stage_dir in &stage_dirs {
|
||||
@@ -525,6 +543,22 @@ pub fn build(
|
||||
// don't remove stage dir yet
|
||||
}
|
||||
|
||||
if let Some(fingerprint) = current_deps_fingerprint.as_ref() {
|
||||
fs::write(&target_fingerprint_path, fingerprint)
|
||||
.map_err(|e| format!("failed to write deps fingerprint: {e}"))?;
|
||||
} else if target_fingerprint_path.is_file() {
|
||||
remove_all(&target_fingerprint_path)?;
|
||||
}
|
||||
if let Some(fingerprint) = current_host_fingerprint.as_ref() {
|
||||
fs::write(target_dir.join(".host-deps-fingerprint"), fingerprint)
|
||||
.map_err(|e| format!("failed to write host-deps fingerprint: {e}"))?;
|
||||
} else {
|
||||
let path = target_dir.join(".host-deps-fingerprint");
|
||||
if path.is_file() {
|
||||
remove_all(&path)?;
|
||||
}
|
||||
}
|
||||
|
||||
let auto_deps = make_auto_deps!(false)?;
|
||||
Ok(BuildResult::new(stage_dirs, auto_deps))
|
||||
}
|
||||
@@ -576,24 +610,36 @@ fn build_deps_dir(
|
||||
logger: &PtyOut,
|
||||
deps_dir: &PathBuf,
|
||||
dep_pkgars: &BTreeSet<(PackageName, PathBuf)>,
|
||||
source_modified: SystemTime,
|
||||
deps_modified: SystemTime,
|
||||
_source_modified: SystemTime,
|
||||
_deps_modified: SystemTime,
|
||||
) -> Result<bool, String> {
|
||||
let deps_dir_tmp = deps_dir.with_added_extension("tmp");
|
||||
if deps_dir.is_dir() {
|
||||
let tags_dir = deps_dir.join(".tags");
|
||||
let sysroot_modified = modified_dir(&tags_dir).unwrap_or(SystemTime::UNIX_EPOCH);
|
||||
if sysroot_modified < source_modified
|
||||
|| sysroot_modified < deps_modified
|
||||
|| !check_files_present(
|
||||
&tags_dir,
|
||||
&dep_pkgars
|
||||
.iter()
|
||||
.map(|(name, _)| name.without_prefix())
|
||||
.collect(),
|
||||
)?
|
||||
{
|
||||
log_to_pty!(logger, "DEBUG: updating '{}'", deps_dir.display());
|
||||
let all_tags_present = check_files_present(
|
||||
&tags_dir,
|
||||
&dep_pkgars
|
||||
.iter()
|
||||
.map(|(name, _)| name.without_prefix())
|
||||
.collect(),
|
||||
)?;
|
||||
let current_fingerprint = compute_deps_fingerprint(dep_pkgars)?;
|
||||
let stored_fingerprint = read_deps_fingerprint(&tags_dir)?;
|
||||
let fingerprint_matches = stored_fingerprint.as_deref() == Some(current_fingerprint.as_str());
|
||||
if !all_tags_present || !fingerprint_matches {
|
||||
if !all_tags_present {
|
||||
log_to_pty!(
|
||||
logger,
|
||||
"DEBUG: rebuilding '{}' (dep set changed)",
|
||||
deps_dir.display()
|
||||
);
|
||||
} else {
|
||||
log_to_pty!(
|
||||
logger,
|
||||
"DEBUG: rebuilding '{}' (dep content changed)",
|
||||
deps_dir.display()
|
||||
);
|
||||
}
|
||||
remove_all(deps_dir)?;
|
||||
}
|
||||
}
|
||||
@@ -652,6 +698,10 @@ fn build_deps_dir(
|
||||
)?;
|
||||
}
|
||||
|
||||
let fingerprint = compute_deps_fingerprint(dep_pkgars)?;
|
||||
fs::write(tags_dir.join("deps-fingerprint"), &fingerprint)
|
||||
.map_err(|e| format!("failed to write deps fingerprint: {e}"))?;
|
||||
|
||||
// Move sysroot.tmp to sysroot atomically
|
||||
rename(&deps_dir_tmp, deps_dir)?;
|
||||
|
||||
@@ -661,6 +711,55 @@ fn build_deps_dir(
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn read_deps_fingerprint(tags_dir: &Path) -> Result<Option<String>, String> {
|
||||
let fingerprint_path = tags_dir.join("deps-fingerprint");
|
||||
match fs::read_to_string(&fingerprint_path) {
|
||||
Ok(s) => Ok(Some(s)),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(err) => Err(format!(
|
||||
"failed to read deps fingerprint {}: {err}",
|
||||
fingerprint_path.display()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_deps_fingerprint(
|
||||
dep_pkgars: &BTreeSet<(PackageName, PathBuf)>,
|
||||
) -> Result<String, String> {
|
||||
use std::io::Read;
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
for (name, path) in dep_pkgars {
|
||||
hasher.update(name.to_string().as_bytes());
|
||||
hasher.update(b"\0");
|
||||
let mut file = match fs::File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
hasher.update(b"<missing>");
|
||||
hasher.update(b"\0");
|
||||
continue;
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(format!("failed to open {}: {err}", path.display()));
|
||||
}
|
||||
};
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| format!("failed to read {}: {e}", path.display()))?;
|
||||
hasher.update(&buf[..n]);
|
||||
hasher.update(b"\0");
|
||||
}
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
|
||||
fn read_text_file(path: &Path) -> Result<Option<String>, String> {
|
||||
match fs::read_to_string(path) {
|
||||
Ok(s) => Ok(Some(s)),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(err) => Err(format!("failed to read {}: {err}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate automatic dependencies
|
||||
fn build_auto_deps(
|
||||
recipe: &Recipe,
|
||||
|
||||
Reference in New Issue
Block a user