build: add audit-patch-idempotency.py + auto-link Qt sysroot dirs
Two S-sized improvements from BUILD-SYSTEM-IMPROVEMENTS.md: 1. local/scripts/audit-patch-idempotency.py (improvement #3): Validates that every external patch in local/patches/ is idempotent (--reverse --check succeeds) and reproducible (re-clone + re-apply produces an identical tree). Catches the patch idempotency class of bug at lint time, where it used to surface as a 2+ hour cookbook failure during a cook. Found a real bug on first run: local/patches/libdrm/02-redox-dispatch.patch has a hunk at xf86drm.c:321 that no longer matches the upstream libdrm-2.4.125. 2. src/cook/script.rs auto-link Qt sysroot dirs (improvement #8): The cookbook's BUILD_PRESCRIPT now auto-detects if the per-recipe sysroot has Qt6 (qtbase or qtdeclarative) and creates the canonical /usr/{plugins,mkspecs,metatypes,modules} symlinks that KF6 recipes need for cmake to find Qt6Config.cmake's INTERFACE_* paths. New KF6 recipes that depend on qtbase no longer need to manually call redbear_qt_link_sysroot_dirs in their build script. Recipes that need more customization can still call the helper directly via 'source $COOKBOOK_ROOT/local/scripts/lib/qt-sysroot.sh'.
This commit is contained in:
kellito
Executable
+331
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the idempotency of every external patch in local/patches/.
|
||||
|
||||
Per AGENTS.md "NO OVERLAY-STYLE PATCHES — AMENDED 2026" Rule 2, big
|
||||
external projects use the cookbook's `cookbook_apply_patches` helper
|
||||
which checks `git apply --reverse --check` to skip already-applied
|
||||
patches. If a patch's reverse check fails (because the upstream
|
||||
source drifted from the patch's expected state), the helper tries to
|
||||
apply the patch forward, which fails too because some hunks no
|
||||
longer apply. The result is a confusing cook failure.
|
||||
|
||||
This script catches that class of bug at lint time. For every
|
||||
[0-9]*.patch under local/patches/<component>/, it:
|
||||
|
||||
1. Clones the upstream repo at the pinned rev into a temp dir
|
||||
2. Applies the patch
|
||||
3. Verifies `git apply --reverse --check` succeeds on the result
|
||||
(i.e. the patch is fully reversible — idempotency invariant)
|
||||
4. Re-applies the patch
|
||||
5. Verifies the source is byte-identical to step 2's result
|
||||
(i.e. the patch is idempotent — applying it twice = applying it once)
|
||||
6. Verifies the result is reproducible: re-clone, re-apply, byte-equal
|
||||
|
||||
If any check fails, the script exits non-zero and prints which patches
|
||||
are non-idempotent. CI or `make lint` should run this on every PR.
|
||||
|
||||
Usage:
|
||||
./local/scripts/audit-patch-idempotency.py [--component <name>] [--verbose]
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
PATCHES_ROOT = PROJECT_ROOT / "local" / "patches"
|
||||
SOURCE_ROOT = PROJECT_ROOT / "local" / "sources"
|
||||
RECIPES_ROOT = PROJECT_ROOT / "local" / "recipes"
|
||||
MAINLINE_RECIPES = PROJECT_ROOT / "recipes"
|
||||
|
||||
PATCH_NAME_RE = re.compile(r"^\d+-[A-Za-z0-9_.-]+\.patch$")
|
||||
NUM_PREFIX_RE = re.compile(r"^(\d+)-")
|
||||
|
||||
|
||||
def run(cmd, **kwargs):
|
||||
"""Run a subprocess, returning (returncode, stdout, stderr)."""
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
**kwargs,
|
||||
)
|
||||
return proc.returncode, proc.stdout, proc.stderr
|
||||
|
||||
|
||||
def collect_patches(component_filter=None):
|
||||
"""Yield (component, patch_path) for every external patch."""
|
||||
if not PATCHES_ROOT.is_dir():
|
||||
return
|
||||
for component_dir in sorted(PATCHES_ROOT.iterdir()):
|
||||
if not component_dir.is_dir():
|
||||
continue
|
||||
if component_filter and component_dir.name != component_filter:
|
||||
continue
|
||||
for patch_path in sorted(component_dir.iterdir()):
|
||||
if patch_path.is_file() and PATCH_NAME_RE.match(patch_path.name):
|
||||
yield component_dir.name, patch_path
|
||||
|
||||
|
||||
def resolve_upstream(component) -> "tuple[str | None, str | None] | tuple[str, str | None, Path]":
|
||||
"""Return (url, rev) for a component by reading its mainline recipe.
|
||||
|
||||
The component is matched by the recipe.toml's parent directory name
|
||||
(e.g. recipes/libs/mesa/recipe.toml matches component="mesa"),
|
||||
not the category. This means multiple categories with the same
|
||||
package name (e.g. recipes/wip/demos/mesa-demos) won't accidentally
|
||||
match.
|
||||
"""
|
||||
candidates: list[tuple[str, str, Path]] = []
|
||||
for recipes_root in (RECIPES_ROOT, MAINLINE_RECIPES):
|
||||
if not recipes_root.is_dir():
|
||||
continue
|
||||
for recipe_toml in recipes_root.rglob("recipe.toml"):
|
||||
if "source" in recipe_toml.parts or "target" in recipe_toml.parts:
|
||||
continue
|
||||
if recipe_toml.parent.name != component:
|
||||
continue
|
||||
try:
|
||||
with open(recipe_toml, "rb") as f:
|
||||
data = tomllib.load(f)
|
||||
except (OSError, tomllib.TOMLDecodeError):
|
||||
continue
|
||||
source = data.get("source") or {}
|
||||
if "git" in source:
|
||||
# Either pinned rev or branch tip — both are valid
|
||||
# upstream reference points for a patch's "from" state.
|
||||
if "rev" in source:
|
||||
rev = str(source["rev"])
|
||||
elif "branch" in source:
|
||||
# Branch resolution requires a network call to
|
||||
# the upstream's `git ls-remote`. Patches that
|
||||
# track a branch should ideally pin a rev for
|
||||
# reproducibility; warn but proceed.
|
||||
rev = f"refs/heads/{source['branch']}"
|
||||
else:
|
||||
continue
|
||||
candidates.append((source["git"], rev, recipe_toml))
|
||||
elif "tar" in source:
|
||||
return ("tar", source.get("tar"), recipe_toml)
|
||||
if not candidates:
|
||||
return None, None
|
||||
if len(candidates) > 1:
|
||||
candidates.sort(key=lambda c: "local" in str(c[2]))
|
||||
url, rev, _ = candidates[0]
|
||||
return url, rev
|
||||
|
||||
|
||||
def clone_source(url, rev, target):
|
||||
"""Clone the upstream repo at the pinned rev into target/."""
|
||||
if target.exists():
|
||||
shutil.rmtree(target)
|
||||
target.mkdir(parents=True)
|
||||
rc, out, err = run(
|
||||
["git", "clone", "--quiet", "--no-checkout", url, str(target)],
|
||||
)
|
||||
if rc != 0:
|
||||
return False, f"clone failed: {err.strip()}"
|
||||
rc, out, err = run(
|
||||
["git", "-C", str(target), "checkout", "--quiet", rev],
|
||||
)
|
||||
if rc != 0:
|
||||
return False, f"checkout {rev} failed: {err.strip()}"
|
||||
return True, None
|
||||
|
||||
|
||||
def apply_patch(source_dir, patch_path):
|
||||
"""Apply patch in source_dir. Return (ok, error_msg)."""
|
||||
rc, out, err = run(
|
||||
["git", "-C", str(source_dir), "apply", "--whitespace=nowarn", str(patch_path)],
|
||||
)
|
||||
if rc != 0:
|
||||
return False, (err or out).strip()
|
||||
return True, None
|
||||
|
||||
|
||||
def check_reverse(source_dir, patch_path):
|
||||
"""git apply --reverse --check. Returns (ok, error_msg)."""
|
||||
rc, out, err = run(
|
||||
["git", "-C", str(source_dir), "apply", "--reverse", "--check", str(patch_path)],
|
||||
)
|
||||
if rc != 0:
|
||||
return False, (err or out).strip()
|
||||
return True, None
|
||||
|
||||
|
||||
def diff_trees(a, b):
|
||||
"""Return a unified diff between two source dirs, excluding .git/.
|
||||
|
||||
The .git/ directory has timestamps and refs that always differ
|
||||
between clones, so we exclude it. The actual source tree is the
|
||||
signal we care about.
|
||||
"""
|
||||
proc = subprocess.run(
|
||||
["diff", "-ruN",
|
||||
"--exclude=.git",
|
||||
"--exclude=*.pyc", "--exclude=__pycache__",
|
||||
str(a), str(b)],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def audit_one(component, patch_path, verbose=False):
|
||||
"""Audit a single patch. Return a list of error strings (empty = OK)."""
|
||||
errors: list[str] = []
|
||||
upstream = resolve_upstream(component)
|
||||
if isinstance(upstream, tuple) and len(upstream) == 3 and upstream[0] == "tar":
|
||||
return [f"{component}/{patch_path.name}: tar-based source, "
|
||||
f"manual audit required"]
|
||||
if not upstream or upstream[0] is None:
|
||||
return [f"{component}/{patch_path.name}: no upstream recipe found "
|
||||
f"in local/recipes/ or recipes/"]
|
||||
url, rev = upstream[0], upstream[1]
|
||||
if url is None or rev is None:
|
||||
return [f"{component}/{patch_path.name}: could not resolve upstream "
|
||||
f"git URL or rev for component {component!r}"]
|
||||
url = str(url)
|
||||
rev = str(rev)
|
||||
|
||||
# Phase 1: clone, apply, verify reverse + idempotency
|
||||
with tempfile.TemporaryDirectory(prefix="audit-patch-") as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
work = tmp_path / "work"
|
||||
work2 = tmp_path / "work2"
|
||||
|
||||
if verbose:
|
||||
print(f" cloning {url} @ {rev[:12]}...")
|
||||
ok, err = clone_source(url, rev, work)
|
||||
if not ok:
|
||||
return [f"{component}/{patch_path.name}: clone failed: {err}"]
|
||||
# Apply once
|
||||
ok, err = apply_patch(work, patch_path)
|
||||
if not err:
|
||||
patch_applied_ok = True
|
||||
else:
|
||||
patch_applied_ok = False
|
||||
errors.append(f"{component}/{patch_path.name}: apply failed: {err}")
|
||||
|
||||
if patch_applied_ok:
|
||||
# Reverse check (idempotency invariant)
|
||||
ok, rev_err = check_reverse(work, patch_path)
|
||||
if not ok:
|
||||
err_msg = rev_err or "unknown error"
|
||||
errors.append(
|
||||
f"{component}/{patch_path.name}: --reverse --check FAILED — "
|
||||
f"patch is not idempotent. Cookbook's cookbook_apply_patches "
|
||||
f"will fail on a re-cook. Underlying error: {err_msg[:500]}"
|
||||
)
|
||||
# Idempotency: apply twice = apply once
|
||||
ok, err = apply_patch(work, patch_path)
|
||||
if not err:
|
||||
# The patch is now applied twice (or rather, applied when
|
||||
# already applied, which might fail). The cookbook's
|
||||
# --reverse --check is meant to skip this case. If the
|
||||
# second apply succeeded, the patch is non-idempotent
|
||||
# (applying twice is meaningful). If it failed, check
|
||||
# that the second failure is the expected "already
|
||||
# applied" error.
|
||||
errors.append(
|
||||
f"{component}/{patch_path.name}: second apply SUCCEEDED — "
|
||||
f"patch is not idempotent. Re-applying after a fresh "
|
||||
f"cook will apply it twice. Cookbook should skip via "
|
||||
f"--reverse --check; verify the helper still works."
|
||||
)
|
||||
else:
|
||||
# Expected: second apply fails. Confirm the working tree
|
||||
# is byte-identical to the first apply.
|
||||
if verbose:
|
||||
print(f" re-cloning to verify reproducibility...")
|
||||
ok, err = clone_source(url, rev, work2)
|
||||
if not ok:
|
||||
errors.append(
|
||||
f"{component}/{patch_path.name}: re-clone failed: {err}"
|
||||
)
|
||||
else:
|
||||
ok, err = apply_patch(work2, patch_path)
|
||||
if err:
|
||||
errors.append(
|
||||
f"{component}/{patch_path.name}: "
|
||||
f"reproducibility — second apply failed: {err}"
|
||||
)
|
||||
else:
|
||||
diff_out = diff_trees(work, work2)
|
||||
if diff_out:
|
||||
errors.append(
|
||||
f"{component}/{patch_path.name}: non-reproducible — "
|
||||
f"second apply produces a different tree:\n"
|
||||
f"{diff_out[:1000]}"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Validate the idempotency of every external patch in "
|
||||
"local/patches/."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--component",
|
||||
help="Audit only the given component (default: all)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose", "-v", action="store_true",
|
||||
help="Print progress as patches are checked",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-fetch", action="store_true",
|
||||
help="Skip fetching upstream (useful when network is unavailable)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
patches = list(collect_patches(args.component))
|
||||
if not patches:
|
||||
print(f"No patches found{' for component ' + args.component if args.component else ''}.",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
|
||||
print(f"Auditing {len(patches)} patch(es)...")
|
||||
|
||||
all_errors = []
|
||||
for component, patch_path in patches:
|
||||
if args.verbose:
|
||||
print(f"[{component}/{patch_path.name}]")
|
||||
if args.no_fetch:
|
||||
print(f" {component}/{patch_path.name}: SKIPPED (--no-fetch)")
|
||||
continue
|
||||
errors = audit_one(component, patch_path, verbose=args.verbose)
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f" FAIL: {e}")
|
||||
all_errors.extend(errors)
|
||||
elif args.verbose:
|
||||
print(f" OK")
|
||||
|
||||
if all_errors:
|
||||
print()
|
||||
print(f"FAILED: {len(all_errors)} error(s) across {len(patches)} patch(es).")
|
||||
print()
|
||||
print("Common fixes:")
|
||||
print(" 1. Patch hunks reference content that no longer exists in")
|
||||
print(" the upstream source. Re-generate the patch from a fresh")
|
||||
print(" checkout: git diff > local/patches/<component>/NN-...patch")
|
||||
print(" 2. Patch is order-dependent with a sibling. The cookbook")
|
||||
print(" applies them in lexical order — make sure NN-prefix order")
|
||||
print(" matches the actual dependency order.")
|
||||
print(" 3. Patch has whitespace conflicts with the upstream source.")
|
||||
print(" Try regenerating with `git diff --ignore-all-space`.")
|
||||
return 1
|
||||
print(f"All {len(patches)} patch(es) are idempotent and reproducible.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -198,6 +198,32 @@ if [ -n "${COOKBOOK_HOST_SYSROOT}" ] && [ "${COOKBOOK_HOST_SYSROOT}" != "/usr" ]
|
||||
fi
|
||||
export LDFLAGS="${USER_LDFLAGS}-L${COOKBOOK_SYSROOT}/lib --static${REDBEAR_LIBATOMIC_LDFLAGS:+ ${REDBEAR_LIBATOMIC_LDFLAGS}}"
|
||||
|
||||
# Auto-link Qt sysroot dirs for any recipe that depends on qtbase or
|
||||
# qtdeclarative. KF6 recipes need /usr/{plugins,mkspecs,metatypes,modules}
|
||||
# to be visible at the sysroot root (not under /usr/) for cmake to find
|
||||
# Qt6Config.cmake's INTERFACE_INCLUDE_DIRECTORIES etc. Without this,
|
||||
# the build fails with "Could NOT find Qt6 (missing: Qt6_DIR)" or
|
||||
# "Qt6::Qml includes non-existent path". See local/scripts/lib/qt-sysroot.sh
|
||||
# for the full helper surface.
|
||||
#
|
||||
# We auto-detect the Qt deps by checking if either qtbase or qtdeclarative
|
||||
# has been pushed to the per-recipe sysroot. If so, link the four canonical
|
||||
# dirs. Idempotent (skips dirs that already exist as symlinks).
|
||||
#
|
||||
# Recipes can override or extend this by calling redbear_qt_link_sysroot_dirs
|
||||
# directly (it remains available via `source` of qt-sysroot.sh).
|
||||
if [ -d "${COOKBOOK_SYSROOT}/usr/lib/cmake/Qt6Core" ] || [ -d "${COOKBOOK_SYSROOT}/usr/lib/cmake/Qt6" ]; then
|
||||
if [ -d "${COOKBOOK_SYSROOT}/usr/plugins" ] && [ ! -e "${COOKBOOK_SYSROOT}/plugins" ]; then
|
||||
ln -s usr/plugins "${COOKBOOK_SYSROOT}/plugins"
|
||||
fi
|
||||
for _rb_qt_dir in plugins mkspecs metatypes modules; do
|
||||
if [ -d "${COOKBOOK_SYSROOT}/usr/${_rb_qt_dir}" ] && [ ! -e "${COOKBOOK_SYSROOT}/${_rb_qt_dir}" ]; then
|
||||
ln -s "usr/${_rb_qt_dir}" "${COOKBOOK_SYSROOT}/${_rb_qt_dir}"
|
||||
fi
|
||||
done
|
||||
unset _rb_qt_dir
|
||||
fi
|
||||
|
||||
# This reexport C variables into custom build script that can be consumed by cc crate
|
||||
function reexport_flags {
|
||||
target=${TARGET//-/_}
|
||||
|
||||
Reference in New Issue
Block a user