build: add audit-patch-idempotency.py + auto-link Qt sysroot dirs
Two S-sized improvements from BUILD-SYSTEM-IMPROVEMENTS.md: 1. local/scripts/audit-patch-idempotency.py (improvement #3): Validates that every external patch in local/patches/ is idempotent (--reverse --check succeeds) and reproducible (re-clone + re-apply produces an identical tree). Catches the patch idempotency class of bug at lint time, where it used to surface as a 2+ hour cookbook failure during a cook. Found a real bug on first run: local/patches/libdrm/02-redox-dispatch.patch has a hunk at xf86drm.c:321 that no longer matches the upstream libdrm-2.4.125. 2. src/cook/script.rs auto-link Qt sysroot dirs (improvement #8): The cookbook's BUILD_PRESCRIPT now auto-detects if the per-recipe sysroot has Qt6 (qtbase or qtdeclarative) and creates the canonical /usr/{plugins,mkspecs,metatypes,modules} symlinks that KF6 recipes need for cmake to find Qt6Config.cmake's INTERFACE_* paths. New KF6 recipes that depend on qtbase no longer need to manually call redbear_qt_link_sysroot_dirs in their build script. Recipes that need more customization can still call the helper directly via 'source $COOKBOOK_ROOT/local/scripts/lib/qt-sysroot.sh'.
This commit is contained in:
Executable
+331
@@ -0,0 +1,331 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Validate the idempotency of every external patch in local/patches/.
|
||||||
|
|
||||||
|
Per AGENTS.md "NO OVERLAY-STYLE PATCHES — AMENDED 2026" Rule 2, big
|
||||||
|
external projects use the cookbook's `cookbook_apply_patches` helper
|
||||||
|
which checks `git apply --reverse --check` to skip already-applied
|
||||||
|
patches. If a patch's reverse check fails (because the upstream
|
||||||
|
source drifted from the patch's expected state), the helper tries to
|
||||||
|
apply the patch forward, which fails too because some hunks no
|
||||||
|
longer apply. The result is a confusing cook failure.
|
||||||
|
|
||||||
|
This script catches that class of bug at lint time. For every
|
||||||
|
[0-9]*.patch under local/patches/<component>/, it:
|
||||||
|
|
||||||
|
1. Clones the upstream repo at the pinned rev into a temp dir
|
||||||
|
2. Applies the patch
|
||||||
|
3. Verifies `git apply --reverse --check` succeeds on the result
|
||||||
|
(i.e. the patch is fully reversible — idempotency invariant)
|
||||||
|
4. Re-applies the patch
|
||||||
|
5. Verifies the source is byte-identical to step 2's result
|
||||||
|
(i.e. the patch is idempotent — applying it twice = applying it once)
|
||||||
|
6. Verifies the result is reproducible: re-clone, re-apply, byte-equal
|
||||||
|
|
||||||
|
If any check fails, the script exits non-zero and prints which patches
|
||||||
|
are non-idempotent. CI or `make lint` should run this on every PR.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
./local/scripts/audit-patch-idempotency.py [--component <name>] [--verbose]
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import tomllib
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
PATCHES_ROOT = PROJECT_ROOT / "local" / "patches"
|
||||||
|
SOURCE_ROOT = PROJECT_ROOT / "local" / "sources"
|
||||||
|
RECIPES_ROOT = PROJECT_ROOT / "local" / "recipes"
|
||||||
|
MAINLINE_RECIPES = PROJECT_ROOT / "recipes"
|
||||||
|
|
||||||
|
PATCH_NAME_RE = re.compile(r"^\d+-[A-Za-z0-9_.-]+\.patch$")
|
||||||
|
NUM_PREFIX_RE = re.compile(r"^(\d+)-")
|
||||||
|
|
||||||
|
|
||||||
|
def run(cmd, **kwargs):
|
||||||
|
"""Run a subprocess, returning (returncode, stdout, stderr)."""
|
||||||
|
proc = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
return proc.returncode, proc.stdout, proc.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def collect_patches(component_filter=None):
|
||||||
|
"""Yield (component, patch_path) for every external patch."""
|
||||||
|
if not PATCHES_ROOT.is_dir():
|
||||||
|
return
|
||||||
|
for component_dir in sorted(PATCHES_ROOT.iterdir()):
|
||||||
|
if not component_dir.is_dir():
|
||||||
|
continue
|
||||||
|
if component_filter and component_dir.name != component_filter:
|
||||||
|
continue
|
||||||
|
for patch_path in sorted(component_dir.iterdir()):
|
||||||
|
if patch_path.is_file() and PATCH_NAME_RE.match(patch_path.name):
|
||||||
|
yield component_dir.name, patch_path
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_upstream(component) -> "tuple[str | None, str | None] | tuple[str, str | None, Path]":
|
||||||
|
"""Return (url, rev) for a component by reading its mainline recipe.
|
||||||
|
|
||||||
|
The component is matched by the recipe.toml's parent directory name
|
||||||
|
(e.g. recipes/libs/mesa/recipe.toml matches component="mesa"),
|
||||||
|
not the category. This means multiple categories with the same
|
||||||
|
package name (e.g. recipes/wip/demos/mesa-demos) won't accidentally
|
||||||
|
match.
|
||||||
|
"""
|
||||||
|
candidates: list[tuple[str, str, Path]] = []
|
||||||
|
for recipes_root in (RECIPES_ROOT, MAINLINE_RECIPES):
|
||||||
|
if not recipes_root.is_dir():
|
||||||
|
continue
|
||||||
|
for recipe_toml in recipes_root.rglob("recipe.toml"):
|
||||||
|
if "source" in recipe_toml.parts or "target" in recipe_toml.parts:
|
||||||
|
continue
|
||||||
|
if recipe_toml.parent.name != component:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
with open(recipe_toml, "rb") as f:
|
||||||
|
data = tomllib.load(f)
|
||||||
|
except (OSError, tomllib.TOMLDecodeError):
|
||||||
|
continue
|
||||||
|
source = data.get("source") or {}
|
||||||
|
if "git" in source:
|
||||||
|
# Either pinned rev or branch tip — both are valid
|
||||||
|
# upstream reference points for a patch's "from" state.
|
||||||
|
if "rev" in source:
|
||||||
|
rev = str(source["rev"])
|
||||||
|
elif "branch" in source:
|
||||||
|
# Branch resolution requires a network call to
|
||||||
|
# the upstream's `git ls-remote`. Patches that
|
||||||
|
# track a branch should ideally pin a rev for
|
||||||
|
# reproducibility; warn but proceed.
|
||||||
|
rev = f"refs/heads/{source['branch']}"
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
candidates.append((source["git"], rev, recipe_toml))
|
||||||
|
elif "tar" in source:
|
||||||
|
return ("tar", source.get("tar"), recipe_toml)
|
||||||
|
if not candidates:
|
||||||
|
return None, None
|
||||||
|
if len(candidates) > 1:
|
||||||
|
candidates.sort(key=lambda c: "local" in str(c[2]))
|
||||||
|
url, rev, _ = candidates[0]
|
||||||
|
return url, rev
|
||||||
|
|
||||||
|
|
||||||
|
def clone_source(url, rev, target):
|
||||||
|
"""Clone the upstream repo at the pinned rev into target/."""
|
||||||
|
if target.exists():
|
||||||
|
shutil.rmtree(target)
|
||||||
|
target.mkdir(parents=True)
|
||||||
|
rc, out, err = run(
|
||||||
|
["git", "clone", "--quiet", "--no-checkout", url, str(target)],
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
return False, f"clone failed: {err.strip()}"
|
||||||
|
rc, out, err = run(
|
||||||
|
["git", "-C", str(target), "checkout", "--quiet", rev],
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
return False, f"checkout {rev} failed: {err.strip()}"
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
|
||||||
|
def apply_patch(source_dir, patch_path):
|
||||||
|
"""Apply patch in source_dir. Return (ok, error_msg)."""
|
||||||
|
rc, out, err = run(
|
||||||
|
["git", "-C", str(source_dir), "apply", "--whitespace=nowarn", str(patch_path)],
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
return False, (err or out).strip()
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
|
||||||
|
def check_reverse(source_dir, patch_path):
|
||||||
|
"""git apply --reverse --check. Returns (ok, error_msg)."""
|
||||||
|
rc, out, err = run(
|
||||||
|
["git", "-C", str(source_dir), "apply", "--reverse", "--check", str(patch_path)],
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
return False, (err or out).strip()
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
|
||||||
|
def diff_trees(a, b):
|
||||||
|
"""Return a unified diff between two source dirs, excluding .git/.
|
||||||
|
|
||||||
|
The .git/ directory has timestamps and refs that always differ
|
||||||
|
between clones, so we exclude it. The actual source tree is the
|
||||||
|
signal we care about.
|
||||||
|
"""
|
||||||
|
proc = subprocess.run(
|
||||||
|
["diff", "-ruN",
|
||||||
|
"--exclude=.git",
|
||||||
|
"--exclude=*.pyc", "--exclude=__pycache__",
|
||||||
|
str(a), str(b)],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
return proc.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def audit_one(component, patch_path, verbose=False):
|
||||||
|
"""Audit a single patch. Return a list of error strings (empty = OK)."""
|
||||||
|
errors: list[str] = []
|
||||||
|
upstream = resolve_upstream(component)
|
||||||
|
if isinstance(upstream, tuple) and len(upstream) == 3 and upstream[0] == "tar":
|
||||||
|
return [f"{component}/{patch_path.name}: tar-based source, "
|
||||||
|
f"manual audit required"]
|
||||||
|
if not upstream or upstream[0] is None:
|
||||||
|
return [f"{component}/{patch_path.name}: no upstream recipe found "
|
||||||
|
f"in local/recipes/ or recipes/"]
|
||||||
|
url, rev = upstream[0], upstream[1]
|
||||||
|
if url is None or rev is None:
|
||||||
|
return [f"{component}/{patch_path.name}: could not resolve upstream "
|
||||||
|
f"git URL or rev for component {component!r}"]
|
||||||
|
url = str(url)
|
||||||
|
rev = str(rev)
|
||||||
|
|
||||||
|
# Phase 1: clone, apply, verify reverse + idempotency
|
||||||
|
with tempfile.TemporaryDirectory(prefix="audit-patch-") as tmp:
|
||||||
|
tmp_path = Path(tmp)
|
||||||
|
work = tmp_path / "work"
|
||||||
|
work2 = tmp_path / "work2"
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print(f" cloning {url} @ {rev[:12]}...")
|
||||||
|
ok, err = clone_source(url, rev, work)
|
||||||
|
if not ok:
|
||||||
|
return [f"{component}/{patch_path.name}: clone failed: {err}"]
|
||||||
|
# Apply once
|
||||||
|
ok, err = apply_patch(work, patch_path)
|
||||||
|
if not err:
|
||||||
|
patch_applied_ok = True
|
||||||
|
else:
|
||||||
|
patch_applied_ok = False
|
||||||
|
errors.append(f"{component}/{patch_path.name}: apply failed: {err}")
|
||||||
|
|
||||||
|
if patch_applied_ok:
|
||||||
|
# Reverse check (idempotency invariant)
|
||||||
|
ok, rev_err = check_reverse(work, patch_path)
|
||||||
|
if not ok:
|
||||||
|
err_msg = rev_err or "unknown error"
|
||||||
|
errors.append(
|
||||||
|
f"{component}/{patch_path.name}: --reverse --check FAILED — "
|
||||||
|
f"patch is not idempotent. Cookbook's cookbook_apply_patches "
|
||||||
|
f"will fail on a re-cook. Underlying error: {err_msg[:500]}"
|
||||||
|
)
|
||||||
|
# Idempotency: apply twice = apply once
|
||||||
|
ok, err = apply_patch(work, patch_path)
|
||||||
|
if not err:
|
||||||
|
# The patch is now applied twice (or rather, applied when
|
||||||
|
# already applied, which might fail). The cookbook's
|
||||||
|
# --reverse --check is meant to skip this case. If the
|
||||||
|
# second apply succeeded, the patch is non-idempotent
|
||||||
|
# (applying twice is meaningful). If it failed, check
|
||||||
|
# that the second failure is the expected "already
|
||||||
|
# applied" error.
|
||||||
|
errors.append(
|
||||||
|
f"{component}/{patch_path.name}: second apply SUCCEEDED — "
|
||||||
|
f"patch is not idempotent. Re-applying after a fresh "
|
||||||
|
f"cook will apply it twice. Cookbook should skip via "
|
||||||
|
f"--reverse --check; verify the helper still works."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Expected: second apply fails. Confirm the working tree
|
||||||
|
# is byte-identical to the first apply.
|
||||||
|
if verbose:
|
||||||
|
print(f" re-cloning to verify reproducibility...")
|
||||||
|
ok, err = clone_source(url, rev, work2)
|
||||||
|
if not ok:
|
||||||
|
errors.append(
|
||||||
|
f"{component}/{patch_path.name}: re-clone failed: {err}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
ok, err = apply_patch(work2, patch_path)
|
||||||
|
if err:
|
||||||
|
errors.append(
|
||||||
|
f"{component}/{patch_path.name}: "
|
||||||
|
f"reproducibility — second apply failed: {err}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
diff_out = diff_trees(work, work2)
|
||||||
|
if diff_out:
|
||||||
|
errors.append(
|
||||||
|
f"{component}/{patch_path.name}: non-reproducible — "
|
||||||
|
f"second apply produces a different tree:\n"
|
||||||
|
f"{diff_out[:1000]}"
|
||||||
|
)
|
||||||
|
return errors
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description=(
|
||||||
|
"Validate the idempotency of every external patch in "
|
||||||
|
"local/patches/."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--component",
|
||||||
|
help="Audit only the given component (default: all)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--verbose", "-v", action="store_true",
|
||||||
|
help="Print progress as patches are checked",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-fetch", action="store_true",
|
||||||
|
help="Skip fetching upstream (useful when network is unavailable)",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
patches = list(collect_patches(args.component))
|
||||||
|
if not patches:
|
||||||
|
print(f"No patches found{' for component ' + args.component if args.component else ''}.",
|
||||||
|
file=sys.stderr)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
print(f"Auditing {len(patches)} patch(es)...")
|
||||||
|
|
||||||
|
all_errors = []
|
||||||
|
for component, patch_path in patches:
|
||||||
|
if args.verbose:
|
||||||
|
print(f"[{component}/{patch_path.name}]")
|
||||||
|
if args.no_fetch:
|
||||||
|
print(f" {component}/{patch_path.name}: SKIPPED (--no-fetch)")
|
||||||
|
continue
|
||||||
|
errors = audit_one(component, patch_path, verbose=args.verbose)
|
||||||
|
if errors:
|
||||||
|
for e in errors:
|
||||||
|
print(f" FAIL: {e}")
|
||||||
|
all_errors.extend(errors)
|
||||||
|
elif args.verbose:
|
||||||
|
print(f" OK")
|
||||||
|
|
||||||
|
if all_errors:
|
||||||
|
print()
|
||||||
|
print(f"FAILED: {len(all_errors)} error(s) across {len(patches)} patch(es).")
|
||||||
|
print()
|
||||||
|
print("Common fixes:")
|
||||||
|
print(" 1. Patch hunks reference content that no longer exists in")
|
||||||
|
print(" the upstream source. Re-generate the patch from a fresh")
|
||||||
|
print(" checkout: git diff > local/patches/<component>/NN-...patch")
|
||||||
|
print(" 2. Patch is order-dependent with a sibling. The cookbook")
|
||||||
|
print(" applies them in lexical order — make sure NN-prefix order")
|
||||||
|
print(" matches the actual dependency order.")
|
||||||
|
print(" 3. Patch has whitespace conflicts with the upstream source.")
|
||||||
|
print(" Try regenerating with `git diff --ignore-all-space`.")
|
||||||
|
return 1
|
||||||
|
print(f"All {len(patches)} patch(es) are idempotent and reproducible.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -198,6 +198,32 @@ if [ -n "${COOKBOOK_HOST_SYSROOT}" ] && [ "${COOKBOOK_HOST_SYSROOT}" != "/usr" ]
|
|||||||
fi
|
fi
|
||||||
export LDFLAGS="${USER_LDFLAGS}-L${COOKBOOK_SYSROOT}/lib --static${REDBEAR_LIBATOMIC_LDFLAGS:+ ${REDBEAR_LIBATOMIC_LDFLAGS}}"
|
export LDFLAGS="${USER_LDFLAGS}-L${COOKBOOK_SYSROOT}/lib --static${REDBEAR_LIBATOMIC_LDFLAGS:+ ${REDBEAR_LIBATOMIC_LDFLAGS}}"
|
||||||
|
|
||||||
|
# Auto-link Qt sysroot dirs for any recipe that depends on qtbase or
|
||||||
|
# qtdeclarative. KF6 recipes need /usr/{plugins,mkspecs,metatypes,modules}
|
||||||
|
# to be visible at the sysroot root (not under /usr/) for cmake to find
|
||||||
|
# Qt6Config.cmake's INTERFACE_INCLUDE_DIRECTORIES etc. Without this,
|
||||||
|
# the build fails with "Could NOT find Qt6 (missing: Qt6_DIR)" or
|
||||||
|
# "Qt6::Qml includes non-existent path". See local/scripts/lib/qt-sysroot.sh
|
||||||
|
# for the full helper surface.
|
||||||
|
#
|
||||||
|
# We auto-detect the Qt deps by checking if either qtbase or qtdeclarative
|
||||||
|
# has been pushed to the per-recipe sysroot. If so, link the four canonical
|
||||||
|
# dirs. Idempotent (skips dirs that already exist as symlinks).
|
||||||
|
#
|
||||||
|
# Recipes can override or extend this by calling redbear_qt_link_sysroot_dirs
|
||||||
|
# directly (it remains available via `source` of qt-sysroot.sh).
|
||||||
|
if [ -d "${COOKBOOK_SYSROOT}/usr/lib/cmake/Qt6Core" ] || [ -d "${COOKBOOK_SYSROOT}/usr/lib/cmake/Qt6" ]; then
|
||||||
|
if [ -d "${COOKBOOK_SYSROOT}/usr/plugins" ] && [ ! -e "${COOKBOOK_SYSROOT}/plugins" ]; then
|
||||||
|
ln -s usr/plugins "${COOKBOOK_SYSROOT}/plugins"
|
||||||
|
fi
|
||||||
|
for _rb_qt_dir in plugins mkspecs metatypes modules; do
|
||||||
|
if [ -d "${COOKBOOK_SYSROOT}/usr/${_rb_qt_dir}" ] && [ ! -e "${COOKBOOK_SYSROOT}/${_rb_qt_dir}" ]; then
|
||||||
|
ln -s "usr/${_rb_qt_dir}" "${COOKBOOK_SYSROOT}/${_rb_qt_dir}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
unset _rb_qt_dir
|
||||||
|
fi
|
||||||
|
|
||||||
# This reexport C variables into custom build script that can be consumed by cc crate
|
# This reexport C variables into custom build script that can be consumed by cc crate
|
||||||
function reexport_flags {
|
function reexport_flags {
|
||||||
target=${TARGET//-/_}
|
target=${TARGET//-/_}
|
||||||
|
|||||||
Reference in New Issue
Block a user