#!/usr/bin/env bash # sync-recipe-source.sh — canonical version-propagation engine for tar recipes. # # THE MISSING STEP. Bumping a recipe's `tar=`/`blake3=` never propagated into the # vendored `source/` tree, so the build silently compiled a stale version (Qt # 6.11.0-vs-6.11.1, the whole KDE stack at 6.10.0 vs recipe 6.28.0). This engine # makes `source.tar`, the vendored `source/` tree, its version stamp, and the # recipe's declared version all agree — treating recipe.toml as the source of # truth (the version to converge ON). # # For each recipe it rebases the vendored fork onto the declared upstream: # source/(Vnew) = pristine(Vnew) + recipe patches + baked local delta # where the "baked local delta" is any change in the current source/ that is NOT # produced by a listed recipe patch (e.g. Redox shims committed directly). That # delta is captured against pristine(Vold) and re-applied onto pristine(Vnew) so # a re-lay can never silently drop a Redox port change. Injected line-duplication # corruption (identical consecutive lines) is dropped. # # Usage: # sync-recipe-source.sh [--check] [--commit] [--all|--config=NAME] ... # --check dry-run: classify each recipe, apply nothing, write nothing # --commit git commit each synced recipe individually # --all every tar recipe under local/recipes (excl. wip) # a recipe dir or bare name (e.g. kde/kf6-kio or kf6-kio) # # Exit: 0 all synced/consistent; 2 some need manual rebase (patches/shims reject). set -uo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" CHECK=0; COMMIT=0; ALL=0; declare -a WANT=() for a in "$@"; do case "$a" in --check) CHECK=1;; --commit) COMMIT=1;; --all) ALL=1;; --config=*) : ;; # reserved --*) echo "unknown opt: $a" >&2; exit 1;; *) WANT+=("$a");; esac; done WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT ver_of(){ grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)*' <<<"$1" | tail -1; } # resolve a recipe token to its dir resolve(){ local t="$1" [ -f "$ROOT/local/recipes/$t/recipe.toml" ] && { echo "$ROOT/local/recipes/$t"; return; } local hit; hit=$(find "$ROOT/local/recipes" -name recipe.toml -not -path '*/wip/*' -path "*/$t/recipe.toml" 2>/dev/null | head -1) [ -n "$hit" ] && echo "$(dirname "$hit")" } # vendored source/ version marker (KDE KF_VERSION, Qt .cmake.conf, generic stamp, autotools) src_version(){ local s="$1" [ -f "$s/.redbear-src-version" ] && { ver_of "$(cat "$s/.redbear-src-version")"; return; } [ -f "$s/.cmake.conf" ] && { local v; v=$(ver_of "$(grep -m1 QT_REPO_MODULE_VERSION "$s/.cmake.conf")"); [ -n "$v" ] && { echo "$v"; return; }; } [ -f "$s/CMakeLists.txt" ] && { local v; v=$(grep -m1 -E 'set\(KF_VERSION|project\([A-Za-z0-9_]+ VERSION [0-9]' "$s/CMakeLists.txt" 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); [ -n "$v" ] && { echo "$v"; return; }; } echo "" } # apply a patch trying -p1 then -p0; $1=tree-root $2=patchfile ; echo level or FAIL. # --fuzz=0: require EXACT context (line offsets are still allowed) so a patch # whose context no longer matches the new upstream is REJECTED, not silently # force-fitted with fuzz — that silent force-fit is how a stale/wrong port would # sneak through. apply_patch(){ local root="$1" pf="$2" lvl for lvl in 1 0; do if ( cd "$root" && patch -p$lvl --fuzz=0 --dry-run -f <"$pf" >/dev/null 2>&1 ); then ( cd "$root" && patch -p$lvl --fuzz=0 -s -f <"$pf" >/dev/null 2>&1 ); echo "$lvl"; return 0 fi done echo FAIL; return 1 } # Drop the line-duplication corruption pattern: a run of >=4 identical # consecutive NON-BLANK lines collapses to a single line (the observed # corruption injected 5-13 copies). Runs of <=3 and blank-line runs are kept # verbatim, so legitimate paired lines (`}`/`}`, `#endif`/`#endif`) survive. decorrupt(){ awk ' { a[NR]=$0 } END { i=1 while (i<=NR) { j=i while (j= 4 && a[i]!="") print a[i] else for (k=i;k<=j;k++) print a[k] i=j+1 } }' "$1"; } download(){ curl -sLf --retry 3 --connect-timeout 20 --max-time 600 "$1" -o "$2" 2>/dev/null; } # Reconcile recipe blake3 to the authoritative source.tar. A vendored bump often # left recipe blake3 stale (URL bumped, hash not), so recipe.toml's declared hash # no longer matched its own tarball. Only reconciles when source.tar exists AND # its top-dir version equals the recipe URL version (so we never stamp the hash # of a wrong-version tarball). No-op under --check. reconcile_blake3(){ # $1=dir $2=recipe.toml $3=vnew [ "$CHECK" = 1 ] && return 0 command -v b3sum >/dev/null 2>&1 || return 0 [ -f "$1/source.tar" ] || return 0 local tvv; tvv="$(ver_of "$(tar tf "$1/source.tar" 2>/dev/null | head -1)")" [ "$tvv" = "$3" ] || return 0 grep -qE '^[[:space:]]*blake3[[:space:]]*=' "$2" || return 0 local h; h="$(b3sum "$1/source.tar" | awk '{print $1}')" sed -i -E "s|^([[:space:]]*blake3[[:space:]]*=[[:space:]]*)\"[0-9a-f]*\"|\1\"$h\"|" "$2" } # build the recipe list declare -a RECIPES=() if [ "$ALL" = 1 ]; then while IFS= read -r r; do grep -qE '^[[:space:]]*tar[[:space:]]*=' "$r" && RECIPES+=("$(dirname "$r")"); done \ < <(find "$ROOT/local/recipes" -name recipe.toml -not -path '*/wip/*') else for t in "${WANT[@]}"; do d=$(resolve "$t"); [ -n "$d" ] && RECIPES+=("$d") || echo "!! not found: $t" >&2; done fi printf '%-26s %-9s %-9s %-8s %-6s %s\n' RECIPE FROM TO PATCHES BAKED STATUS RC=0 for dir in "${RECIPES[@]}"; do rc="$dir/recipe.toml"; name="$(basename "$dir")" url="$(grep -m1 -E '^[[:space:]]*tar[[:space:]]*=' "$rc" | sed -E 's/^[^"]*"([^"]+)".*/\1/')"; [ -n "$url" ] || continue vnew="$(ver_of "$(basename "$url")")"; [ -n "$vnew" ] || continue blake="$(grep -m1 -E '^[[:space:]]*blake3[[:space:]]*=' "$rc" | grep -oE '[0-9a-f]{64}')" if [ ! -d "$dir/source" ]; then # Transient recipe: cookbook builds from source.tar/URL, so a stale cached # source.tar is a stale build. Ensure it matches the declared version. tst=""; [ -f "$dir/source.tar" ] && tst="$(ver_of "$(tar tf "$dir/source.tar" 2>/dev/null | head -1)")" if [ -z "$tst" ] || [ "$tst" = "$vnew" ]; then # no cached tar (cookbook fetches fresh) or already correct: just reconcile blake3 reconcile_blake3 "$dir" "$rc" "$vnew" printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "${tst:-none}" "$vnew" "-" "-" "transient(ok)" elif [ "$CHECK" = 1 ]; then printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(stale-tar)" elif download "$url" "$WORK/$name-tr.tar"; then cp "$WORK/$name-tr.tar" "$dir/source.tar"; reconcile_blake3 "$dir" "$rc" "$vnew" printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(tar-bumped)" else printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(DL-FAIL)"; RC=2 fi continue fi vold="$(src_version "$dir/source")"; vold="${vold:-unknown}" mapfile -t patches < <(awk '/^[[:space:]]*patches[[:space:]]*=/{p=1} p{print} p&&/\]/{exit}' "$rc" | grep -oE '"[^"]+\.patch"' | tr -d '"') # baked marker files (Redox) not covered by patches baked=$(git -C "$ROOT" grep -lI "__redox__\|Q_OS_REDOX\|defined(__redox" HEAD -- "$dir/source" 2>/dev/null | wc -l | tr -d ' ') if [ "$vold" = "$vnew" ]; then printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "consistent" [ "$CHECK" = 1 ] && continue reconcile_blake3 "$dir" "$rc" "$vnew" printf '%s\n' "$vnew" > "$dir/source/.redbear-src-version"; continue fi # Version marker unreadable: we cannot prove divergence, so we must NOT rebase # (that could clobber a carefully-ported tree like mesa). Report + skip; the # fix is to add a source/.redbear-src-version stamp once the tree is verified. if [ "$vold" = "unknown" ]; then printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "unknown" "$vnew" "${#patches[@]}" "$baked" "no-marker(skip; stamp to enable)" continue fi # --- obtain pristine Vnew --- base="$WORK/$name-new"; rm -rf "$base"; mkdir -p "$base" tarball="$dir/source.tar"; tv="" [ -f "$tarball" ] && tv="$(ver_of "$(tar tf "$tarball" 2>/dev/null | head -1)")" if [ "$tv" != "$vnew" ]; then download "$url" "$WORK/$name-new.tar" && tarball="$WORK/$name-new.tar" || { printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "DL-FAIL"; RC=2; continue; } fi tar xf "$tarball" -C "$base" 2>/dev/null top="$(tar tf "$tarball" 2>/dev/null | head -1 | cut -d/ -f1)" mv "$base/$top" "$base/source" 2>/dev/null || { printf '%-26s %-9s %-9s %s\n' "$name" "$vold" "$vnew" "EXTRACT-FAIL"; RC=2; continue; } # --- apply recipe patches --- preject=0; for p in "${patches[@]}"; do pf="$dir/$p"; [ -f "$pf" ] || pf="$ROOT/local/patches/$p"; [ -f "$pf" ] || { preject=1; continue; } lv=$(apply_patch "$base/source" "$pf"); [ "$lv" = FAIL ] && preject=1 done # --- preserve RedBear additions + reapply baked Redox deltas --- # This runs for EVERY vendored rebase (not only marker-bearing ones): a # RedBear-added file (e.g. a Redox Wayland-crash wrapper .sh, a utmp compat # header) has no __redox__ token, so a grep-only capture misses it and # rsync --delete drops it. We compare against pristine(Vold) by PATH to keep # every RedBear addition, content-agnostic. breject=0 if [ "$vold" != unknown ]; then # Old-version URL: replace the full version, then the major.minor dir # component (KDE mirrors nest under .../6.28/foo-6.28.0 -> 6.10/foo-6.10.0). oldurl="${url//$vnew/$vold}" vnm="${vnew%.*}"; vom="${vold%.*}" [ "$vnm" != "$vnew" ] && [ "$vnm" != "$vom" ] && oldurl="${oldurl//$vnm/$vom}" if download "$oldurl" "$WORK/$name-old.tar"; then op="$WORK/$name-old"; rm -rf "$op"; mkdir -p "$op"; tar xf "$WORK/$name-old.tar" -C "$op" 2>/dev/null otop="$(tar tf "$WORK/$name-old.tar" 2>/dev/null | head -1 | cut -d/ -f1)"; mv "$op/$otop" "$op/source" 2>/dev/null relroot="${dir#$ROOT/}" # repo-relative recipe dir (git needs this) gitf="$WORK/gitf" # (A) Carry forward every committed file absent from pristine(Vold): these # are RedBear additions upstream never had. Skip the stamp and files # the new pristine/patches already provide. while IFS= read -r rel; do [ "$rel" = ".redbear-src-version" ] && continue [ -e "$op/source/$rel" ] && continue # exists upstream -> not a RedBear addition [ -e "$base/source/$rel" ] && continue # already provided by pristine(Vnew) or a patch if ! git -C "$ROOT" show "HEAD:$relroot/source/$rel" > "$gitf" 2>/dev/null; then breject=1; continue; fi decorrupt "$gitf" > "$gitf.clean" mkdir -p "$base/source/$(dirname "$rel")"; cp "$gitf.clean" "$base/source/$rel" done < <(git -C "$ROOT" ls-tree -r --name-only HEAD -- "$relroot/source" | while IFS= read -r p; do printf '%s\n' "${p#"$relroot"/source/}"; done) # (B) For marker-bearing files that ALSO exist in Vold (modified upstream # files), port the Redox delta. Added marker files were handled by (A). if [ "$baked" -gt 0 ]; then while IFS= read -r line; do rel="${line#*/source/}" [ -f "$op/source/$rel" ] || continue # added file -> already carried by (A) if ! git -C "$ROOT" show "HEAD:$relroot/source/$rel" > "$gitf" 2>/dev/null || [ ! -s "$gitf" ]; then breject=1; continue fi decorrupt "$gitf" > "$gitf.clean" if [ ! -f "$base/source/$rel" ]; then breject=1; continue; fi # gone in new upstream -> manual diff -q "$op/source/$rel" "$gitf.clean" >/dev/null 2>&1 && continue # marker is upstream's own # Capture delta to a file (never pipe diff|patch: diff exits 1 on any # difference and pipefail would false-reject every shim). diff -u "$op/source/$rel" "$gitf.clean" > "$WORK/delta.patch" 2>/dev/null if ! patch --fuzz=0 -s "$base/source/$rel" < "$WORK/delta.patch" >/dev/null 2>&1; then breject=1 fi done < <(git -C "$ROOT" grep -lI "__redox__\|Q_OS_REDOX\|defined(__redox" HEAD -- "$dir/source" 2>/dev/null) fi else breject=1; fi fi printf '%s\n' "$vnew" > "$base/source/.redbear-src-version" status="ready"; [ "$preject" = 1 ] && status="PATCH-REJECT(manual)"; [ "$breject" = 1 ] && status="${status};SHIM-REJECT(manual)" [ "$preject" = 1 ] || [ "$breject" = 1 ] && RC=2 printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "$status" if [ "$CHECK" = 0 ] && [ "$preject" = 0 ] && [ "$breject" = 0 ]; then # --checksum: compare by content, never by size+mtime. Two files of equal # size with a coincidental mtime (e.g. "v1\n" vs "v2\n") would otherwise be # skipped by rsync's quick-check, leaving stale content in source/. rsync -a --delete --checksum "$base/source/" "$dir/source/" # keep source.tar authoritative for Vnew, and reconcile recipe blake3 to it [ -f "$WORK/$name-new.tar" ] && cp "$WORK/$name-new.tar" "$dir/source.tar" reconcile_blake3 "$dir" "$rc" "$vnew" if [ "$COMMIT" = 1 ]; then git -C "$ROOT" add "$dir/source" "$dir/source.tar" >/dev/null 2>&1 git -C "$ROOT" -c commit.gpgsign=false commit -q -m "$name: propagate vendored source/ bump $vold -> $vnew (sync-recipe-source)" -- "$dir" >/dev/null 2>&1 fi fi done exit $RC