vasilito
1c09490949
The engine silently no-op'd ('!! not found') when given full paths like
local/recipes/kde/kf6-kio/ — resolve only understood bare names or group/name
tokens. Now strips a trailing slash, an absolute ROOT prefix, and a
local/recipes/ prefix before matching. Test E22.
260 lines
14 KiB
Bash
Executable File
260 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# sync-recipe-source.sh — canonical version-propagation engine for tar recipes.
|
|
#
|
|
# THE MISSING STEP. Bumping a recipe's `tar=`/`blake3=` never propagated into the
|
|
# vendored `source/` tree, so the build silently compiled a stale version (Qt
|
|
# 6.11.0-vs-6.11.1, the whole KDE stack at 6.10.0 vs recipe 6.28.0). This engine
|
|
# makes `source.tar`, the vendored `source/` tree, its version stamp, and the
|
|
# recipe's declared version all agree — treating recipe.toml as the source of
|
|
# truth (the version to converge ON).
|
|
#
|
|
# For each recipe it rebases the vendored fork onto the declared upstream:
|
|
# source/(Vnew) = pristine(Vnew) + recipe patches + baked local delta
|
|
# where the "baked local delta" is any change in the current source/ that is NOT
|
|
# produced by a listed recipe patch (e.g. Redox shims committed directly). That
|
|
# delta is captured against pristine(Vold) and re-applied onto pristine(Vnew) so
|
|
# a re-lay can never silently drop a Redox port change. Injected line-duplication
|
|
# corruption (identical consecutive lines) is dropped.
|
|
#
|
|
# Usage:
|
|
# sync-recipe-source.sh [--check] [--commit] [--all|--config=NAME] <recipe>...
|
|
# --check dry-run: classify each recipe, apply nothing, write nothing
|
|
# --commit git commit each synced recipe individually
|
|
# --all every tar recipe under local/recipes (excl. wip)
|
|
# <recipe> a recipe dir or bare name (e.g. kde/kf6-kio or kf6-kio)
|
|
#
|
|
# Exit: 0 all synced/consistent; 2 some need manual rebase (patches/shims reject).
|
|
|
|
set -uo pipefail
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
CHECK=0; COMMIT=0; ALL=0; declare -a WANT=()
|
|
for a in "$@"; do case "$a" in
|
|
--check) CHECK=1;; --commit) COMMIT=1;; --all) ALL=1;;
|
|
--config=*) : ;; # reserved
|
|
--*) echo "unknown opt: $a" >&2; exit 1;;
|
|
*) WANT+=("$a");;
|
|
esac; done
|
|
|
|
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
|
ver_of(){ grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)*' <<<"$1" | tail -1; }
|
|
|
|
# resolve a recipe token to its dir. Accepts: a bare name (kf6-kio), a
|
|
# recipes-relative path (kde/kf6-kio), a repo-relative path
|
|
# (local/recipes/kde/kf6-kio), or an absolute path — with or without a trailing
|
|
# slash.
|
|
resolve(){ local t="${1%/}" # strip trailing slash
|
|
t="${t#"$ROOT"/}" # strip absolute ROOT prefix
|
|
t="${t#local/recipes/}" # strip the recipes/ prefix -> group/name or bare name
|
|
[ -f "$ROOT/local/recipes/$t/recipe.toml" ] && { echo "$ROOT/local/recipes/$t"; return; }
|
|
local hit; hit=$(find "$ROOT/local/recipes" -name recipe.toml -not -path '*/wip/*' -path "*/$t/recipe.toml" 2>/dev/null | head -1)
|
|
[ -n "$hit" ] && echo "$(dirname "$hit")"
|
|
}
|
|
|
|
# vendored source/ version marker. ACTUAL code markers (Qt .cmake.conf, KDE
|
|
# KF_VERSION) are authoritative and checked FIRST; the generic .redbear-src-version
|
|
# stamp is only a fallback for trees with no code marker. Rationale: the stamp is
|
|
# a cache the engine writes, and it can go stale (e.g. a code-only revert) — a
|
|
# stale stamp must never mask the real version and skip a needed rebase.
|
|
src_version(){ local s="$1"
|
|
[ -f "$s/.cmake.conf" ] && { local v; v=$(ver_of "$(grep -m1 QT_REPO_MODULE_VERSION "$s/.cmake.conf")"); [ -n "$v" ] && { echo "$v"; return; }; }
|
|
[ -f "$s/CMakeLists.txt" ] && { local v; v=$(grep -m1 -E 'set\(KF_VERSION|project\([A-Za-z0-9_]+ VERSION [0-9]' "$s/CMakeLists.txt" 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); [ -n "$v" ] && { echo "$v"; return; }; }
|
|
[ -f "$s/.redbear-src-version" ] && { ver_of "$(cat "$s/.redbear-src-version")"; return; }
|
|
echo ""
|
|
}
|
|
|
|
# apply a patch trying -p1 then -p0; $1=tree-root $2=patchfile ; echo level or FAIL.
|
|
# --fuzz=0: require EXACT context (line offsets are still allowed) so a patch
|
|
# whose context no longer matches the new upstream is REJECTED, not silently
|
|
# force-fitted with fuzz — that silent force-fit is how a stale/wrong port would
|
|
# sneak through.
|
|
apply_patch(){ local root="$1" pf="$2" lvl
|
|
for lvl in 1 0; do
|
|
if ( cd "$root" && patch -p$lvl --fuzz=0 --dry-run -f <"$pf" >/dev/null 2>&1 ); then
|
|
( cd "$root" && patch -p$lvl --fuzz=0 -s -f <"$pf" >/dev/null 2>&1 ); echo "$lvl"; return 0
|
|
fi
|
|
done
|
|
echo FAIL; return 1
|
|
}
|
|
|
|
# Drop the line-duplication corruption pattern: a run of >=4 identical
|
|
# consecutive NON-BLANK lines collapses to a single line (the observed
|
|
# corruption injected 5-13 copies). Runs of <=3 and blank-line runs are kept
|
|
# verbatim, so legitimate paired lines (`}`/`}`, `#endif`/`#endif`) survive.
|
|
decorrupt(){ awk '
|
|
{ a[NR]=$0 }
|
|
END {
|
|
i=1
|
|
while (i<=NR) {
|
|
j=i
|
|
while (j<NR && a[j+1]==a[i] && a[i]!="") j++
|
|
if (j-i+1 >= 4 && a[i]!="") print a[i]
|
|
else for (k=i;k<=j;k++) print a[k]
|
|
i=j+1
|
|
}
|
|
}' "$1"; }
|
|
|
|
# --max-time 120 + --speed-limit/--speed-time: a stalled mirror fails fast (a
|
|
# module then reports DL-FAIL and is flagged) instead of hanging the whole run.
|
|
download(){ curl -sLf --retry 2 --connect-timeout 15 --max-time 120 \
|
|
--speed-limit 1024 --speed-time 30 "$1" -o "$2" 2>/dev/null; }
|
|
|
|
# Reconcile recipe blake3 to the authoritative source.tar. A vendored bump often
|
|
# left recipe blake3 stale (URL bumped, hash not), so recipe.toml's declared hash
|
|
# no longer matched its own tarball. Only reconciles when source.tar exists AND
|
|
# its top-dir version equals the recipe URL version (so we never stamp the hash
|
|
# of a wrong-version tarball). No-op under --check.
|
|
reconcile_blake3(){ # $1=dir $2=recipe.toml $3=vnew
|
|
[ "$CHECK" = 1 ] && return 0
|
|
command -v b3sum >/dev/null 2>&1 || return 0
|
|
[ -f "$1/source.tar" ] || return 0
|
|
local tvv; tvv="$(ver_of "$(tar tf "$1/source.tar" 2>/dev/null | head -1)")"
|
|
[ "$tvv" = "$3" ] || return 0
|
|
grep -qE '^[[:space:]]*blake3[[:space:]]*=' "$2" || return 0
|
|
local h; h="$(b3sum "$1/source.tar" | awk '{print $1}')"
|
|
sed -i -E "s|^([[:space:]]*blake3[[:space:]]*=[[:space:]]*)\"[0-9a-f]*\"|\1\"$h\"|" "$2"
|
|
}
|
|
|
|
# build the recipe list
|
|
declare -a RECIPES=()
|
|
if [ "$ALL" = 1 ]; then
|
|
while IFS= read -r r; do grep -qE '^[[:space:]]*tar[[:space:]]*=' "$r" && RECIPES+=("$(dirname "$r")"); done \
|
|
< <(find "$ROOT/local/recipes" -name recipe.toml -not -path '*/wip/*')
|
|
else
|
|
for t in "${WANT[@]}"; do d=$(resolve "$t"); [ -n "$d" ] && RECIPES+=("$d") || echo "!! not found: $t" >&2; done
|
|
fi
|
|
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' RECIPE FROM TO PATCHES BAKED STATUS
|
|
RC=0
|
|
for dir in "${RECIPES[@]}"; do
|
|
rc="$dir/recipe.toml"; name="$(basename "$dir")"
|
|
url="$(grep -m1 -E '^[[:space:]]*tar[[:space:]]*=' "$rc" | sed -E 's/^[^"]*"([^"]+)".*/\1/')"; [ -n "$url" ] || continue
|
|
vnew="$(ver_of "$(basename "$url")")"; [ -n "$vnew" ] || continue
|
|
blake="$(grep -m1 -E '^[[:space:]]*blake3[[:space:]]*=' "$rc" | grep -oE '[0-9a-f]{64}')"
|
|
if [ ! -d "$dir/source" ]; then
|
|
# Transient recipe: cookbook builds from source.tar/URL, so a stale cached
|
|
# source.tar is a stale build. Ensure it matches the declared version.
|
|
tst=""; [ -f "$dir/source.tar" ] && tst="$(ver_of "$(tar tf "$dir/source.tar" 2>/dev/null | head -1)")"
|
|
if [ -z "$tst" ] || [ "$tst" = "$vnew" ]; then
|
|
# no cached tar (cookbook fetches fresh) or already correct: just reconcile blake3
|
|
reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "${tst:-none}" "$vnew" "-" "-" "transient(ok)"
|
|
elif [ "$CHECK" = 1 ]; then
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(stale-tar)"
|
|
elif download "$url" "$WORK/$name-tr.tar"; then
|
|
cp "$WORK/$name-tr.tar" "$dir/source.tar"; reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(tar-bumped)"
|
|
else
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(DL-FAIL)"; RC=2
|
|
fi
|
|
continue
|
|
fi
|
|
vold="$(src_version "$dir/source")"; vold="${vold:-unknown}"
|
|
mapfile -t patches < <(awk '/^[[:space:]]*patches[[:space:]]*=/{p=1} p{print} p&&/\]/{exit}' "$rc" | grep -oE '"[^"]+\.patch"' | tr -d '"')
|
|
# baked marker files (Redox) not covered by patches
|
|
baked=$(git -C "$ROOT" grep -lI "__redox__\|Q_OS_REDOX\|defined(__redox" HEAD -- "$dir/source" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$vold" = "$vnew" ]; then
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "consistent"
|
|
[ "$CHECK" = 1 ] && continue
|
|
reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
printf '%s\n' "$vnew" > "$dir/source/.redbear-src-version"; continue
|
|
fi
|
|
# Version marker unreadable: we cannot prove divergence, so we must NOT rebase
|
|
# (that could clobber a carefully-ported tree like mesa). Report + skip; the
|
|
# fix is to add a source/.redbear-src-version stamp once the tree is verified.
|
|
if [ "$vold" = "unknown" ]; then
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "unknown" "$vnew" "${#patches[@]}" "$baked" "no-marker(skip; stamp to enable)"
|
|
continue
|
|
fi
|
|
|
|
# --- obtain pristine Vnew ---
|
|
base="$WORK/$name-new"; rm -rf "$base"; mkdir -p "$base"
|
|
tarball="$dir/source.tar"; tv=""
|
|
[ -f "$tarball" ] && tv="$(ver_of "$(tar tf "$tarball" 2>/dev/null | head -1)")"
|
|
if [ "$tv" != "$vnew" ]; then
|
|
download "$url" "$WORK/$name-new.tar" && tarball="$WORK/$name-new.tar" || { printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "DL-FAIL"; RC=2; continue; }
|
|
fi
|
|
tar xf "$tarball" -C "$base" 2>/dev/null
|
|
top="$(tar tf "$tarball" 2>/dev/null | head -1 | cut -d/ -f1)"
|
|
mv "$base/$top" "$base/source" 2>/dev/null || { printf '%-26s %-9s %-9s %s\n' "$name" "$vold" "$vnew" "EXTRACT-FAIL"; RC=2; continue; }
|
|
|
|
# --- apply recipe patches ---
|
|
preject=0; for p in "${patches[@]}"; do
|
|
pf="$dir/$p"; [ -f "$pf" ] || pf="$ROOT/local/patches/$p"; [ -f "$pf" ] || { preject=1; continue; }
|
|
lv=$(apply_patch "$base/source" "$pf"); [ "$lv" = FAIL ] && preject=1
|
|
done
|
|
|
|
# --- preserve RedBear additions + reapply baked Redox deltas ---
|
|
# This runs for EVERY vendored rebase (not only marker-bearing ones): a
|
|
# RedBear-added file (e.g. a Redox Wayland-crash wrapper .sh, a utmp compat
|
|
# header) has no __redox__ token, so a grep-only capture misses it and
|
|
# rsync --delete drops it. We compare against pristine(Vold) by PATH to keep
|
|
# every RedBear addition, content-agnostic.
|
|
breject=0
|
|
if [ "$vold" != unknown ]; then
|
|
# Old-version URL: replace the full version, then the major.minor dir
|
|
# component (KDE mirrors nest under .../6.28/foo-6.28.0 -> 6.10/foo-6.10.0).
|
|
oldurl="${url//$vnew/$vold}"
|
|
vnm="${vnew%.*}"; vom="${vold%.*}"
|
|
[ "$vnm" != "$vnew" ] && [ "$vnm" != "$vom" ] && oldurl="${oldurl//$vnm/$vom}"
|
|
if download "$oldurl" "$WORK/$name-old.tar"; then
|
|
op="$WORK/$name-old"; rm -rf "$op"; mkdir -p "$op"; tar xf "$WORK/$name-old.tar" -C "$op" 2>/dev/null
|
|
otop="$(tar tf "$WORK/$name-old.tar" 2>/dev/null | head -1 | cut -d/ -f1)"; mv "$op/$otop" "$op/source" 2>/dev/null
|
|
relroot="${dir#$ROOT/}" # repo-relative recipe dir (git needs this)
|
|
gitf="$WORK/gitf"
|
|
|
|
# (A) Carry forward every committed file absent from pristine(Vold): these
|
|
# are RedBear additions upstream never had. Skip the stamp and files
|
|
# the new pristine/patches already provide.
|
|
while IFS= read -r rel; do
|
|
[ "$rel" = ".redbear-src-version" ] && continue
|
|
[ -e "$op/source/$rel" ] && continue # exists upstream -> not a RedBear addition
|
|
[ -e "$base/source/$rel" ] && continue # already provided by pristine(Vnew) or a patch
|
|
if ! git -C "$ROOT" show "HEAD:$relroot/source/$rel" > "$gitf" 2>/dev/null; then breject=1; continue; fi
|
|
decorrupt "$gitf" > "$gitf.clean"
|
|
mkdir -p "$base/source/$(dirname "$rel")"; cp "$gitf.clean" "$base/source/$rel"
|
|
done < <(git -C "$ROOT" ls-tree -r --name-only HEAD -- "$relroot/source" | while IFS= read -r p; do printf '%s\n' "${p#"$relroot"/source/}"; done)
|
|
|
|
# (B) For marker-bearing files that ALSO exist in Vold (modified upstream
|
|
# files), port the Redox delta. Added marker files were handled by (A).
|
|
if [ "$baked" -gt 0 ]; then
|
|
while IFS= read -r line; do
|
|
rel="${line#*/source/}"
|
|
[ -f "$op/source/$rel" ] || continue # added file -> already carried by (A)
|
|
if ! git -C "$ROOT" show "HEAD:$relroot/source/$rel" > "$gitf" 2>/dev/null || [ ! -s "$gitf" ]; then
|
|
breject=1; continue
|
|
fi
|
|
decorrupt "$gitf" > "$gitf.clean"
|
|
if [ ! -f "$base/source/$rel" ]; then breject=1; continue; fi # gone in new upstream -> manual
|
|
diff -q "$op/source/$rel" "$gitf.clean" >/dev/null 2>&1 && continue # marker is upstream's own
|
|
# Capture delta to a file (never pipe diff|patch: diff exits 1 on any
|
|
# difference and pipefail would false-reject every shim).
|
|
diff -u "$op/source/$rel" "$gitf.clean" > "$WORK/delta.patch" 2>/dev/null
|
|
if ! patch --fuzz=0 -s "$base/source/$rel" < "$WORK/delta.patch" >/dev/null 2>&1; then
|
|
breject=1
|
|
fi
|
|
done < <(git -C "$ROOT" grep -lI "__redox__\|Q_OS_REDOX\|defined(__redox" HEAD -- "$dir/source" 2>/dev/null)
|
|
fi
|
|
else breject=1; fi
|
|
fi
|
|
|
|
printf '%s\n' "$vnew" > "$base/source/.redbear-src-version"
|
|
status="ready"; [ "$preject" = 1 ] && status="PATCH-REJECT(manual)"; [ "$breject" = 1 ] && status="${status};SHIM-REJECT(manual)"
|
|
[ "$preject" = 1 ] || [ "$breject" = 1 ] && RC=2
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "$status"
|
|
|
|
if [ "$CHECK" = 0 ] && [ "$preject" = 0 ] && [ "$breject" = 0 ]; then
|
|
# --checksum: compare by content, never by size+mtime. Two files of equal
|
|
# size with a coincidental mtime (e.g. "v1\n" vs "v2\n") would otherwise be
|
|
# skipped by rsync's quick-check, leaving stale content in source/.
|
|
rsync -a --delete --checksum "$base/source/" "$dir/source/"
|
|
# keep source.tar authoritative for Vnew, and reconcile recipe blake3 to it
|
|
[ -f "$WORK/$name-new.tar" ] && cp "$WORK/$name-new.tar" "$dir/source.tar"
|
|
reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
if [ "$COMMIT" = 1 ]; then
|
|
git -C "$ROOT" add "$dir/source" "$dir/source.tar" >/dev/null 2>&1
|
|
git -C "$ROOT" -c commit.gpgsign=false commit -q -m "$name: propagate vendored source/ bump $vold -> $vnew (sync-recipe-source)" -- "$dir" >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
done
|
|
exit $RC
|