vasilito
49377f768c
Found by extending the edge-case suite to the baked-shim path (E17-E19) after a
sync reported 'ready' but silently dropped a module's Redox shim:
- git show used an ABSOLUTE recipe path ('HEAD:/mnt/.../source/f') which git
rejects (needs repo-relative) -> empty content -> garbage delta. Now uses
${dir#$ROOT/}.
- diff|patch under 'set -o pipefail': diff exits 1 whenever files differ (always
here), so the pipeline looked failed even when patch succeeded -> EVERY shim
false-rejected. Now captures the delta to a file and checks patch's own exit.
- a shim whose file was renamed/removed upstream, or unreadable from HEAD, was
silently skipped. Now: Redox-added files are carried in verbatim; a missing
target or unreadable HEAD file -> SHIM-REJECT (manual), never a silent drop.
Also: old-version URL derivation now rewrites the major.minor dir component too
(KDE mirrors nest tarballs under .../6.28/foo-6.28.0). CI branch glob widened to
[0-9]* so 0.4.0/0.5.0 release branches keep triggering.
238 lines
13 KiB
Bash
Executable File
238 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# sync-recipe-source.sh — canonical version-propagation engine for tar recipes.
|
|
#
|
|
# THE MISSING STEP. Bumping a recipe's `tar=`/`blake3=` never propagated into the
|
|
# vendored `source/` tree, so the build silently compiled a stale version (Qt
|
|
# 6.11.0-vs-6.11.1, the whole KDE stack at 6.10.0 vs recipe 6.28.0). This engine
|
|
# makes `source.tar`, the vendored `source/` tree, its version stamp, and the
|
|
# recipe's declared version all agree — treating recipe.toml as the source of
|
|
# truth (the version to converge ON).
|
|
#
|
|
# For each recipe it rebases the vendored fork onto the declared upstream:
|
|
# source/(Vnew) = pristine(Vnew) + recipe patches + baked local delta
|
|
# where the "baked local delta" is any change in the current source/ that is NOT
|
|
# produced by a listed recipe patch (e.g. Redox shims committed directly). That
|
|
# delta is captured against pristine(Vold) and re-applied onto pristine(Vnew) so
|
|
# a re-lay can never silently drop a Redox port change. Injected line-duplication
|
|
# corruption (identical consecutive lines) is dropped.
|
|
#
|
|
# Usage:
|
|
# sync-recipe-source.sh [--check] [--commit] [--all|--config=NAME] <recipe>...
|
|
# --check dry-run: classify each recipe, apply nothing, write nothing
|
|
# --commit git commit each synced recipe individually
|
|
# --all every tar recipe under local/recipes (excl. wip)
|
|
# <recipe> a recipe dir or bare name (e.g. kde/kf6-kio or kf6-kio)
|
|
#
|
|
# Exit: 0 all synced/consistent; 2 some need manual rebase (patches/shims reject).
|
|
|
|
set -uo pipefail
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
CHECK=0; COMMIT=0; ALL=0; declare -a WANT=()
|
|
for a in "$@"; do case "$a" in
|
|
--check) CHECK=1;; --commit) COMMIT=1;; --all) ALL=1;;
|
|
--config=*) : ;; # reserved
|
|
--*) echo "unknown opt: $a" >&2; exit 1;;
|
|
*) WANT+=("$a");;
|
|
esac; done
|
|
|
|
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
|
ver_of(){ grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)*' <<<"$1" | tail -1; }
|
|
|
|
# resolve a recipe token to its dir
|
|
resolve(){ local t="$1"
|
|
[ -f "$ROOT/local/recipes/$t/recipe.toml" ] && { echo "$ROOT/local/recipes/$t"; return; }
|
|
local hit; hit=$(find "$ROOT/local/recipes" -name recipe.toml -not -path '*/wip/*' -path "*/$t/recipe.toml" 2>/dev/null | head -1)
|
|
[ -n "$hit" ] && echo "$(dirname "$hit")"
|
|
}
|
|
|
|
# vendored source/ version marker (KDE KF_VERSION, Qt .cmake.conf, generic stamp, autotools)
|
|
src_version(){ local s="$1"
|
|
[ -f "$s/.redbear-src-version" ] && { ver_of "$(cat "$s/.redbear-src-version")"; return; }
|
|
[ -f "$s/.cmake.conf" ] && { local v; v=$(ver_of "$(grep -m1 QT_REPO_MODULE_VERSION "$s/.cmake.conf")"); [ -n "$v" ] && { echo "$v"; return; }; }
|
|
[ -f "$s/CMakeLists.txt" ] && { local v; v=$(grep -m1 -E 'set\(KF_VERSION|project\([A-Za-z0-9_]+ VERSION [0-9]' "$s/CMakeLists.txt" 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); [ -n "$v" ] && { echo "$v"; return; }; }
|
|
echo ""
|
|
}
|
|
|
|
# apply a patch trying -p1 then -p0; $1=tree-root $2=patchfile ; echo level or FAIL.
|
|
# --fuzz=0: require EXACT context (line offsets are still allowed) so a patch
|
|
# whose context no longer matches the new upstream is REJECTED, not silently
|
|
# force-fitted with fuzz — that silent force-fit is how a stale/wrong port would
|
|
# sneak through.
|
|
apply_patch(){ local root="$1" pf="$2" lvl
|
|
for lvl in 1 0; do
|
|
if ( cd "$root" && patch -p$lvl --fuzz=0 --dry-run -f <"$pf" >/dev/null 2>&1 ); then
|
|
( cd "$root" && patch -p$lvl --fuzz=0 -s -f <"$pf" >/dev/null 2>&1 ); echo "$lvl"; return 0
|
|
fi
|
|
done
|
|
echo FAIL; return 1
|
|
}
|
|
|
|
# Drop the line-duplication corruption pattern: a run of >=4 identical
|
|
# consecutive NON-BLANK lines collapses to a single line (the observed
|
|
# corruption injected 5-13 copies). Runs of <=3 and blank-line runs are kept
|
|
# verbatim, so legitimate paired lines (`}`/`}`, `#endif`/`#endif`) survive.
|
|
decorrupt(){ awk '
|
|
{ a[NR]=$0 }
|
|
END {
|
|
i=1
|
|
while (i<=NR) {
|
|
j=i
|
|
while (j<NR && a[j+1]==a[i] && a[i]!="") j++
|
|
if (j-i+1 >= 4 && a[i]!="") print a[i]
|
|
else for (k=i;k<=j;k++) print a[k]
|
|
i=j+1
|
|
}
|
|
}' "$1"; }
|
|
|
|
download(){ curl -sLf --retry 3 --connect-timeout 20 --max-time 600 "$1" -o "$2" 2>/dev/null; }
|
|
|
|
# Reconcile recipe blake3 to the authoritative source.tar. A vendored bump often
|
|
# left recipe blake3 stale (URL bumped, hash not), so recipe.toml's declared hash
|
|
# no longer matched its own tarball. Only reconciles when source.tar exists AND
|
|
# its top-dir version equals the recipe URL version (so we never stamp the hash
|
|
# of a wrong-version tarball). No-op under --check.
|
|
reconcile_blake3(){ # $1=dir $2=recipe.toml $3=vnew
|
|
[ "$CHECK" = 1 ] && return 0
|
|
command -v b3sum >/dev/null 2>&1 || return 0
|
|
[ -f "$1/source.tar" ] || return 0
|
|
local tvv; tvv="$(ver_of "$(tar tf "$1/source.tar" 2>/dev/null | head -1)")"
|
|
[ "$tvv" = "$3" ] || return 0
|
|
grep -qE '^[[:space:]]*blake3[[:space:]]*=' "$2" || return 0
|
|
local h; h="$(b3sum "$1/source.tar" | awk '{print $1}')"
|
|
sed -i -E "s|^([[:space:]]*blake3[[:space:]]*=[[:space:]]*)\"[0-9a-f]*\"|\1\"$h\"|" "$2"
|
|
}
|
|
|
|
# build the recipe list
|
|
declare -a RECIPES=()
|
|
if [ "$ALL" = 1 ]; then
|
|
while IFS= read -r r; do grep -qE '^[[:space:]]*tar[[:space:]]*=' "$r" && RECIPES+=("$(dirname "$r")"); done \
|
|
< <(find "$ROOT/local/recipes" -name recipe.toml -not -path '*/wip/*')
|
|
else
|
|
for t in "${WANT[@]}"; do d=$(resolve "$t"); [ -n "$d" ] && RECIPES+=("$d") || echo "!! not found: $t" >&2; done
|
|
fi
|
|
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' RECIPE FROM TO PATCHES BAKED STATUS
|
|
RC=0
|
|
for dir in "${RECIPES[@]}"; do
|
|
rc="$dir/recipe.toml"; name="$(basename "$dir")"
|
|
url="$(grep -m1 -E '^[[:space:]]*tar[[:space:]]*=' "$rc" | sed -E 's/^[^"]*"([^"]+)".*/\1/')"; [ -n "$url" ] || continue
|
|
vnew="$(ver_of "$(basename "$url")")"; [ -n "$vnew" ] || continue
|
|
blake="$(grep -m1 -E '^[[:space:]]*blake3[[:space:]]*=' "$rc" | grep -oE '[0-9a-f]{64}')"
|
|
if [ ! -d "$dir/source" ]; then
|
|
# Transient recipe: cookbook builds from source.tar/URL, so a stale cached
|
|
# source.tar is a stale build. Ensure it matches the declared version.
|
|
tst=""; [ -f "$dir/source.tar" ] && tst="$(ver_of "$(tar tf "$dir/source.tar" 2>/dev/null | head -1)")"
|
|
if [ -z "$tst" ] || [ "$tst" = "$vnew" ]; then
|
|
# no cached tar (cookbook fetches fresh) or already correct: just reconcile blake3
|
|
reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "${tst:-none}" "$vnew" "-" "-" "transient(ok)"
|
|
elif [ "$CHECK" = 1 ]; then
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(stale-tar)"
|
|
elif download "$url" "$WORK/$name-tr.tar"; then
|
|
cp "$WORK/$name-tr.tar" "$dir/source.tar"; reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(tar-bumped)"
|
|
else
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$tst" "$vnew" "-" "-" "transient(DL-FAIL)"; RC=2
|
|
fi
|
|
continue
|
|
fi
|
|
vold="$(src_version "$dir/source")"; vold="${vold:-unknown}"
|
|
mapfile -t patches < <(awk '/^[[:space:]]*patches[[:space:]]*=/{p=1} p{print} p&&/\]/{exit}' "$rc" | grep -oE '"[^"]+\.patch"' | tr -d '"')
|
|
# baked marker files (Redox) not covered by patches
|
|
baked=$(git -C "$ROOT" grep -lI "__redox__\|Q_OS_REDOX\|defined(__redox" HEAD -- "$dir/source" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$vold" = "$vnew" ]; then
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "consistent"
|
|
[ "$CHECK" = 1 ] && continue
|
|
reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
printf '%s\n' "$vnew" > "$dir/source/.redbear-src-version"; continue
|
|
fi
|
|
# Version marker unreadable: we cannot prove divergence, so we must NOT rebase
|
|
# (that could clobber a carefully-ported tree like mesa). Report + skip; the
|
|
# fix is to add a source/.redbear-src-version stamp once the tree is verified.
|
|
if [ "$vold" = "unknown" ]; then
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "unknown" "$vnew" "${#patches[@]}" "$baked" "no-marker(skip; stamp to enable)"
|
|
continue
|
|
fi
|
|
|
|
# --- obtain pristine Vnew ---
|
|
base="$WORK/$name-new"; rm -rf "$base"; mkdir -p "$base"
|
|
tarball="$dir/source.tar"; tv=""
|
|
[ -f "$tarball" ] && tv="$(ver_of "$(tar tf "$tarball" 2>/dev/null | head -1)")"
|
|
if [ "$tv" != "$vnew" ]; then
|
|
download "$url" "$WORK/$name-new.tar" && tarball="$WORK/$name-new.tar" || { printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "DL-FAIL"; RC=2; continue; }
|
|
fi
|
|
tar xf "$tarball" -C "$base" 2>/dev/null
|
|
top="$(tar tf "$tarball" 2>/dev/null | head -1 | cut -d/ -f1)"
|
|
mv "$base/$top" "$base/source" 2>/dev/null || { printf '%-26s %-9s %-9s %s\n' "$name" "$vold" "$vnew" "EXTRACT-FAIL"; RC=2; continue; }
|
|
|
|
# --- apply recipe patches ---
|
|
preject=0; for p in "${patches[@]}"; do
|
|
pf="$dir/$p"; [ -f "$pf" ] || pf="$ROOT/local/patches/$p"; [ -f "$pf" ] || { preject=1; continue; }
|
|
lv=$(apply_patch "$base/source" "$pf"); [ "$lv" = FAIL ] && preject=1
|
|
done
|
|
|
|
# --- capture + reapply baked delta (changes beyond patches) ---
|
|
breject=0
|
|
if [ "$baked" -gt 0 ] && [ "$vold" != unknown ]; then
|
|
# Derive the old-version URL: replace the full version, then the
|
|
# major.minor directory component (KDE mirrors nest tarballs under a
|
|
# major.minor dir, e.g. .../frameworks/6.28/foo-6.28.0.tar.xz -> 6.10/foo-6.10.0).
|
|
oldurl="${url//$vnew/$vold}"
|
|
vnm="${vnew%.*}"; vom="${vold%.*}"
|
|
[ "$vnm" != "$vnew" ] && [ "$vnm" != "$vom" ] && oldurl="${oldurl//$vnm/$vom}"
|
|
if download "$oldurl" "$WORK/$name-old.tar"; then
|
|
op="$WORK/$name-old"; rm -rf "$op"; mkdir -p "$op"; tar xf "$WORK/$name-old.tar" -C "$op" 2>/dev/null
|
|
otop="$(tar tf "$WORK/$name-old.tar" 2>/dev/null | head -1 | cut -d/ -f1)"; mv "$op/$otop" "$op/source" 2>/dev/null
|
|
# For each Redox-marker file uncovered by patches, port its delta onto base.
|
|
# A shim must NEVER be silently dropped: any failure -> breject (manual).
|
|
relroot="${dir#$ROOT/}" # repo-relative recipe dir (git show needs this)
|
|
while IFS= read -r line; do
|
|
rel="${line#*/source/}"
|
|
gitf="$WORK/gitf"
|
|
if ! git -C "$ROOT" show "HEAD:$relroot/source/$rel" > "$gitf" 2>/dev/null || [ ! -s "$gitf" ]; then
|
|
breject=1; continue # cannot read the Redox-modified file from HEAD
|
|
fi
|
|
decorrupt "$gitf" > "$gitf.clean"
|
|
if [ ! -f "$op/source/$rel" ]; then
|
|
# Redox-ADDED file (absent from old upstream): carry it into the new tree verbatim.
|
|
mkdir -p "$base/source/$(dirname "$rel")"; cp "$gitf.clean" "$base/source/$rel"; continue
|
|
fi
|
|
if [ ! -f "$base/source/$rel" ]; then
|
|
breject=1; continue # file existed in old upstream+HEAD but gone in new -> manual
|
|
fi
|
|
if diff -q "$op/source/$rel" "$gitf.clean" >/dev/null 2>&1; then
|
|
continue # marker is upstream's own (no Redox delta) -> nothing to port
|
|
fi
|
|
# Capture the delta to a file — do NOT pipe diff|patch. `diff` exits 1
|
|
# whenever the files differ (always, here), and under `set -o pipefail`
|
|
# that makes the pipeline look failed even when patch succeeded, which
|
|
# would false-reject EVERY shim. Check patch's own exit status instead.
|
|
diff -u "$op/source/$rel" "$gitf.clean" > "$WORK/delta.patch" 2>/dev/null
|
|
if ! patch --fuzz=0 -s "$base/source/$rel" < "$WORK/delta.patch" >/dev/null 2>&1; then
|
|
breject=1 # Redox delta will not apply cleanly at new upstream -> manual
|
|
fi
|
|
done < <(git -C "$ROOT" grep -lI "__redox__\|Q_OS_REDOX\|defined(__redox" HEAD -- "$dir/source" 2>/dev/null)
|
|
else breject=1; fi
|
|
fi
|
|
|
|
printf '%s\n' "$vnew" > "$base/source/.redbear-src-version"
|
|
status="ready"; [ "$preject" = 1 ] && status="PATCH-REJECT(manual)"; [ "$breject" = 1 ] && status="${status};SHIM-REJECT(manual)"
|
|
[ "$preject" = 1 ] || [ "$breject" = 1 ] && RC=2
|
|
printf '%-26s %-9s %-9s %-8s %-6s %s\n' "$name" "$vold" "$vnew" "${#patches[@]}" "$baked" "$status"
|
|
|
|
if [ "$CHECK" = 0 ] && [ "$preject" = 0 ] && [ "$breject" = 0 ]; then
|
|
# --checksum: compare by content, never by size+mtime. Two files of equal
|
|
# size with a coincidental mtime (e.g. "v1\n" vs "v2\n") would otherwise be
|
|
# skipped by rsync's quick-check, leaving stale content in source/.
|
|
rsync -a --delete --checksum "$base/source/" "$dir/source/"
|
|
# keep source.tar authoritative for Vnew, and reconcile recipe blake3 to it
|
|
[ -f "$WORK/$name-new.tar" ] && cp "$WORK/$name-new.tar" "$dir/source.tar"
|
|
reconcile_blake3 "$dir" "$rc" "$vnew"
|
|
if [ "$COMMIT" = 1 ]; then
|
|
git -C "$ROOT" add "$dir/source" "$dir/source.tar" >/dev/null 2>&1
|
|
git -C "$ROOT" -c commit.gpgsign=false commit -q -m "$name: propagate vendored source/ bump $vold -> $vnew (sync-recipe-source)" -- "$dir" >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
done
|
|
exit $RC
|