From 2153bf5f6e34503c7d2d35269f664864179df1b2 Mon Sep 17 00:00:00 2001 From: vasilito Date: Sat, 11 Jul 2026 23:18:41 +0300 Subject: [PATCH] build-system: add function-level fork verification + fix critical bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRITICAL: Add verify-fork-functions.sh — detects silent upstream code loss from bad merges that file-level verification cannot catch. This is the verification that would have caught the kfdwrite drop bug. Wire it into build-preflight.sh (pre-build gate) and upgrade-forks.sh (post-upgrade gate with automatic rollback on failure). Fix upgrade-forks.sh: - Fetch failures now abort instead of being silently swallowed - Backup branch names include fork name + PID to prevent collisions - Post-upgrade verification runs verify-fork-functions.sh and rolls back if upstream functions are missing Fix bump-fork.sh: - Replace wrong 'git tag -e' with 'git rev-parse --verify refs/tags/' - Clone failures now error explicitly instead of silent fallback - Version sed uses proper regex escaping for + characters - Atomic swap has rollback recovery if mv fails - Instructions now mention fork-upstream-map.toml update Fix sync-upstream.sh: - Define PROJECT_ROOT before use (was crashing under set -u) Fix build-preflight.sh: - Add REDBEAR_SKIP_FUNCTION_CHECK gate for verify-fork-functions.sh Kernel submodule pointer updated to 0.3.1 branch with kfdwrite restored. --- local/scripts/build-preflight.sh | 15 +++ local/scripts/bump-fork.sh | 53 +++++--- local/scripts/sync-upstream.sh | 5 +- local/scripts/upgrade-forks.sh | 27 +++- local/scripts/verify-fork-functions.sh | 165 +++++++++++++++++++++++++ local/sources/kernel | 2 +- 6 files changed, 248 insertions(+), 19 deletions(-) create mode 100755 local/scripts/verify-fork-functions.sh diff --git a/local/scripts/build-preflight.sh b/local/scripts/build-preflight.sh index 674c1e4020..12d685d9a8 100755 --- a/local/scripts/build-preflight.sh +++ b/local/scripts/build-preflight.sh @@ -69,6 +69,21 @@ if [ -x "$SCRIPT_DIR/check-fork-drift.sh" ] && [ "${REDBEAR_SKIP_DRIFT_CHECK:-0} fi fi +if [ -x "$SCRIPT_DIR/verify-fork-functions.sh" ] && [ "${REDBEAR_SKIP_FUNCTION_CHECK:-0}" != "1" ]; then + if ! "$SCRIPT_DIR/verify-fork-functions.sh" --no-fetch --quiet >/tmp/fork-functions.out 2>&1; then + cat /tmp/fork-functions.out >&2 + echo ">>> ERROR: Fork function verification failed — upstream functions are missing." >&2 + echo ">>> This indicates a bad merge dropped upstream code silently." >&2 + echo ">>> Run ./local/scripts/upgrade-forks.sh to fix." >&2 + if [ "${REDBEAR_SKIP_FORK_VERIFY:-0}" != "1" ]; then + exit 1 + fi + echo ">>> WARNING: Bypassed via REDBEAR_SKIP_FORK_VERIFY=1." >&2 + else + echo ">>> Preflight: fork function verification passed." + fi +fi + if [ -n "$RELEASE" ]; then bash "$SCRIPT_DIR/build-release-mode.sh" --release="$RELEASE" --config="$CONFIG" "${EXTRA_PACKAGES[@]/#/--extra-package=}" fi diff --git a/local/scripts/bump-fork.sh b/local/scripts/bump-fork.sh index fb5883dd72..cad9a9f554 100755 --- a/local/scripts/bump-fork.sh +++ b/local/scripts/bump-fork.sh @@ -114,17 +114,27 @@ STAGING_DIR="/tmp/redbear-bump-${COMPONENT}-$$" trap "rm -rf ${STAGING_DIR}" EXIT echo "Cloning ${UPSTREAM_GIT} at v${VERSION} into ${STAGING_DIR}..." -git clone --depth 1 --branch "v${VERSION}" "${UPSTREAM_GIT}" "${STAGING_DIR}" 2>/dev/null \ - || git clone --depth 1 "${UPSTREAM_GIT}" "${STAGING_DIR}" - -cd "${STAGING_DIR}" -git fetch --tags -if git tag -e "v${VERSION}" 2>/dev/null; then - git checkout "v${VERSION}" +if ! git clone --depth 1 --branch "v${VERSION}" "${UPSTREAM_GIT}" "${STAGING_DIR}" 2>&1; then + echo "WARNING: clone with tag v${VERSION} failed, trying without tag..." + if ! git clone --depth 1 "${UPSTREAM_GIT}" "${STAGING_DIR}" 2>&1; then + echo "ERROR: git clone of ${UPSTREAM_GIT} failed" >&2 + exit 1 + fi + cd "${STAGING_DIR}" + git fetch --tags --quiet + if git rev-parse --verify "refs/tags/v${VERSION}" >/dev/null 2>&1; then + git checkout "v${VERSION}" --quiet + elif git rev-parse --verify "refs/tags/${VERSION}" >/dev/null 2>&1; then + git checkout "${VERSION}" --quiet + else + echo "WARNING: tag v${VERSION} not found — using default branch HEAD" + fi + cd - >/dev/null else - echo "WARNING: tag v${VERSION} not found — using main branch HEAD" + cd "${STAGING_DIR}" + git fetch --tags --quiet 2>/dev/null || true + cd - >/dev/null fi -cd - >/dev/null # Apply Red Bear patches PATCH_DIR="local/patches/${COMPONENT}" @@ -144,15 +154,27 @@ if [[ -d "$PATCH_DIR" ]]; then fi # Update version in Cargo.toml -if [[ -f "${STAGING_DIR}/Cargo.toml" ]]; then - sed -i "s/^version = \"${CURRENT_VERSION}\"/version = \"${VERSION}\"/" "${STAGING_DIR}/Cargo.toml" +if [[ -f "${STAGING_DIR}/Cargo.toml" && -n "${CURRENT_VERSION}" ]]; then + escaped_current=$(printf '%s\n' "${CURRENT_VERSION}" | sed 's/[.[\*^$()+?{|\\]/\\&/g') + sed -i "s/^version = \"${escaped_current}\"/version = \"${VERSION}+rb${BRANCH_VERSION:-0.0.0}\"/" "${STAGING_DIR}/Cargo.toml" fi -# Replace local fork contents with staged (atomic via temp dir rename) +# Replace local fork contents echo "Updating ${LOCAL_DIR}..." -mv "${LOCAL_DIR}" "${LOCAL_DIR}.old-${$$}" -mv "${STAGING_DIR}" "${LOCAL_DIR}" -rm -rf "${LOCAL_DIR}.old-${$$}" +OLD_DIR="${LOCAL_DIR}.old-${$}" +if ! mv "${STAGING_DIR}" "${LOCAL_DIR}.new-${$}" 2>/dev/null; then + echo "ERROR: cannot move staging dir — disk full or permission denied" >&2 + rm -rf "${STAGING_DIR}" + exit 1 +fi +mv "${LOCAL_DIR}" "${OLD_DIR}" && mv "${LOCAL_DIR}.new-${$}" "${LOCAL_DIR}" +if [[ $? -ne 0 ]]; then + echo "ERROR: swap failed — restoring original" >&2 + mv "${OLD_DIR}" "${LOCAL_DIR}" 2>/dev/null || true + rm -rf "${LOCAL_DIR}.new-${$}" + exit 1 +fi +rm -rf "${OLD_DIR}" echo "" echo "SUCCESS: ${COMPONENT} bumped to v${VERSION}" @@ -162,4 +184,5 @@ echo " 1. cd ${LOCAL_DIR}" echo " 2. git diff (verify patches applied correctly)" echo " 3. git add -A && git commit -m 'bump: ${COMPONENT} v${CURRENT_VERSION} -> v${VERSION}'" echo " 4. git push origin submodule/${COMPONENT}" +echo " 5. Update local/fork-upstream-map.toml if the upstream tag changed" echo " 5. Rebuild affected packages: ./local/scripts/build-redbear.sh redbear-mini" \ No newline at end of file diff --git a/local/scripts/sync-upstream.sh b/local/scripts/sync-upstream.sh index 4ef4307a84..e3fdc73883 100755 --- a/local/scripts/sync-upstream.sh +++ b/local/scripts/sync-upstream.sh @@ -17,7 +17,10 @@ # Documentation: # local/docs/CONSOLE-TO-KDE-DESKTOP-PLAN.md -set -euo pipefail +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" GREEN='\033[1;32m' BLUE='\033[1;34m' diff --git a/local/scripts/upgrade-forks.sh b/local/scripts/upgrade-forks.sh index c8fd232ada..471cdf187e 100755 --- a/local/scripts/upgrade-forks.sh +++ b/local/scripts/upgrade-forks.sh @@ -132,7 +132,13 @@ for fork in "${TARGET_FORKS[@]}"; do # Determine upstream branch upstream_branch="master" - (cd "$fork_dir" && git fetch upstream --quiet 2>/dev/null) || true + if ! (cd "$fork_dir" && git fetch upstream --quiet 2>&1); then + echo -e " ${RED}FAIL: git fetch upstream failed (network error?)${NC}" + FAIL_COUNT=$((FAIL_COUNT + 1)) + FAILED_FORKS+=("$fork") + echo "" + continue + fi if ! (cd "$fork_dir" && git rev-parse --verify upstream/master >/dev/null 2>&1); then if (cd "$fork_dir" && git rev-parse --verify upstream/main >/dev/null 2>&1); then upstream_branch="main" @@ -210,7 +216,7 @@ for fork in "${TARGET_FORKS[@]}"; do fi # Create backup branch - backup_branch="rb-backup/$(date +%Y%m%d-%H%M%S)" + backup_branch="rb-backup/${fork}-$(date +%Y%m%d-%H%M%S)-$$" (cd "$fork_dir" && git branch "$backup_branch" HEAD 2>/dev/null) echo " Backup: $backup_branch" @@ -245,6 +251,23 @@ for fork in "${TARGET_FORKS[@]}"; do new_sha=$(cd "$fork_dir" && git rev-parse --short HEAD) echo -e " ${GREEN}DONE: $old_sha → $new_sha ($ahead patches reapplied)${NC}" echo -e " ${YELLOW}Backup at $backup_branch${NC}" + + # Post-upgrade verification: check no upstream functions were dropped + if [[ -x "$SCRIPT_DIR/verify-fork-functions.sh" ]]; then + if ! "$SCRIPT_DIR/verify-fork-functions.sh" "$fork" --no-fetch --quiet 2>/dev/null; then + echo -e " ${RED}VERIFICATION FAILED: upstream functions missing after upgrade!${NC}" + "$SCRIPT_DIR/verify-fork-functions.sh" "$fork" --no-fetch 2>&1 | grep -E "^ " | head -20 + echo -e " ${YELLOW}Restoring from backup — manual rebase needed${NC}" + (cd "$fork_dir" && git reset --hard "$backup_branch" 2>/dev/null) + FAIL_COUNT=$((FAIL_COUNT + 1)) + FAILED_FORKS+=("$fork") + echo "" + continue + else + echo -e " ${GREEN}VERIFIED: all upstream functions present${NC}" + fi + fi + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) SUCCEEDED_FORKS+=("$fork") echo "" diff --git a/local/scripts/verify-fork-functions.sh b/local/scripts/verify-fork-functions.sh new file mode 100755 index 0000000000..55e7fcb721 --- /dev/null +++ b/local/scripts/verify-fork-functions.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# verify-fork-functions.sh — Verify that local forks retain ALL upstream functions. +# +# This is the FUNCTION-LEVEL verification that prevents the silent code-loss bug +# that occurred when git merge conflict resolution dropped upstream functions +# (kfdwrite, bulk_insert_files, resize, read_with_timeout, etc.) from the kernel fork. +# +# For each fork with an upstream remote, this script: +# 1. Extracts all function definitions (fn ) from upstream source files +# 2. Checks that each function exists in the local fork +# 3. Reports any missing functions with file paths and line numbers +# +# This catches the class of bugs that file-level verification CANNOT detect: +# when a file is touched by RB patches, file-level diff allows ANY difference, +# including silently dropped upstream functions. +# +# Usage: +# ./local/scripts/verify-fork-functions.sh # Check all forks +# ./local/scripts/verify-fork-functions.sh kernel relibc # Check specific forks +# ./local/scripts/verify-fork-functions.sh --no-fetch # Skip network fetch +# ./local/scripts/verify-fork-functions.sh --quiet # Only print violations +# +# Exit codes: +# 0 = All upstream functions present in all forks +# 1 = One or more upstream functions are missing from forks + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +NO_FETCH=0 +QUIET=0 +declare -a TARGET_FORKS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-fetch) NO_FETCH=1 ;; + --quiet) QUIET=1 ;; + -h|--help) + echo "Usage: $0 [--no-fetch] [--quiet] [fork1 fork2 ...]" + echo "" + echo "Verifies that local forks retain ALL upstream functions." + echo "Catches silent code loss from bad merge conflict resolution." + exit 0 + ;; + --*) echo "Unknown: $1" >&2; exit 1 ;; + *) TARGET_FORKS+=("$1") ;; + esac + shift +done + +cd "$PROJECT_ROOT" + +# Discover forks if not specified +if [[ ${#TARGET_FORKS[@]} -eq 0 ]]; then + while IFS= read -r line; do + TARGET_FORKS+=("$line") + done < <(for d in local/sources/*/; do + [[ -d "$d/.git" ]] || continue + name=$(basename "$d") + git -C "$d" remote get-url upstream >/dev/null 2>&1 && echo "$name" + done | sort) +fi + +VIOLATIONS=0 +TOTAL_MISSING=0 + +for fork in "${TARGET_FORKS[@]}"; do + fork_dir="local/sources/${fork}" + + if [[ ! -d "$fork_dir/.git" ]]; then + continue + fi + + upstream_url=$(cd "$fork_dir" && git remote get-url upstream 2>/dev/null || true) + if [[ -z "$upstream_url" ]]; then + continue + fi + + # Determine upstream branch + upstream_branch="master" + if ! (cd "$fork_dir" && git rev-parse --verify upstream/master >/dev/null 2>&1); then + if (cd "$fork_dir" && git rev-parse --verify upstream/main >/dev/null 2>&1); then + upstream_branch="main" + else + if [[ "$NO_FETCH" -eq 0 ]]; then + (cd "$fork_dir" && git fetch upstream --depth=200 --quiet 2>/dev/null) || true + fi + if (cd "$fork_dir" && git rev-parse --verify upstream/master >/dev/null 2>&1); then + upstream_branch="master" + elif (cd "$fork_dir" && git rev-parse --verify upstream/main >/dev/null 2>&1); then + upstream_branch="main" + else + continue + fi + fi + fi + + # Fetch if needed + if [[ "$NO_FETCH" -eq 0 ]]; then + (cd "$fork_dir" && git fetch upstream --depth=200 --quiet 2>/dev/null) || true + fi + + upstream_ref="upstream/$upstream_branch" + + # Get list of source files that exist in BOTH upstream and our fork + upstream_files=$(cd "$fork_dir" && git ls-tree -r --name-only "$upstream_ref" 2>/dev/null | grep '\.rs$' | sort) + local_files=$(cd "$fork_dir" && find . -name '*.rs' -not -path './.git/*' -not -path './target/*' -not -path './stage/*' | sed 's|^\./||' | sort) + mapfile -t shared_files < <(comm -12 <(echo "$upstream_files") <(echo "$local_files")) + + fork_missing=0 + declare -a missing_details=() + + for f in "${shared_files[@]}"; do + # Extract function names from upstream version of the file + upstream_fns=$(cd "$fork_dir" && git show "${upstream_ref}:${f}" 2>/dev/null | \ + grep -oP '(?:pub )?(?:async )?(?:unsafe )?fn \w+' | \ + sed 's/fn //' | sed 's/ *$//' | sort -u) + + [[ -z "$upstream_fns" ]] && continue + + # Extract function names from our version of the file + local_fns=$(cd "$fork_dir" && grep -oP '(?:pub )?(?:async )?(?:unsafe )?fn \w+' "$f" 2>/dev/null | \ + sed 's/fn //' | sed 's/ *$//' | sort -u) + + # Find functions in upstream but not in our fork + missing=$(comm -23 <(echo "$upstream_fns") <(echo "$local_fns") 2>/dev/null) + + if [[ -n "$missing" ]]; then + while IFS= read -r fn; do + [[ -z "$fn" ]] && continue + missing_details+=(" $f: fn $fn") + fork_missing=$((fork_missing + 1)) + TOTAL_MISSING=$((TOTAL_MISSING + 1)) + done <<< "$missing" + fi + done + + if [[ "$fork_missing" -gt 0 ]]; then + if [[ "$QUIET" -eq 0 ]] || [[ "$fork_missing" -gt 0 ]]; then + echo "" + echo -e "\033[1;31mFAIL: $fork is missing $fork_missing upstream function(s):\033[0m" + for detail in "${missing_details[@]}"; do + echo "$detail" + done + echo "" + echo " These functions exist in upstream $upstream_ref but are MISSING from the local fork." + echo " This likely means a bad merge or cherry-pick dropped them silently." + echo " Fix: ./local/scripts/upgrade-forks.sh $fork" + fi + VIOLATIONS=$((VIOLATIONS + 1)) + elif [[ "$QUIET" -eq 0 ]]; then + echo -e "\033[1;32mOK: $fork — all upstream functions present\033[0m" + fi +done + +echo "" +if [[ "$VIOLATIONS" -gt 0 ]]; then + echo -e "\033[1;31mFUNCTION VERIFICATION FAILED: $VIOLATIONS fork(s) missing $TOTAL_MISSING upstream function(s).\033[0m" + exit 1 +fi + +echo -e "\033[1;32mAll forks retain all upstream functions.\033[0m" +exit 0 diff --git a/local/sources/kernel b/local/sources/kernel index 00e7ffb7bc..19b936efff 160000 --- a/local/sources/kernel +++ b/local/sources/kernel @@ -1 +1 @@ -Subproject commit 00e7ffb7bc18d2175ca69379d2dc7b6d76f612a9 +Subproject commit 19b936effff6fb3f06cc9cc8a1f6028ec8f1d413