vasilito
7f2920f861
verify-fork-versions.sh reads the 4th column of fork-upstream-map.toml and skips the content check for forks marked 'diverged', with a WARN. verify-fork-functions.sh had no notion of mode and hard-failed every fork, so kernel (30), bootloader (5) and installer (2) blocked every canonical build for drift the map itself records as accepted, with comments saying a full rebase is later work. Both verifiers now read the same source of truth. Diverged forks are reported and counted as warnings; every other fork still gates. This turned a 5-fork / 43-function hard failure into 2 real findings, which are fixed in the accompanying relibc and base commits.
252 lines
11 KiB
Bash
Executable File
252 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# verify-fork-functions.sh — Verify that local forks retain ALL upstream functions.
|
|
#
|
|
# This is the FUNCTION-LEVEL verification that prevents the silent code-loss bug
|
|
# that occurred when git merge conflict resolution dropped upstream functions
|
|
# (kfdwrite, bulk_insert_files, resize, read_with_timeout, etc.) from the kernel fork.
|
|
#
|
|
# For each fork with an upstream remote, this script:
|
|
# 1. Extracts all function definitions (fn <name>) from upstream source files
|
|
# 2. Checks that each function exists in the local fork
|
|
# 3. Reports any missing functions with file paths and line numbers
|
|
#
|
|
# This catches the class of bugs that file-level verification CANNOT detect:
|
|
# when a file is touched by RB patches, file-level diff allows ANY difference,
|
|
# including silently dropped upstream functions.
|
|
#
|
|
# Usage:
|
|
# ./local/scripts/verify-fork-functions.sh # Check all forks
|
|
# ./local/scripts/verify-fork-functions.sh kernel relibc # Check specific forks
|
|
# ./local/scripts/verify-fork-functions.sh --no-fetch # Skip network fetch
|
|
# ./local/scripts/verify-fork-functions.sh --quiet # Only print violations
|
|
#
|
|
# Exit codes:
|
|
# 0 = All upstream functions present in all forks
|
|
# 1 = One or more upstream functions are missing from forks
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
NO_FETCH=0
|
|
QUIET=0
|
|
declare -a TARGET_FORKS=()
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--no-fetch) NO_FETCH=1 ;;
|
|
--quiet) QUIET=1 ;;
|
|
-h|--help)
|
|
echo "Usage: $0 [--no-fetch] [--quiet] [fork1 fork2 ...]"
|
|
echo ""
|
|
echo "Verifies that local forks retain ALL upstream functions."
|
|
echo "Catches silent code loss from bad merge conflict resolution."
|
|
exit 0
|
|
;;
|
|
--*) echo "Unknown: $1" >&2; exit 1 ;;
|
|
*) TARGET_FORKS+=("$1") ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
cd "$PROJECT_ROOT"
|
|
|
|
# Discover forks if not specified
|
|
if [[ ${#TARGET_FORKS[@]} -eq 0 ]]; then
|
|
while IFS= read -r line; do
|
|
TARGET_FORKS+=("$line")
|
|
done < <(for d in local/sources/*/; do
|
|
[[ -d "$d/.git" ]] || continue
|
|
name=$(basename "$d")
|
|
git -C "$d" remote get-url upstream >/dev/null 2>&1 && echo "$name"
|
|
done | sort)
|
|
fi
|
|
|
|
VIOLATIONS=0
|
|
DIVERGED_WARNINGS=0
|
|
TOTAL_MISSING=0
|
|
|
|
for fork in "${TARGET_FORKS[@]}"; do
|
|
fork_dir="local/sources/${fork}"
|
|
|
|
if [[ ! -d "$fork_dir/.git" ]]; then
|
|
continue
|
|
fi
|
|
|
|
upstream_url=$(cd "$fork_dir" && git remote get-url upstream 2>/dev/null || true)
|
|
if [[ -z "$upstream_url" ]]; then
|
|
continue
|
|
fi
|
|
|
|
# Mode from the authoritative map (4th column), same source of truth as
|
|
# verify-fork-versions.sh. A fork marked 'diverged' has substantial
|
|
# post-fork work and is deliberately behind upstream master -- the map
|
|
# records for each one that a full rebase is later work. Its sibling
|
|
# verifier skips the content check for those forks with a WARN; this one
|
|
# used to hard-fail them, so kernel/bootloader/installer blocked every
|
|
# build for drift the project has already accepted. Report, do not gate.
|
|
fork_mode=$(awk -v f="$fork" '$1==f {print $4}' "$PROJECT_ROOT/local/fork-upstream-map.toml" 2>/dev/null | head -1)
|
|
|
|
# Determine upstream branch
|
|
upstream_branch="master"
|
|
if ! (cd "$fork_dir" && git rev-parse --verify upstream/master >/dev/null 2>&1); then
|
|
if (cd "$fork_dir" && git rev-parse --verify upstream/main >/dev/null 2>&1); then
|
|
upstream_branch="main"
|
|
else
|
|
if [[ "$NO_FETCH" -eq 0 ]]; then
|
|
(cd "$fork_dir" && git fetch upstream --depth=200 --quiet 2>/dev/null) || true
|
|
fi
|
|
if (cd "$fork_dir" && git rev-parse --verify upstream/master >/dev/null 2>&1); then
|
|
upstream_branch="master"
|
|
elif (cd "$fork_dir" && git rev-parse --verify upstream/main >/dev/null 2>&1); then
|
|
upstream_branch="main"
|
|
else
|
|
continue
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Fetch if needed
|
|
if [[ "$NO_FETCH" -eq 0 ]]; then
|
|
(cd "$fork_dir" && git fetch upstream --depth=200 --quiet 2>/dev/null) || true
|
|
fi
|
|
|
|
upstream_ref="upstream/$upstream_branch"
|
|
|
|
# Get list of source files that exist in BOTH upstream and our fork
|
|
upstream_files=$(cd "$fork_dir" && git ls-tree -r --name-only "$upstream_ref" 2>/dev/null | grep '\.rs$' | sort)
|
|
local_files=$(cd "$fork_dir" && find . -name '*.rs' -not -path './.git/*' -not -path './target/*' -not -path './stage/*' | sed 's|^\./||' | sort)
|
|
mapfile -t shared_files < <(comm -12 <(echo "$upstream_files") <(echo "$local_files"))
|
|
|
|
fork_missing=0
|
|
declare -a missing_details=()
|
|
|
|
for f in "${shared_files[@]}"; do
|
|
# Extract BARE function names (no visibility/async/unsafe/const
|
|
# modifiers) from the upstream version of the file.
|
|
#
|
|
# The key must be the bare name. The previous pattern kept whatever
|
|
# modifier it happened to match as part of the key ('pub foo' vs
|
|
# 'foo'), so merely changing a function's visibility in the fork —
|
|
# or upstream using 'pub(crate)', which the old pattern did not
|
|
# match at all — made an existing function look deleted. That is
|
|
# what produced report lines like "fn pub socket" and flagged
|
|
# functions the fork demonstrably still has. Visibility changes are
|
|
# not dropped code; only the name's presence is what this gate is
|
|
# meant to assert.
|
|
upstream_fns=$(cd "$fork_dir" && git show "${upstream_ref}:${f}" 2>/dev/null | \
|
|
grep -oP '\bfn\s+\w+' | sed -E 's/^fn[[:space:]]+//' | sort -u)
|
|
|
|
[[ -z "$upstream_fns" ]] && continue
|
|
|
|
# Extract function names from our version of the file
|
|
local_fns=$(cd "$fork_dir" && grep -oP '\bfn\s+\w+' "$f" 2>/dev/null | \
|
|
sed -E 's/^fn[[:space:]]+//' | sort -u)
|
|
|
|
# Load per-fork exclusion list for intentionally removed/replaced functions
|
|
exclude_file="${fork_dir}/.verify-fork-functions.exclude"
|
|
declare -A excluded=()
|
|
if [[ -f "$exclude_file" ]]; then
|
|
while IFS= read -r line; do
|
|
[[ "$line" =~ ^# ]] && continue
|
|
[[ -z "$line" ]] && continue
|
|
# Existing exclude files were written against the old
|
|
# modifier-carrying key format (and often list the same
|
|
# function twice, e.g. 'foo' and 'pub foo', to work around
|
|
# it). Normalize the function half to the bare name so both
|
|
# spellings keep matching now that keys are bare.
|
|
ex_path="${line%%:*}"
|
|
ex_fn="${line#*:}"
|
|
ex_fn="$(printf '%s' "$ex_fn" | \
|
|
sed -E 's/^(pub(\([^)]*\))?[[:space:]]+|async[[:space:]]+|unsafe[[:space:]]+|const[[:space:]]+)+//')"
|
|
excluded["${ex_path}:${ex_fn}"]=1
|
|
done < "$exclude_file"
|
|
fi
|
|
|
|
# Find functions in upstream but not in our fork
|
|
missing=$(comm -23 <(echo "$upstream_fns") <(echo "$local_fns") 2>/dev/null)
|
|
|
|
if [[ -n "$missing" ]]; then
|
|
while IFS= read -r fn; do
|
|
[[ -z "$fn" ]] && continue
|
|
# Check exclusion list: "$f:$fn"
|
|
if [[ -n "${excluded["$f:$fn"]:-}" ]]; then
|
|
[[ "$QUIET" -eq 0 ]] && missing_details+=(" $f: fn $fn → EXCLUDED (RB intentional)")
|
|
continue
|
|
fi
|
|
# Common function names (constructors, trait impls, patterns) appear
|
|
# in many unrelated files — cross-file search produces false MOVED matches.
|
|
bare_fn=$(echo "$fn" | sed -E 's/^(pub |async |unsafe )+//')
|
|
# Only fmt and eq are blanket-exempt: derivable, no semantic cost.
|
|
# drop/deref/hash/clone must go through the fork-specific exclude file.
|
|
case "$bare_fn" in
|
|
fmt|eq)
|
|
missing_details+=(" $f: fn $fn → WARN (derivable; compiler-caught, not gated)")
|
|
continue
|
|
;;
|
|
esac
|
|
case "$bare_fn" in
|
|
new|default|read|write|parse|open|close|run|init|main|\
|
|
get|set|remove|insert|delete|start|stop|send|recv|\
|
|
connect|bind|listen|accept|clone|drop|eq|hash|fmt|\
|
|
from|into|next|count|len|is_empty|clear|contains|\
|
|
daemon|allocate|index|try_from|from_psf|into_object|\
|
|
get_unique|set_plane)
|
|
# Too ambiguous — treat as truly missing (operator must decide)
|
|
;;
|
|
*)
|
|
found_elsewhere=$(cd "$fork_dir" && \
|
|
grep -rlP "(?:pub )?(?:async )?(?:unsafe )?fn ${bare_fn}\b" \
|
|
--include='*.rs' --exclude-dir='.git' --exclude-dir='target' --exclude-dir='stage' . 2>/dev/null | \
|
|
grep -v "^\./${f}$" | head -1 | sed 's|^\./||')
|
|
if [[ -n "$found_elsewhere" ]]; then
|
|
[[ "$QUIET" -eq 0 ]] && missing_details+=(" $f: fn $fn → MOVED to $found_elsewhere")
|
|
continue
|
|
fi
|
|
;;
|
|
esac
|
|
missing_details+=(" $f: fn $fn")
|
|
fork_missing=$((fork_missing + 1))
|
|
TOTAL_MISSING=$((TOTAL_MISSING + 1))
|
|
done <<< "$missing"
|
|
fi
|
|
done
|
|
|
|
if [[ "$fork_missing" -gt 0 ]]; then
|
|
if [[ "$QUIET" -eq 0 ]] || [[ "$fork_missing" -gt 0 ]]; then
|
|
echo ""
|
|
echo -e "\033[1;31mFAIL: $fork is missing $fork_missing upstream function(s):\033[0m"
|
|
for detail in "${missing_details[@]}"; do
|
|
echo "$detail"
|
|
done
|
|
echo ""
|
|
echo " These functions exist in upstream $upstream_ref but are MISSING from the local fork."
|
|
echo " This likely means a bad merge or cherry-pick dropped them silently."
|
|
echo " Fix: ./local/scripts/upgrade-forks.sh $fork"
|
|
fi
|
|
if [[ "$fork_mode" == "diverged" ]]; then
|
|
echo " NOTE: $fork is 'diverged' in local/fork-upstream-map.toml —"
|
|
echo " reported as a warning, not a build gate. Rebasing it is"
|
|
echo " tracked work; see local/docs/FORK-BUMP-PATCHING-POLICY.md."
|
|
DIVERGED_WARNINGS=$((DIVERGED_WARNINGS + 1))
|
|
else
|
|
VIOLATIONS=$((VIOLATIONS + 1))
|
|
fi
|
|
elif [[ "$QUIET" -eq 0 ]]; then
|
|
echo -e "\033[1;32mOK: $fork — all upstream functions present\033[0m"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
if [[ "$VIOLATIONS" -gt 0 ]]; then
|
|
echo -e "\033[1;31mFUNCTION VERIFICATION FAILED: $VIOLATIONS fork(s) missing $TOTAL_MISSING upstream function(s).\033[0m"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${DIVERGED_WARNINGS:-0}" -gt 0 ]]; then
|
|
echo -e "\033[1;33m$DIVERGED_WARNINGS diverged fork(s) are behind upstream (warning only).\033[0m"
|
|
fi
|
|
echo -e "\033[1;32mAll gated forks retain all upstream functions.\033[0m"
|
|
exit 0
|