vasilito
36ddc1f2db
brush records `# Upstream: https://github.com/reubeno/brush`, but the check matched `upstream:` case-sensitively AND required the URL to end in .tar/.git, so it missed on both counts and filed a genuine fork as first-party. Now matches an explicit upstream/snapshot/origin label followed by a URL, any case. Deliberately NOT any bare URL in a comment: a stray bug-tracker link must never read as provenance, since that error direction (first-party treated as vendored) is the one that loses irreplaceable work. Remaining inaccuracy is missing DATA, not detection. libepoxy, libpciaccess, libudev, libxcvt and libdisplay-info are upstream projects whose recipes carry only `path = "source"` with no origin recorded anywhere, so nothing can tell them apart from our own code. They classify first-party, which is merely stricter. The real fix is to record their upstream in the recipe.
298 lines
13 KiB
Bash
Executable File
298 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# verify-tracked-sources.sh — integrity gate for git-tracked vendored source trees.
|
|
#
|
|
# WHY THIS EXISTS
|
|
# ---------------
|
|
# Several recipes under local/recipes/ are "vendored forks": their source/ tree
|
|
# is committed to git and carries real Red Bear work baked directly into files
|
|
# that also exist upstream (see local/docs/VERSIONING.md, "baked shim"). That
|
|
# tree is durable state, but nothing guarded it, and two failure modes have
|
|
# actually happened:
|
|
#
|
|
# 1. DELETION. An ad-hoc `rm -rf <recipe>/source` (to force a clean
|
|
# re-extract) removes thousands of tracked files. AGENTS.md already warns
|
|
# that this "has wiped tracked local/recipes/*/source/ trees in the past".
|
|
#
|
|
# 2. SILENT OVERWRITE. For a `tar` recipe, deleting source/ makes the
|
|
# cookbook re-extract PRISTINE upstream over it. Files that exist upstream
|
|
# get their committed Red Bear edits replaced with upstream content. No
|
|
# error is raised -- the package may even still build -- so the loss ships
|
|
# as a silent regression. This is how kwin lost its std::expected /
|
|
# vulkan-hpp and X11-gating fixes, and kio lost the Q_OS_REDOX guards in
|
|
# hostinfo.cpp.
|
|
#
|
|
# Neither mode is detectable by the existing gates: verify-fork-versions.sh
|
|
# covers local/sources/ (the submodule forks), and the build-redbear.sh
|
|
# dirty-source gate likewise only looks at local/sources/. The vendored recipe
|
|
# trees carry equally durable work and had no equivalent check.
|
|
#
|
|
# WHAT IT CHECKS
|
|
# --------------
|
|
# Recipe seds legitimately rewrite tracked source files on every build (that is
|
|
# the current in-tree build model), so "modified" alone cannot mean "damaged".
|
|
# The gate therefore separates three cases:
|
|
#
|
|
# DELETED tracked file under */source/ -> ERROR, always. A build never
|
|
# legitimately deletes tracked
|
|
# source.
|
|
# MODIFIED, not in the baseline -> ERROR. Something changed a
|
|
# tracked source file that no
|
|
# recorded recipe sed accounts
|
|
# for -- the overwrite signature.
|
|
# MODIFIED, listed in the baseline -> known churn. Reported as a
|
|
# durability WARNING (this is
|
|
# uncommitted work living in a
|
|
# durable tree); escalate to an
|
|
# error with
|
|
# REDBEAR_STRICT_TRACKED_SOURCES=1.
|
|
#
|
|
# The baseline (local/tracked-source-baseline.txt) is itself tracked, so the set
|
|
# of "expected to be dirty" files is reviewable in git rather than invisible.
|
|
#
|
|
# USAGE
|
|
# verify-tracked-sources.sh [--quiet] # check (preflight mode)
|
|
# verify-tracked-sources.sh --update # regenerate the baseline
|
|
# verify-tracked-sources.sh --list # show current dirty tracked sources
|
|
#
|
|
# Bypass: REDBEAR_SKIP_TRACKED_SOURCE_CHECK=1 (emergency only).
|
|
# Escalate baseline churn to an error: REDBEAR_STRICT_TRACKED_SOURCES=1.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
BASELINE="$ROOT/local/tracked-source-baseline.txt"
|
|
|
|
QUIET=0
|
|
MODE=check
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--quiet) QUIET=1 ;;
|
|
--update) MODE=update ;;
|
|
--list) MODE=list ;;
|
|
-h|--help)
|
|
sed -n '2,60p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
exit 0 ;;
|
|
*) echo "verify-tracked-sources.sh: unknown argument '$arg'" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
say() { [ "$QUIET" -eq 1 ] || echo "$@"; }
|
|
|
|
cd "$ROOT" || exit 2
|
|
|
|
# Collect tracked files under any local/recipes/**/source/ that git reports as
|
|
# changed. -z keeps paths with spaces intact (several KDE avatar assets have
|
|
# them, which is exactly where a naive `xargs` split goes wrong).
|
|
collect() { # $1 = status filter letter (M or D)
|
|
local want="$1"
|
|
git status --porcelain -z -- local/recipes 2>/dev/null \
|
|
| while IFS= read -r -d '' entry; do
|
|
local st="${entry:0:2}" path="${entry:3}"
|
|
case "$st" in
|
|
*"$want"*) [[ "$path" == */source/* ]] && printf '%s\n' "$path" ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
mapfile -t DELETED < <(collect D | LC_ALL=C sort)
|
|
mapfile -t MODIFIED < <(collect M | LC_ALL=C sort)
|
|
|
|
if [ "$MODE" = list ]; then
|
|
printf '%s\n' "${MODIFIED[@]}" | sed '/^$/d'
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$MODE" = update ]; then
|
|
{
|
|
echo "# tracked-source-baseline.txt"
|
|
echo "#"
|
|
echo "# Tracked files under local/recipes/*/source/ that are EXPECTED to differ"
|
|
echo "# from HEAD during a normal build -- almost all of it recipe-sed churn."
|
|
echo "# Consumed by local/scripts/verify-tracked-sources.sh."
|
|
echo "#"
|
|
echo "# A file listed here is uncommitted work sitting in a DURABLE tree. That is"
|
|
echo "# tolerated, not endorsed: commit it, or convert the change into a recipe sed"
|
|
echo "# or a patch, so a clean re-extract cannot lose it."
|
|
echo "#"
|
|
echo "# A line prefixed 'D ' records an INTENTIONAL deletion of a tracked source"
|
|
echo "# file (e.g. removing a vendored stub). Without an entry, any deletion is"
|
|
echo "# treated as damage."
|
|
echo "#"
|
|
echo "# Regenerate with: ./local/scripts/verify-tracked-sources.sh --update"
|
|
printf '%s\n' "${MODIFIED[@]}" | sed '/^$/d'
|
|
printf 'D %s\n' "${DELETED[@]}" | sed '/^D $/d'
|
|
} > "$BASELINE"
|
|
echo ">>> baseline updated: $(printf '%s\n' "${MODIFIED[@]}" | sed '/^$/d' | wc -l) modified," \
|
|
"$(printf '%s\n' "${DELETED[@]}" | sed '/^$/d' | wc -l) deleted -> $BASELINE"
|
|
exit 0
|
|
fi
|
|
|
|
# ---- check mode ----------------------------------------------------------
|
|
rc=0
|
|
|
|
if [ "${REDBEAR_SKIP_TRACKED_SOURCE_CHECK:-0}" = "1" ]; then
|
|
echo ">>> WARNING: tracked-source integrity check bypassed via REDBEAR_SKIP_TRACKED_SOURCE_CHECK=1." >&2
|
|
exit 0
|
|
fi
|
|
|
|
# Load the baseline first: it records both expected modifications (plain lines)
|
|
# and intentional deletions ("D <path>").
|
|
declare -A known=()
|
|
declare -A known_del=()
|
|
if [ -f "$BASELINE" ]; then
|
|
while IFS= read -r line; do
|
|
[[ -z "$line" || "$line" == \#* ]] && continue
|
|
if [[ "$line" == "D "* ]]; then
|
|
known_del["${line:2}"]=1
|
|
else
|
|
known["$line"]=1
|
|
fi
|
|
done < "$BASELINE"
|
|
fi
|
|
|
|
# 1. Deletions are damage unless explicitly recorded as intentional.
|
|
unexpected_del=()
|
|
for f in "${DELETED[@]}"; do
|
|
[ -z "$f" ] && continue
|
|
[ -n "${known_del[$f]:-}" ] || unexpected_del+=("$f")
|
|
done
|
|
DELETED=("${unexpected_del[@]}")
|
|
|
|
if [ "${#DELETED[@]}" -gt 0 ] && [ -n "${DELETED[0]:-}" ]; then
|
|
echo "" >&2
|
|
echo ">>> ERROR: ${#DELETED[@]} git-tracked file(s) under a vendored source/ tree are DELETED." >&2
|
|
echo " A build never legitimately deletes tracked source. This is the signature of an" >&2
|
|
echo " ad-hoc 'rm -rf <recipe>/source'." >&2
|
|
printf ' %s\n' "${DELETED[@]:0:15}" >&2
|
|
[ "${#DELETED[@]}" -gt 15 ] && echo " ... and $(( ${#DELETED[@]} - 15 )) more" >&2
|
|
echo "" >&2
|
|
echo " Restore with: git checkout -- local/recipes/<recipe>/source" >&2
|
|
echo " To force a clean re-extract of a tracked tree, restore it from git afterwards;" >&2
|
|
echo " never delete it." >&2
|
|
rc=1
|
|
fi
|
|
|
|
# 2. Modifications the baseline does not account for.
|
|
unexpected=()
|
|
for f in "${MODIFIED[@]}"; do
|
|
[ -z "$f" ] && continue
|
|
[ -n "${known[$f]:-}" ] || unexpected+=("$f")
|
|
done
|
|
|
|
if [ "${#unexpected[@]}" -gt 0 ]; then
|
|
echo "" >&2
|
|
echo ">>> ERROR: ${#unexpected[@]} tracked source file(s) changed with no baseline entry." >&2
|
|
echo " Either a recipe gained a new sed, or committed Red Bear work was overwritten" >&2
|
|
echo " (e.g. a pristine tarball re-extract replacing baked-in fixes)." >&2
|
|
printf ' %s\n' "${unexpected[@]:0:15}" >&2
|
|
[ "${#unexpected[@]}" -gt 15 ] && echo " ... and $(( ${#unexpected[@]} - 15 )) more" >&2
|
|
echo "" >&2
|
|
echo " Inspect: git diff -- <path>" >&2
|
|
echo " If the change is LOST WORK: git checkout -- <path>" >&2
|
|
echo " If the change is EXPECTED: ./local/scripts/verify-tracked-sources.sh --update" >&2
|
|
rc=1
|
|
fi
|
|
|
|
# 3. Baseline churn: uncommitted work in a durable tree. Visible, not fatal by
|
|
# default -- this is the local/recipes/ counterpart of the local/sources/
|
|
# dirty gate in build-redbear.sh.
|
|
baseline_dirty=0
|
|
for f in "${MODIFIED[@]}"; do
|
|
[ -z "$f" ] && continue
|
|
[ -n "${known[$f]:-}" ] && baseline_dirty=$(( baseline_dirty + 1 ))
|
|
done
|
|
|
|
# 3a. FIRST-PARTY source is always fatal, never a note.
|
|
#
|
|
# redbear-* recipes are not vendored upstream code -- they are Red Bear's own
|
|
# programs and exist NOWHERE ELSE. A vendored tree can be restored from its
|
|
# upstream tarball or git remote; first-party source cannot. If a recipe sed or
|
|
# an `rm` damages it and the damage is committed, the work is simply gone.
|
|
#
|
|
# This is not hypothetical: seven redbear-* recipes rewrite their own source
|
|
# during the build (redbear-greeter, -btusb, -btctl, -ime, -dnsd,
|
|
# -accessibility, -keymapd), and all seven are exempt from out-of-tree staging
|
|
# because their cargo manifests carry path dependencies that escape the source
|
|
# tree. They are therefore the LEAST protected and the MOST irreplaceable code
|
|
# in the repository, so uncommitted drift in them stops the build outright
|
|
# rather than printing a note that scrolls past.
|
|
# FIRST-PARTY detection, DERIVED -- deliberately not a hardcoded list.
|
|
#
|
|
# A name list was the first attempt (redbear-*, then cub, then tlc) and it was
|
|
# wrong in principle: protection that depends on someone remembering to edit
|
|
# this file is not protection. Every new internal program would start out
|
|
# unguarded, and the omission is invisible until the code is already lost.
|
|
# `tlc` proved the point -- it is first-party, it is exempt from out-of-tree
|
|
# staging, and nothing but this gate protects it, yet it matched no pattern.
|
|
#
|
|
# The real property is structural: OUR code has no upstream to restore from.
|
|
# So a recipe is first-party when its recipe.toml records no fetchable origin
|
|
# -- no `tar =`, no `git =`, and no upstream URL. That currently identifies ~96
|
|
# recipes, against the 3 the list covered.
|
|
#
|
|
# The heuristic ERRS TOWARD first-party on purpose. A vendored recipe that
|
|
# records its origin only in prose gets classified as ours, which merely makes
|
|
# drift fatal instead of a warning -- conservative. The opposite mistake, a
|
|
# first-party recipe treated as vendored, is the one that loses irreplaceable
|
|
# work, so the default must fail in this direction.
|
|
redbear_is_firstparty() {
|
|
local path="$1" dir
|
|
dir="$(dirname "$path")"
|
|
# Walk up to the recipe root (the directory holding recipe.toml).
|
|
while [ "$dir" != "." ] && [ "$dir" != "/" ] && [ ! -f "$dir/recipe.toml" ]; do
|
|
dir="$(dirname "$dir")"
|
|
done
|
|
[ -f "$dir/recipe.toml" ] || return 1
|
|
# Fetchable origin keys.
|
|
grep -qE '^[[:space:]]*(tar|git)[[:space:]]*=' "$dir/recipe.toml" && return 1
|
|
# Provenance recorded in prose by a vendored fork. Case-INSENSITIVE and not
|
|
# restricted to .tar/.git URLs: brush writes
|
|
# # Upstream: https://github.com/reubeno/brush
|
|
# which a case-sensitive `upstream:` + \.(tar|git) match missed on both
|
|
# counts, misfiling a genuine fork as first-party.
|
|
#
|
|
# Matched narrowly -- an explicit `upstream:`/`snapshot:` label followed by a
|
|
# URL -- NOT any bare URL in a comment. A stray bug-tracker link must never
|
|
# be read as provenance, because that error direction (first-party treated
|
|
# as vendored) is the one that loses irreplaceable work.
|
|
grep -qiE '^[[:space:]]*#?[[:space:]]*(upstream|snapshot|origin)[[:space:]]*:[[:space:]]*(https?://|[A-Za-z0-9_.-]+/)' \
|
|
"$dir/recipe.toml" && return 1
|
|
return 0
|
|
}
|
|
|
|
firstparty_dirty=0
|
|
for f in "${MODIFIED[@]}" "${DELETED[@]}"; do
|
|
[ -z "$f" ] && continue
|
|
redbear_is_firstparty "$f" && firstparty_dirty=$(( firstparty_dirty + 1 ))
|
|
done
|
|
|
|
if [ "$firstparty_dirty" -gt 0 ]; then
|
|
echo ">>> ERROR: $firstparty_dirty uncommitted change(s) in FIRST-PARTY source (redbear-*, cub, tlc)." >&2
|
|
echo " This code exists nowhere but this project -- there is no upstream to restore" >&2
|
|
echo " from. Commit it, or revert it with: git checkout -- <path>" >&2
|
|
for f in "${MODIFIED[@]}" "${DELETED[@]}"; do
|
|
redbear_is_firstparty "$f" && echo " $f" >&2
|
|
done
|
|
echo " Override (accepts the risk): REDBEAR_ALLOW_DIRTY_FIRSTPARTY=1" >&2
|
|
[ "${REDBEAR_ALLOW_DIRTY_FIRSTPARTY:-0}" = "1" ] || rc=1
|
|
fi
|
|
|
|
if [ "$baseline_dirty" -gt 0 ]; then
|
|
if [ "${REDBEAR_STRICT_TRACKED_SOURCES:-0}" = "1" ]; then
|
|
echo ">>> ERROR: $baseline_dirty uncommitted change(s) in tracked source trees" \
|
|
"(REDBEAR_STRICT_TRACKED_SOURCES=1)." >&2
|
|
rc=1
|
|
else
|
|
say ">>> Preflight note: $baseline_dirty uncommitted change(s) in tracked vendored" \
|
|
"source trees (baseline-known)."
|
|
say " These live in a DURABLE tree but are not committed; a clean re-extract would" \
|
|
"lose them. Commit them, or move them into a recipe sed / patch."
|
|
fi
|
|
fi
|
|
|
|
if [ "$rc" -eq 0 ]; then
|
|
say ">>> Preflight: tracked vendored source trees intact (no deletions, no unexplained edits)."
|
|
fi
|
|
exit "$rc"
|