RedBear-OS/local/scripts/scratch-rebuild.sh

#!/usr/bin/env bash
# scratch-rebuild.sh — build-system improvement #10
#
# Rebuild-from-scratch the subset of packages that use autotools
# (or anything that transitively depends on them) after a
# low-level source change (relibc, kernel, base, autotools
# recipes themselves). Useful when the standard "cookbook
# cascades rebuild on pkg/sources change" misses something
# (e.g. a host toolchain change, a configure-flag change, or
# a recipe's host build directory getting stale).
#
# The script:
#   1. Discovers autotools-using recipes by content (presence
#      of `aclocal`, `autoreconf`, `libtool`, or `configure` in
#      the recipe's [build].script).
#   2. Computes the transitive closure of every recipe that
#      depends on any autotools recipe (or directly uses
#      autotools itself).
#   3. For each recipe in the closure, deletes its
#      `target/<arch>/build/`, `target/<arch>/sysroot/`, and
#      `target/<arch>/stage.tmp/` (preserving `source/` so we
#      don't have to re-fetch the upstream tar).
#   4. Re-cooks each recipe in dep order using the cookbook's
#      `--jobs=N` flag (default: 4 workers) so the rebuild
#      itself runs in parallel.
#
# Per `local/docs/BUILD-SYSTEM-IMPROVEMENTS.md` #10. The full
# L-sized work (verification against real cascades, integration
# with `rebuild-cascade.sh`, the cross-host-toolchain case) is
# deferred to a separate session. This script is the
# M-sized foundation: a runnable tool that does the right
# thing in the common case.
#
# Usage:
#   ./local/scripts/scratch-rebuild.sh [--dry-run] [--jobs=N]
#       --dry-run    print what would be done; do not rm or cook
#       --jobs=N     parallel rebuild workers (default 4, max N)
#   Env:
#       REDBEAR_SCRATCH_RECIPES_DIR   override the recipe root
#       SCRATCH_LOG_DIR               where to write rebuild.log
#       SCRATCH_JOBS                  default --jobs value

set -euo pipefail

PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
RECIPES_DIR="${REDBEAR_SCRATCH_RECIPES_DIR:-$PROJECT_ROOT/local/recipes}"
LOG_DIR="${SCRATCH_LOG_DIR:-/tmp/scratch-rebuild-logs}"
JOBS="${SCRATCH_JOBS:-4}"
DRY_RUN=0

# Subcommands / flags
case "${1:-}" in
        -h|--help)
        sed -n '2,40p' "$0" | sed 's/^# \?//'
        exit 0 ;;
    --dry-run) DRY_RUN=1; shift ;;
    --jobs=*) JOBS="${1#--jobs=}"; shift ;;
esac

mkdir -p "$LOG_DIR"

cd "$PROJECT_ROOT"

# Cookbook-binary check (only relevant for non-dry-run).
if [ "$DRY_RUN" != "1" ] && [ ! -x "./target/release/repo" ]; then
    echo "./target/release/repo not built. Run: cargo build --release --bin repo" >&2
    exit 1
fi

# ---------------------------------------------------------------------------
# Step 1: discover autotools-using recipes
# ---------------------------------------------------------------------------
# A recipe "uses autotools" if its [build].script contains one of
# the canonical autotools commands. We also include any recipe
# whose name is in the AUTOTOOLS_CORE set (m4, autoconf implicit,
# libtool, automake implicit, gettext — these are needed even
# when the recipe itself doesn't run aclocal directly).
AUTOTOOLS_CORE="m4 autoconf automake libtool bison flex gettext"

shopt -s nullglob
autotools_recipes=()
for d in "$RECIPES_DIR"/*/*/; do
    [ -f "$d/recipe.toml" ] || continue
    name=$(basename "$d")
    # Skip if explicitly excluded
    case " $name " in *" m4 "*) autotools_recipes+=("$name"); continue ;; esac
    case " $name " in *" libtool "*) autotools_recipes+=("$name"); continue ;; esac
    case " $name " in *" bison "*) autotools_recipes+=("$name"); continue ;; esac
    case " $name " in *" flex "*) autotools_recipes+=("$name"); continue ;; esac
    # Content-based detection
    if grep -qE '^([[:space:]]*(aclocal|autoreconf|libtoolize|automake|autoconf|gettextize)\b|\./configure\b|./configure\b)' "$d/recipe.toml" 2>/dev/null; then
        autotools_recipes+=("$name")
    fi
done

# Deduplicate
if [ ${#autotools_recipes[@]} -gt 0 ]; then
    autotools_recipes=($(printf "%s\n" "${autotools_recipes[@]}" | sort -u))
fi

if [ ${#autotools_recipes[@]} -eq 0 ]; then
    echo "No autotools-using recipes found in $RECIPES_DIR." >&2
    exit 1
fi

echo "=== Step 1: autotools users ==="
echo "Found ${#autotools_recipes[@]} autotools-using recipes:"
printf '  %s\n' "${autotools_recipes[@]}"
echo

# ---------------------------------------------------------------------------
# Step 2: compute transitive closure (every recipe that depends
# on any autotools recipe, plus the autotools recipes themselves)
# ---------------------------------------------------------------------------
# Walk all recipes' [build].dependencies and recipe metadata.
# For each recipe, parse its [build].dependencies + [build].dev_dependencies
# and add it to the closure if any of its (transitive) deps is in
# autotools_recipes.
#
# This is intentionally a BFS over the dep graph read from the
# recipe TOML files. We do not call into the cookbook binary
# because that requires a built repo and full dep tree.
declare -A recipe_deps
for d in "$RECIPES_DIR"/*/*/; do
    [ -f "$d/recipe.toml" ] || continue
    name=$(basename "$d")
    deps=$(awk '
        /^\[build\]/ { in_build=1; next }
        /^\[/ { in_build=0 }
        in_build && /^(dependencies|dev-dependencies)/ {
            sub(/^[[:space:]]*dependencies[[:space:]]*=[[:space:]]*\[/, "")
            sub(/^[[:space:]]*dev-dependencies[[:space:]]*=[[:space:]]*\[/, "")
            gsub(/\]/, "")
            gsub(/,/, " ")
            gsub(/^[[:space:]]+|[[:space:]]+$/, "")
            gsub(/[[:space:]]+/, " ")
            print
        }
    ' "$d/recipe.toml")
    recipe_deps["$name"]="$deps"
done

closure=("${autotools_recipes[@]}")
declare -A in_closure
for r in "${autotools_recipes[@]}"; do
    in_closure["$r"]=1
done

# BFS over all recipes, adding any recipe whose deps include
# something already in the closure.
changed=1
while [ "$changed" -eq 1 ]; do
    changed=0
    for r in "${!recipe_deps[@]}"; do
        if [ -n "${in_closure[$r]:-}" ]; then
            continue
        fi
        for dep in ${recipe_deps[$r]}; do
            if [ -n "${in_closure[$dep]:-}" ]; then
                closure+=("$r")
                in_closure["$r"]=1
                changed=1
                break
            fi
        done
    done
done

echo "=== Step 2: closure ==="
echo "Closure has ${#closure[@]} recipes."
echo

# ---------------------------------------------------------------------------
# Step 3: for each recipe in the closure, clean build/ + sysroot/
# ---------------------------------------------------------------------------
# Cookbook convention (per src/cook/cook_build.rs): per-recipe
# target layout is target/<arch>/{build,sysroot,stage.tmp,...}
# We delete build/ + sysroot/ + stage.tmp/ but PRESERVE source/
# (the upstream tar was already extracted there; re-fetching is
# slow and unnecessary).
echo "=== Step 3: clean target dirs ==="
for r in "${closure[@]}"; do
    recipe_dir="$RECIPES_DIR"/*/"$r"
    if [ ! -d "$recipe_dir" ]; then
        continue
    fi
    target_dir="$recipe_dir/target"
    if [ ! -d "$target_dir" ]; then
        continue
    fi
    for arch_target in "$target_dir"/*/; do
        [ -d "$arch_target" ] || continue
        for sub in build sysroot stage.tmp; do
            if [ -d "$arch_target/$sub" ]; then
                if [ "$DRY_RUN" = "1" ]; then
                    echo "  [dry-run] would rm -rf $arch_target/$sub"
                else
                    rm -rf "$arch_target/$sub"
                    echo "  cleaned $arch_target/$sub"
                fi
            fi
        done
    done
done
echo

# ---------------------------------------------------------------------------
# Step 4: re-cook in dep order with parallel jobs
# ---------------------------------------------------------------------------
echo "=== Step 4: rebuild ==="
echo "Running: ./target/release/repo cook --jobs=$JOBS <closure>"
echo "(Cookbook walks the closure in dep-first order; --jobs runs"
echo " independent recipes in the same dep level in parallel.)"
echo

if [ "$DRY_RUN" = "1" ]; then
    echo "  [dry-run] would cook: ${closure[*]}"
else
    # The rebuild may legitimately fail if upstream deps aren't
    # all built (a fresh checkout has no cooked sysroot). The
    # user's intent is "rebuild from scratch", not "ensure
    # every dep is present". Report the failure but don't
    # exit 1 — let the user inspect the log and re-run after
    # addressing the missing dep.
    if ./target/release/repo cook --jobs="$JOBS" "${closure[@]}" 2>&1 | tee "$LOG_DIR/rebuild.log"; then
        rebuild_status="success"
    else
        rebuild_status="FAILED (see log)"
    fi
fi

echo
echo "=== Scratch rebuild complete (status: ${rebuild_status:-skipped/dry-run}) ==="
echo "Log: $LOG_DIR/rebuild.log"