#!/usr/bin/env bash # scratch-rebuild.sh — build-system improvement #10 # # Rebuild-from-scratch the subset of packages that use autotools # (or anything that transitively depends on them) after a # low-level source change (relibc, kernel, base, autotools # recipes themselves). Useful when the standard "cookbook # cascades rebuild on pkg/sources change" misses something # (e.g. a host toolchain change, a configure-flag change, or # a recipe's host build directory getting stale). # # The script: # 1. Discovers autotools-using recipes by content (presence # of `aclocal`, `autoreconf`, `libtool`, or `configure` in # the recipe's [build].script). # 2. Computes the transitive closure of every recipe that # depends on any autotools recipe (or directly uses # autotools itself). # 3. For each recipe in the closure, deletes its # `target//build/`, `target//sysroot/`, and # `target//stage.tmp/` (preserving `source/` so we # don't have to re-fetch the upstream tar). # 4. Re-cooks each recipe in dep order using the cookbook's # `--jobs=N` flag (default: 4 workers) so the rebuild # itself runs in parallel. # # Per `local/docs/BUILD-SYSTEM-IMPROVEMENTS.md` #10. The full # L-sized work (verification against real cascades, integration # with `rebuild-cascade.sh`, the cross-host-toolchain case) is # deferred to a separate session. This script is the # M-sized foundation: a runnable tool that does the right # thing in the common case. # # Usage: # ./local/scripts/scratch-rebuild.sh [--dry-run] [--jobs=N] # --dry-run print what would be done; do not rm or cook # --jobs=N parallel rebuild workers (default 4, max N) # Env: # REDBEAR_SCRATCH_RECIPES_DIR override the recipe root # SCRATCH_LOG_DIR where to write rebuild.log # SCRATCH_JOBS default --jobs value set -euo pipefail PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" RECIPES_DIR="${REDBEAR_SCRATCH_RECIPES_DIR:-$PROJECT_ROOT/local/recipes}" LOG_DIR="${SCRATCH_LOG_DIR:-/tmp/scratch-rebuild-logs}" JOBS="${SCRATCH_JOBS:-4}" DRY_RUN=0 # Subcommands / flags case "${1:-}" in -h|--help) sed -n '2,40p' "$0" | sed 's/^# \?//' exit 0 ;; --dry-run) DRY_RUN=1; shift ;; --jobs=*) JOBS="${1#--jobs=}"; shift ;; esac mkdir -p "$LOG_DIR" cd "$PROJECT_ROOT" # Cookbook-binary check (only relevant for non-dry-run). if [ "$DRY_RUN" != "1" ] && [ ! -x "./target/release/repo" ]; then echo "./target/release/repo not built. Run: cargo build --release --bin repo" >&2 exit 1 fi # --------------------------------------------------------------------------- # Step 1: discover autotools-using recipes # --------------------------------------------------------------------------- # A recipe "uses autotools" if its [build].script contains one of # the canonical autotools commands. We also include any recipe # whose name is in the AUTOTOOLS_CORE set (m4, autoconf implicit, # libtool, automake implicit, gettext — these are needed even # when the recipe itself doesn't run aclocal directly). AUTOTOOLS_CORE="m4 autoconf automake libtool bison flex gettext" shopt -s nullglob autotools_recipes=() for d in "$RECIPES_DIR"/*/*/; do [ -f "$d/recipe.toml" ] || continue name=$(basename "$d") # Skip if explicitly excluded case " $name " in *" m4 "*) autotools_recipes+=("$name"); continue ;; esac case " $name " in *" libtool "*) autotools_recipes+=("$name"); continue ;; esac case " $name " in *" bison "*) autotools_recipes+=("$name"); continue ;; esac case " $name " in *" flex "*) autotools_recipes+=("$name"); continue ;; esac # Content-based detection if grep -qE '^([[:space:]]*(aclocal|autoreconf|libtoolize|automake|autoconf|gettextize)\b|\./configure\b|./configure\b)' "$d/recipe.toml" 2>/dev/null; then autotools_recipes+=("$name") fi done # Deduplicate if [ ${#autotools_recipes[@]} -gt 0 ]; then autotools_recipes=($(printf "%s\n" "${autotools_recipes[@]}" | sort -u)) fi if [ ${#autotools_recipes[@]} -eq 0 ]; then echo "No autotools-using recipes found in $RECIPES_DIR." >&2 exit 1 fi echo "=== Step 1: autotools users ===" echo "Found ${#autotools_recipes[@]} autotools-using recipes:" printf ' %s\n' "${autotools_recipes[@]}" echo # --------------------------------------------------------------------------- # Step 2: compute transitive closure (every recipe that depends # on any autotools recipe, plus the autotools recipes themselves) # --------------------------------------------------------------------------- # Walk all recipes' [build].dependencies and recipe metadata. # For each recipe, parse its [build].dependencies + [build].dev_dependencies # and add it to the closure if any of its (transitive) deps is in # autotools_recipes. # # This is intentionally a BFS over the dep graph read from the # recipe TOML files. We do not call into the cookbook binary # because that requires a built repo and full dep tree. declare -A recipe_deps for d in "$RECIPES_DIR"/*/*/; do [ -f "$d/recipe.toml" ] || continue name=$(basename "$d") deps=$(awk ' /^\[build\]/ { in_build=1; next } /^\[/ { in_build=0 } in_build && /^(dependencies|dev-dependencies)/ { sub(/^[[:space:]]*dependencies[[:space:]]*=[[:space:]]*\[/, "") sub(/^[[:space:]]*dev-dependencies[[:space:]]*=[[:space:]]*\[/, "") gsub(/\]/, "") gsub(/,/, " ") gsub(/^[[:space:]]+|[[:space:]]+$/, "") gsub(/[[:space:]]+/, " ") print } ' "$d/recipe.toml") recipe_deps["$name"]="$deps" done closure=("${autotools_recipes[@]}") declare -A in_closure for r in "${autotools_recipes[@]}"; do in_closure["$r"]=1 done # BFS over all recipes, adding any recipe whose deps include # something already in the closure. changed=1 while [ "$changed" -eq 1 ]; do changed=0 for r in "${!recipe_deps[@]}"; do if [ -n "${in_closure[$r]:-}" ]; then continue fi for dep in ${recipe_deps[$r]}; do if [ -n "${in_closure[$dep]:-}" ]; then closure+=("$r") in_closure["$r"]=1 changed=1 break fi done done done echo "=== Step 2: closure ===" echo "Closure has ${#closure[@]} recipes." echo # --------------------------------------------------------------------------- # Step 3: for each recipe in the closure, clean build/ + sysroot/ # --------------------------------------------------------------------------- # Cookbook convention (per src/cook/cook_build.rs): per-recipe # target layout is target//{build,sysroot,stage.tmp,...} # We delete build/ + sysroot/ + stage.tmp/ but PRESERVE source/ # (the upstream tar was already extracted there; re-fetching is # slow and unnecessary). echo "=== Step 3: clean target dirs ===" for r in "${closure[@]}"; do recipe_dir="$RECIPES_DIR"/*/"$r" if [ ! -d "$recipe_dir" ]; then continue fi target_dir="$recipe_dir/target" if [ ! -d "$target_dir" ]; then continue fi for arch_target in "$target_dir"/*/; do [ -d "$arch_target" ] || continue for sub in build sysroot stage.tmp; do if [ -d "$arch_target/$sub" ]; then if [ "$DRY_RUN" = "1" ]; then echo " [dry-run] would rm -rf $arch_target/$sub" else rm -rf "$arch_target/$sub" echo " cleaned $arch_target/$sub" fi fi done done done echo # --------------------------------------------------------------------------- # Step 4: re-cook in dep order with parallel jobs # --------------------------------------------------------------------------- echo "=== Step 4: rebuild ===" echo "Running: ./target/release/repo cook --jobs=$JOBS " echo "(Cookbook walks the closure in dep-first order; --jobs runs" echo " independent recipes in the same dep level in parallel.)" echo if [ "$DRY_RUN" = "1" ]; then echo " [dry-run] would cook: ${closure[*]}" else # The rebuild may legitimately fail if upstream deps aren't # all built (a fresh checkout has no cooked sysroot). The # user's intent is "rebuild from scratch", not "ensure # every dep is present". Report the failure but don't # exit 1 — let the user inspect the log and re-run after # addressing the missing dep. if ./target/release/repo cook --jobs="$JOBS" "${closure[@]}" 2>&1 | tee "$LOG_DIR/rebuild.log"; then rebuild_status="success" else rebuild_status="FAILED (see log)" fi fi echo echo "=== Scratch rebuild complete (status: ${rebuild_status:-skipped/dry-run}) ===" echo "Log: $LOG_DIR/rebuild.log"