build: Red Bear cache system — resilient to make clean
Adds comprehensive build cache snapshot and restore for overlay OS. Problem: Upstream Redox build system is single-stream — make clean destroys cached stage.pkgar files permanently. Build can't recover without full from-scratch rebuild (2-4 hours). Solution: Red Bear cache system provides: - snapshot-cache.sh: Save all stage.pkgar to local/cache/ - restore-cache.sh: Restore from snapshot after make clean - Auto-restore: Makefile auto-restores cache before build - Essential cache: Pre-built caches for boot packages tracked in git - Cookbook fixes: Missing deps trigger rebuild instead of crash With cache restore, make clean recovery is measured in seconds, not hours. Gaps fixed in cookbook: - modified_all_btree: missing dep → UNIX_EPOCH (rebuild trigger) - sysroot install: missing dep → skip + rebuild
This commit is contained in:
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# Red Bear OS — Build Cache Restore
|
||||
# Restores recipe stage.pkgar files from a cache snapshot.
|
||||
# Automatically restores the latest snapshot if no argument given.
|
||||
#
|
||||
# Usage:
|
||||
# ./local/scripts/restore-cache.sh # Auto: latest snapshot
|
||||
# ./local/scripts/restore-cache.sh rbos-cache-XXXXXX # Specific snapshot
|
||||
# ./local/scripts/restore-cache.sh --verify # Verify cache integrity
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CACHE_DIR="${CACHE_DIR:-local/cache}"
|
||||
|
||||
if [ "${1:-}" = "--verify" ]; then
|
||||
echo "=== Cache Integrity Check ==="
|
||||
latest=$(ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | head -1)
|
||||
if [ -z "$latest" ]; then
|
||||
echo "No cache snapshots found in ${CACHE_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
SNAPSHOT="$latest"
|
||||
echo "Checking: $(basename "$SNAPSHOT")"
|
||||
|
||||
ok=0; missing=0
|
||||
for pkgar in "${SNAPSHOT}"/*.pkgar; do
|
||||
[ -f "$pkgar" ] || continue
|
||||
pkg=$(basename "$pkgar" .pkgar)
|
||||
recipe_dir=$(find recipes -maxdepth 3 -name "$pkg" -type d 2>/dev/null | head -1)
|
||||
if [ -z "$recipe_dir" ]; then
|
||||
echo " WARN $pkg: recipe not found"
|
||||
missing=$((missing + 1))
|
||||
else
|
||||
ok=$((ok + 1))
|
||||
fi
|
||||
done
|
||||
echo "Valid: $ok, Missing recipes: $missing"
|
||||
exit $missing
|
||||
fi
|
||||
|
||||
SNAPSHOT="${1:-}"
|
||||
if [ -z "$SNAPSHOT" ]; then
|
||||
SNAPSHOT=$(ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | head -1)
|
||||
if [ -z "$SNAPSHOT" ]; then
|
||||
echo "No cache snapshots found in ${CACHE_DIR}"
|
||||
echo "Run ./local/scripts/snapshot-cache.sh first"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
SNAPSHOT="${CACHE_DIR}/${SNAPSHOT}"
|
||||
fi
|
||||
|
||||
if [ ! -d "$SNAPSHOT" ]; then
|
||||
echo "Snapshot not found: $SNAPSHOT"
|
||||
echo "Available:"
|
||||
ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | while read d; do
|
||||
echo " $(basename "$d")"
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Red Bear OS Cache Restore ==="
|
||||
echo "Snapshot: $(basename "$SNAPSHOT")"
|
||||
|
||||
if [ -f "${SNAPSHOT}/manifest.toml" ]; then
|
||||
echo "Manifest:"
|
||||
grep -E "packages|total_size|timestamp" "${SNAPSHOT}/manifest.toml" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
count=0
|
||||
for pkgar in "${SNAPSHOT}"/*.pkgar; do
|
||||
[ -f "$pkgar" ] || continue
|
||||
pkg=$(basename "$pkgar" .pkgar)
|
||||
recipe_dir=$(find recipes -maxdepth 3 -name "$pkg" -type d 2>/dev/null | head -1)
|
||||
|
||||
if [ -z "$recipe_dir" ]; then
|
||||
echo " SKIP $pkg: recipe not found in tree"
|
||||
continue
|
||||
fi
|
||||
|
||||
stage_dir="${recipe_dir}/target/x86_64-unknown-redox"
|
||||
mkdir -p "$stage_dir"
|
||||
|
||||
# Only restore if stage.pkgar is missing or older
|
||||
if [ ! -f "${stage_dir}/stage.pkgar" ] || [ "$pkgar" -nt "${stage_dir}/stage.pkgar" ]; then
|
||||
cp "$pkgar" "${stage_dir}/stage.pkgar"
|
||||
echo " RESTORED $pkg"
|
||||
count=$((count + 1))
|
||||
else
|
||||
echo " SKIP $pkg: already up to date"
|
||||
fi
|
||||
|
||||
# Restore deps if present
|
||||
deps_file="${SNAPSHOT}/${pkg}.deps"
|
||||
if [ -f "$deps_file" ] && [ ! -f "${stage_dir}/auto_deps.toml" ]; then
|
||||
cp "$deps_file" "${stage_dir}/auto_deps.toml"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Restore Complete ==="
|
||||
echo "Restored: $count packages"
|
||||
echo ""
|
||||
echo "Ready to build: make all CONFIG_NAME=redbear-full"
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
# Red Bear OS — Build Cache Snapshot
|
||||
# Saves all recipe stage.pkgar files to a compressed archive in local/cache/
|
||||
# This makes the build system resilient to make clean / make distclean.
|
||||
#
|
||||
# Usage:
|
||||
# ./local/scripts/snapshot-cache.sh # Full snapshot
|
||||
# ./local/scripts/snapshot-cache.sh --essential # Only essential packages
|
||||
# ./local/scripts/snapshot-cache.sh --list # List available snapshots
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CACHE_DIR="${CACHE_DIR:-local/cache}"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
SNAPSHOT_NAME="rbos-cache-${TIMESTAMP}"
|
||||
SNAPSHOT_DIR="${CACHE_DIR}/${SNAPSHOT_NAME}"
|
||||
|
||||
# Essential packages for boot — small enough to commit to git
|
||||
ESSENTIAL_PKGS=(
|
||||
"kernel" "relibc" "base" "base-initfs" "bootloader" "init"
|
||||
"ion" "installer" "redoxfs"
|
||||
)
|
||||
|
||||
mkdir -p "${CACHE_DIR}"
|
||||
|
||||
if [ "${1:-}" = "--list" ]; then
|
||||
echo "Available cache snapshots:"
|
||||
ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | while read d; do
|
||||
size=$(du -sh "$d" 2>/dev/null | cut -f1)
|
||||
echo " $(basename "$d") $size"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
MODE="${1:---full}"
|
||||
|
||||
echo "=== Red Bear OS Cache Snapshot ==="
|
||||
echo "Mode: ${MODE}"
|
||||
echo "Snapshot: ${SNAPSHOT_DIR}"
|
||||
|
||||
mkdir -p "${SNAPSHOT_DIR}"
|
||||
|
||||
count=0
|
||||
total_size=0
|
||||
|
||||
if [ "$MODE" = "--essential" ]; then
|
||||
PKGS=("${ESSENTIAL_PKGS[@]}")
|
||||
else
|
||||
# Find ALL recipes with stage.pkgar
|
||||
PKGS=()
|
||||
while IFS= read -r pkgar; do
|
||||
recipe_dir=$(dirname "$(dirname "$(dirname "$pkgar")")")
|
||||
pkg_name=$(basename "$recipe_dir")
|
||||
PKGS+=("$pkg_name:$recipe_dir")
|
||||
done < <(find recipes -name "stage.pkgar" -path "*/target/x86_64-unknown-redox/*" 2>/dev/null)
|
||||
fi
|
||||
|
||||
for entry in "${PKGS[@]}"; do
|
||||
if [ "$MODE" = "--essential" ]; then
|
||||
pkg_name="$entry"
|
||||
# Find the recipe directory
|
||||
recipe_dir=$(find recipes -maxdepth 3 -name "$pkg_name" -type d 2>/dev/null | head -1)
|
||||
if [ -z "$recipe_dir" ]; then
|
||||
echo " SKIP $pkg_name: recipe not found"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
pkg_name="${entry%%:*}"
|
||||
recipe_dir="${entry#*:}"
|
||||
fi
|
||||
|
||||
stage_dir="${recipe_dir}/target/x86_64-unknown-redox"
|
||||
stage_pkgar="${stage_dir}/stage.pkgar"
|
||||
|
||||
if [ ! -f "$stage_pkgar" ]; then
|
||||
if [ "$MODE" != "--essential" ]; then
|
||||
echo " SKIP $pkg_name: no stage.pkgar"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# Create package dir in snapshot
|
||||
pkg_snapshot="${SNAPSHOT_DIR}/${pkg_name}"
|
||||
mkdir -p "$(dirname "$pkg_snapshot")"
|
||||
|
||||
# Copy stage.pkgar and auto_deps.toml
|
||||
cp "$stage_pkgar" "${pkg_snapshot}.pkgar"
|
||||
if [ -f "${stage_dir}/auto_deps.toml" ]; then
|
||||
cp "${stage_dir}/auto_deps.toml" "${pkg_snapshot}.deps"
|
||||
fi
|
||||
|
||||
size=$(stat -c%s "$stage_pkgar" 2>/dev/null || echo 0)
|
||||
total_size=$((total_size + size))
|
||||
count=$((count + 1))
|
||||
echo " SAVED $pkg_name ($(numfmt --to=iec $size 2>/dev/null || echo ${size}B))"
|
||||
done
|
||||
|
||||
# Create manifest
|
||||
cat > "${SNAPSHOT_DIR}/manifest.toml" << EOF
|
||||
[snapshot]
|
||||
name = "${SNAPSHOT_NAME}"
|
||||
timestamp = "${TIMESTAMP}"
|
||||
mode = "${MODE}"
|
||||
packages = ${count}
|
||||
total_size = ${total_size}
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "=== Snapshot Complete ==="
|
||||
echo "Packages: $count"
|
||||
echo "Total size: $(numfmt --to=iec $total_size 2>/dev/null || echo ${total_size}B)"
|
||||
echo "Location: ${SNAPSHOT_DIR}"
|
||||
echo ""
|
||||
echo "To restore: ./local/scripts/restore-cache.sh ${SNAPSHOT_NAME}"
|
||||
|
||||
# Clean up old snapshots (keep last 5)
|
||||
snapshots=($(ls -1dt "${CACHE_DIR}"/rbos-cache-* 2>/dev/null))
|
||||
if [ ${#snapshots[@]} -gt 5 ]; then
|
||||
for old in "${snapshots[@]:5}"; do
|
||||
echo "Cleaning old snapshot: $(basename "$old")"
|
||||
rm -rf "$old"
|
||||
done
|
||||
fi
|
||||
Reference in New Issue
Block a user