build: Red Bear cache system — resilient to make clean
Adds comprehensive build cache snapshot and restore for overlay OS. Problem: Upstream Redox build system is single-stream — make clean destroys cached stage.pkgar files permanently. Build can't recover without full from-scratch rebuild (2-4 hours). Solution: Red Bear cache system provides: - snapshot-cache.sh: Save all stage.pkgar to local/cache/ - restore-cache.sh: Restore from snapshot after make clean - Auto-restore: Makefile auto-restores cache before build - Essential cache: Pre-built caches for boot packages tracked in git - Cookbook fixes: Missing deps trigger rebuild instead of crash With cache restore, make clean recovery is measured in seconds, not hours. Gaps fixed in cookbook: - modified_all_btree: missing dep → UNIX_EPOCH (rebuild trigger) - sysroot install: missing dep → skip + rebuild
This commit is contained in:
@@ -7,6 +7,45 @@ include mk/depends.mk
|
|||||||
|
|
||||||
all: $(BUILD)/harddrive.img
|
all: $(BUILD)/harddrive.img
|
||||||
|
|
||||||
|
# ── Red Bear OS Build Cache ──────────────────────────────────────────────
|
||||||
|
# The upstream Redox build system loses cached stages on make clean.
|
||||||
|
# Red Bear provides snapshot/restore so the build can recover quickly.
|
||||||
|
# Essential package caches are tracked in git under local/cache/essential/.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# make cache-save Save full cache snapshot
|
||||||
|
# make cache-save-essential Save only essential (boot) packages
|
||||||
|
# make cache-restore Auto-restore latest cache before build
|
||||||
|
# make cache-verify Check cache integrity
|
||||||
|
|
||||||
|
CACHE_SCRIPT = local/scripts/snapshot-cache.sh
|
||||||
|
RESTORE_SCRIPT = local/scripts/restore-cache.sh
|
||||||
|
|
||||||
|
cache-save:
|
||||||
|
@bash $(CACHE_SCRIPT)
|
||||||
|
|
||||||
|
cache-save-essential:
|
||||||
|
@bash $(CACHE_SCRIPT) --essential
|
||||||
|
|
||||||
|
cache-restore:
|
||||||
|
@bash $(RESTORE_SCRIPT)
|
||||||
|
|
||||||
|
cache-verify:
|
||||||
|
@bash $(RESTORE_SCRIPT) --verify
|
||||||
|
|
||||||
|
cache-list:
|
||||||
|
@bash $(CACHE_SCRIPT) --list
|
||||||
|
|
||||||
|
# Auto-restore cache if available (runs before all builds)
|
||||||
|
cache-auto:
|
||||||
|
@if [ -d local/cache ] && ls local/cache/rbos-cache-* >/dev/null 2>&1; then \
|
||||||
|
echo "Red Bear: restoring build cache..."; \
|
||||||
|
bash $(RESTORE_SCRIPT); \
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Ensure cache is restored before build
|
||||||
|
$(BUILD)/harddrive.img: cache-auto
|
||||||
|
|
||||||
live:
|
live:
|
||||||
-$(FUMOUNT) $(BUILD)/filesystem/ || true
|
-$(FUMOUNT) $(BUILD)/filesystem/ || true
|
||||||
-$(FUMOUNT) /tmp/redbear_installer/ || true
|
-$(FUMOUNT) /tmp/redbear_installer/ || true
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
*
|
||||||
|
!essential/
|
||||||
|
!.gitignore
|
||||||
|
!.gitkeep
|
||||||
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
# Red Bear cache — essential packages tracked, full snapshots gitignored
|
||||||
Executable
+104
@@ -0,0 +1,104 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Red Bear OS — Build Cache Restore
|
||||||
|
# Restores recipe stage.pkgar files from a cache snapshot.
|
||||||
|
# Automatically restores the latest snapshot if no argument given.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./local/scripts/restore-cache.sh # Auto: latest snapshot
|
||||||
|
# ./local/scripts/restore-cache.sh rbos-cache-XXXXXX # Specific snapshot
|
||||||
|
# ./local/scripts/restore-cache.sh --verify # Verify cache integrity
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CACHE_DIR="${CACHE_DIR:-local/cache}"
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--verify" ]; then
|
||||||
|
echo "=== Cache Integrity Check ==="
|
||||||
|
latest=$(ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | head -1)
|
||||||
|
if [ -z "$latest" ]; then
|
||||||
|
echo "No cache snapshots found in ${CACHE_DIR}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
SNAPSHOT="$latest"
|
||||||
|
echo "Checking: $(basename "$SNAPSHOT")"
|
||||||
|
|
||||||
|
ok=0; missing=0
|
||||||
|
for pkgar in "${SNAPSHOT}"/*.pkgar; do
|
||||||
|
[ -f "$pkgar" ] || continue
|
||||||
|
pkg=$(basename "$pkgar" .pkgar)
|
||||||
|
recipe_dir=$(find recipes -maxdepth 3 -name "$pkg" -type d 2>/dev/null | head -1)
|
||||||
|
if [ -z "$recipe_dir" ]; then
|
||||||
|
echo " WARN $pkg: recipe not found"
|
||||||
|
missing=$((missing + 1))
|
||||||
|
else
|
||||||
|
ok=$((ok + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "Valid: $ok, Missing recipes: $missing"
|
||||||
|
exit $missing
|
||||||
|
fi
|
||||||
|
|
||||||
|
SNAPSHOT="${1:-}"
|
||||||
|
if [ -z "$SNAPSHOT" ]; then
|
||||||
|
SNAPSHOT=$(ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | head -1)
|
||||||
|
if [ -z "$SNAPSHOT" ]; then
|
||||||
|
echo "No cache snapshots found in ${CACHE_DIR}"
|
||||||
|
echo "Run ./local/scripts/snapshot-cache.sh first"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
SNAPSHOT="${CACHE_DIR}/${SNAPSHOT}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d "$SNAPSHOT" ]; then
|
||||||
|
echo "Snapshot not found: $SNAPSHOT"
|
||||||
|
echo "Available:"
|
||||||
|
ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | while read d; do
|
||||||
|
echo " $(basename "$d")"
|
||||||
|
done
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== Red Bear OS Cache Restore ==="
|
||||||
|
echo "Snapshot: $(basename "$SNAPSHOT")"
|
||||||
|
|
||||||
|
if [ -f "${SNAPSHOT}/manifest.toml" ]; then
|
||||||
|
echo "Manifest:"
|
||||||
|
grep -E "packages|total_size|timestamp" "${SNAPSHOT}/manifest.toml" | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
|
||||||
|
count=0
|
||||||
|
for pkgar in "${SNAPSHOT}"/*.pkgar; do
|
||||||
|
[ -f "$pkgar" ] || continue
|
||||||
|
pkg=$(basename "$pkgar" .pkgar)
|
||||||
|
recipe_dir=$(find recipes -maxdepth 3 -name "$pkg" -type d 2>/dev/null | head -1)
|
||||||
|
|
||||||
|
if [ -z "$recipe_dir" ]; then
|
||||||
|
echo " SKIP $pkg: recipe not found in tree"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
stage_dir="${recipe_dir}/target/x86_64-unknown-redox"
|
||||||
|
mkdir -p "$stage_dir"
|
||||||
|
|
||||||
|
# Only restore if stage.pkgar is missing or older
|
||||||
|
if [ ! -f "${stage_dir}/stage.pkgar" ] || [ "$pkgar" -nt "${stage_dir}/stage.pkgar" ]; then
|
||||||
|
cp "$pkgar" "${stage_dir}/stage.pkgar"
|
||||||
|
echo " RESTORED $pkg"
|
||||||
|
count=$((count + 1))
|
||||||
|
else
|
||||||
|
echo " SKIP $pkg: already up to date"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Restore deps if present
|
||||||
|
deps_file="${SNAPSHOT}/${pkg}.deps"
|
||||||
|
if [ -f "$deps_file" ] && [ ! -f "${stage_dir}/auto_deps.toml" ]; then
|
||||||
|
cp "$deps_file" "${stage_dir}/auto_deps.toml"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Restore Complete ==="
|
||||||
|
echo "Restored: $count packages"
|
||||||
|
echo ""
|
||||||
|
echo "Ready to build: make all CONFIG_NAME=redbear-full"
|
||||||
Executable
+123
@@ -0,0 +1,123 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Red Bear OS — Build Cache Snapshot
|
||||||
|
# Saves all recipe stage.pkgar files to a compressed archive in local/cache/
|
||||||
|
# This makes the build system resilient to make clean / make distclean.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./local/scripts/snapshot-cache.sh # Full snapshot
|
||||||
|
# ./local/scripts/snapshot-cache.sh --essential # Only essential packages
|
||||||
|
# ./local/scripts/snapshot-cache.sh --list # List available snapshots
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CACHE_DIR="${CACHE_DIR:-local/cache}"
|
||||||
|
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||||
|
SNAPSHOT_NAME="rbos-cache-${TIMESTAMP}"
|
||||||
|
SNAPSHOT_DIR="${CACHE_DIR}/${SNAPSHOT_NAME}"
|
||||||
|
|
||||||
|
# Essential packages for boot — small enough to commit to git
|
||||||
|
ESSENTIAL_PKGS=(
|
||||||
|
"kernel" "relibc" "base" "base-initfs" "bootloader" "init"
|
||||||
|
"ion" "installer" "redoxfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
mkdir -p "${CACHE_DIR}"
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--list" ]; then
|
||||||
|
echo "Available cache snapshots:"
|
||||||
|
ls -1t "${CACHE_DIR}"/rbos-cache-* 2>/dev/null | while read d; do
|
||||||
|
size=$(du -sh "$d" 2>/dev/null | cut -f1)
|
||||||
|
echo " $(basename "$d") $size"
|
||||||
|
done
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
MODE="${1:---full}"
|
||||||
|
|
||||||
|
echo "=== Red Bear OS Cache Snapshot ==="
|
||||||
|
echo "Mode: ${MODE}"
|
||||||
|
echo "Snapshot: ${SNAPSHOT_DIR}"
|
||||||
|
|
||||||
|
mkdir -p "${SNAPSHOT_DIR}"
|
||||||
|
|
||||||
|
count=0
|
||||||
|
total_size=0
|
||||||
|
|
||||||
|
if [ "$MODE" = "--essential" ]; then
|
||||||
|
PKGS=("${ESSENTIAL_PKGS[@]}")
|
||||||
|
else
|
||||||
|
# Find ALL recipes with stage.pkgar
|
||||||
|
PKGS=()
|
||||||
|
while IFS= read -r pkgar; do
|
||||||
|
recipe_dir=$(dirname "$(dirname "$(dirname "$pkgar")")")
|
||||||
|
pkg_name=$(basename "$recipe_dir")
|
||||||
|
PKGS+=("$pkg_name:$recipe_dir")
|
||||||
|
done < <(find recipes -name "stage.pkgar" -path "*/target/x86_64-unknown-redox/*" 2>/dev/null)
|
||||||
|
fi
|
||||||
|
|
||||||
|
for entry in "${PKGS[@]}"; do
|
||||||
|
if [ "$MODE" = "--essential" ]; then
|
||||||
|
pkg_name="$entry"
|
||||||
|
# Find the recipe directory
|
||||||
|
recipe_dir=$(find recipes -maxdepth 3 -name "$pkg_name" -type d 2>/dev/null | head -1)
|
||||||
|
if [ -z "$recipe_dir" ]; then
|
||||||
|
echo " SKIP $pkg_name: recipe not found"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
pkg_name="${entry%%:*}"
|
||||||
|
recipe_dir="${entry#*:}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
stage_dir="${recipe_dir}/target/x86_64-unknown-redox"
|
||||||
|
stage_pkgar="${stage_dir}/stage.pkgar"
|
||||||
|
|
||||||
|
if [ ! -f "$stage_pkgar" ]; then
|
||||||
|
if [ "$MODE" != "--essential" ]; then
|
||||||
|
echo " SKIP $pkg_name: no stage.pkgar"
|
||||||
|
fi
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create package dir in snapshot
|
||||||
|
pkg_snapshot="${SNAPSHOT_DIR}/${pkg_name}"
|
||||||
|
mkdir -p "$(dirname "$pkg_snapshot")"
|
||||||
|
|
||||||
|
# Copy stage.pkgar and auto_deps.toml
|
||||||
|
cp "$stage_pkgar" "${pkg_snapshot}.pkgar"
|
||||||
|
if [ -f "${stage_dir}/auto_deps.toml" ]; then
|
||||||
|
cp "${stage_dir}/auto_deps.toml" "${pkg_snapshot}.deps"
|
||||||
|
fi
|
||||||
|
|
||||||
|
size=$(stat -c%s "$stage_pkgar" 2>/dev/null || echo 0)
|
||||||
|
total_size=$((total_size + size))
|
||||||
|
count=$((count + 1))
|
||||||
|
echo " SAVED $pkg_name ($(numfmt --to=iec $size 2>/dev/null || echo ${size}B))"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Create manifest
|
||||||
|
cat > "${SNAPSHOT_DIR}/manifest.toml" << EOF
|
||||||
|
[snapshot]
|
||||||
|
name = "${SNAPSHOT_NAME}"
|
||||||
|
timestamp = "${TIMESTAMP}"
|
||||||
|
mode = "${MODE}"
|
||||||
|
packages = ${count}
|
||||||
|
total_size = ${total_size}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "=== Snapshot Complete ==="
|
||||||
|
echo "Packages: $count"
|
||||||
|
echo "Total size: $(numfmt --to=iec $total_size 2>/dev/null || echo ${total_size}B)"
|
||||||
|
echo "Location: ${SNAPSHOT_DIR}"
|
||||||
|
echo ""
|
||||||
|
echo "To restore: ./local/scripts/restore-cache.sh ${SNAPSHOT_NAME}"
|
||||||
|
|
||||||
|
# Clean up old snapshots (keep last 5)
|
||||||
|
snapshots=($(ls -1dt "${CACHE_DIR}"/rbos-cache-* 2>/dev/null))
|
||||||
|
if [ ${#snapshots[@]} -gt 5 ]; then
|
||||||
|
for old in "${snapshots[@]:5}"; do
|
||||||
|
echo "Cleaning old snapshot: $(basename "$old")"
|
||||||
|
rm -rf "$old"
|
||||||
|
done
|
||||||
|
fi
|
||||||
Reference in New Issue
Block a user