diff --git a/local/docs/UPSTREAM-SYNC-PROCEDURE.md b/local/docs/UPSTREAM-SYNC-PROCEDURE.md new file mode 100644 index 0000000000..fb31a4e3ae --- /dev/null +++ b/local/docs/UPSTREAM-SYNC-PROCEDURE.md @@ -0,0 +1,375 @@ +# Upstream Sync Procedure for Local Forks + +**Created:** 2026-06-18 +**Scope:** relibc, kernel, bootloader, installer, redoxfs, userutils, base +**Baseline:** Red Bear OS 0.1.0 (Redox snapshot) + +## Purpose + +Red Bear OS maintains local forks of core Redox components. Upstream Redox is +actively developed and regularly merges refactoring, bug fixes, and new +features. This document defines the **standard procedure** for updating each +local fork to the latest upstream HEAD and re-applying Red Bear-specific +changes on top. + +This is a **repeatable, per-package operation**. Every base system package +must go through this process periodically to avoid accumulating fork drift. + +## When to Sync + +- A Rust nightly bump causes compile errors in the fork (unstable API changes). +- Upstream adds features we were manually patching (eliminates a Red Bear patch). +- Accumulated fork drift makes rebasing individual patches impractical. +- Before a new Red Bear OS release. +- When a bug in the fork is already fixed upstream. + +## Prerequisites + +```bash +# Ensure upstream remote exists in the fork repo +cd local/sources/ +git remote -v +# Should show: upstream -> https://gitlab.redox-os.org/redox-os/.git + +# If missing: +git remote add upstream https://gitlab.redox-os.org/redox-os/.git +``` + +## Procedure + +### Step 1: Backup Current State + +**Never skip this.** The backup branch is your rollback point. + +```bash +cd local/sources/ +git fetch upstream --quiet + +# Create timestamped backup branch +git branch backup-master-pre-upstream-update-$(date +%Y%m%d) master + +# Verify +git log --oneline -1 backup-master-pre-upstream-update-* +``` + +### Step 2: Analyze Divergence + +Understand what you have vs what upstream has: + +```bash +# Latest upstream HEAD +git log --oneline upstream/master -3 + +# Divergence point (merge-base) +git merge-base master upstream/master + +# Your commits on top of upstream +git log --oneline upstream/master..master + +# Files changed (scope assessment) +git diff upstream/master..master --stat | tail -5 +``` + +Categorize each commit: +- **Feature port** — ported FROM upstream (e.g., posix_spawn). Upstream now + has it natively → **DROP**, use upstream's version. +- **Red Bear feature** — unique to Red Bear (e.g., eventfd, signalfd, timerfd, + __fseterr, getprogname). Not in upstream → **KEEP**. +- **Compatibility shim** — gnulib/glibc/musl compat (e.g., rpl_malloc, + __freadahead). → **KEEP** unless upstream added equivalent. +- **Toolchain fix** — adapts to Rust nightly changes (e.g., unsafe blocks, + VaList API). → **CHECK** if still needed against current upstream. +- **Revert/cleanup** — noise from previous iterations. → **DROP**. + +### Step 3: Create Sync Branch + +```bash +# Branch from latest upstream +git checkout -b redbear-upstream-sync-$(date +%Y%m%d) upstream/master +``` + +### Step 4: Squash-Merge Your Changes + +```bash +# Squash-merge brings ALL your changes as staged, conflict-marked if needed +git merge --squash master +``` + +If untracked files block the merge (files that exist in your fork but not +upstream, left over from failed attempts): + +```bash +git merge --abort +git reset --hard upstream/master +git clean -fd +git merge --squash master +``` + +### Step 5: Resolve Conflicts + +This is the most time-consuming step. For each conflicted file: + +```bash +# List conflicts +git diff --name-only --diff-filter=U +``` + +**Resolution strategy per file:** + +| Conflict Type | Resolution | +|---|---| +| Add/Add (both added same file) | Check which is canonical. If upstream now has it (e.g., spawn), take upstream: `git checkout --ours ` | +| Feature you ported from upstream | Take upstream: `git checkout --ours ` | +| Red Bear unique feature | Take ours: `git checkout --theirs ` | +| Both sides modified same code | **Manual merge** — keep both sets of changes | +| Cargo.toml/Cargo.lock | Resolve Cargo.toml manually, regenerate Cargo.lock with `cargo update` | + +```bash +# Take upstream (current branch = upstream): +git checkout --ours && git add + +# Take ours (the merged branch = our fork): +git checkout --theirs && git add + +# Manual resolution: +# Edit file, remove conflict markers, keep both sides' relevant changes +git add +``` + +**Conflict count expectation:** +- relibc: 5-15 conflict files (heavy upstream refactoring) +- kernel: 10-30 conflict files +- base: 5-10 conflict files +- Others: 2-5 conflict files + +### Step 6: Commit the Squash + +```bash +git commit -m "sync: upstream to $(git rev-parse --short upstream/master) + +Re-applied Red Bear-specific changes on top of latest upstream. +Dropped commits for features now in upstream. +Backup: backup-master-pre-upstream-update-$(date +%Y%m%d)" +``` + +### Step 7: Compile Test + +```bash +# Build the component via the cookbook +cd /home/kellito/Builds/RedBear-OS +rm -rf recipes/core//source recipes/core//target +export CI=1 COOKBOOK_OFFLINE=false REDBEAR_ALLOW_PROTECTED_FETCH=1 +./target/release/repo cook recipes/core/ +``` + +Fix any compile errors. These are usually: +- **Unsafe block changes** (Rust 2024 edition) — wrap in `unsafe { }`. +- **API renames** — upstream renamed a function/type. +- **Module path changes** — upstream reorganized headers. +- **Trait signature changes** — upstream changed a trait method signature. + +### Step 8: Verify Functionality + +For relibc specifically: +```bash +# Check critical symbols are present +nm recipes/core/relibc/target/x86_64-unknown-redox/stage/x86_64-unknown-redox/lib/libc.a | \ + grep -E "__fseterr|eventfd|signalfd|timerfd|open_memstream" + +# Check static libs exist +ls -la recipes/core/relibc/target/x86_64-unknown-redox/stage/x86_64-unknown-redox/lib/*.a +``` + +### Step 9: Push to Gitea + +```bash +cd local/sources/ +# Switch sync branch to master (or merge sync into master) +git checkout master +git reset --hard redbear-upstream-sync-$(date +%Y%m%d) +git push gitea master --force-with-lease +``` + +### Step 10: Update Recipe + +If the recipe uses `git + patches` (not `path`): +```bash +# Update the pinned rev in recipe.toml to match new HEAD +# Remove patches that are now in upstream +``` + +If the recipe uses `path` (local fork): +```bash +# No recipe changes needed — the fork IS the source +``` + +### Step 11: Rebuild Dependent Components + +After updating a base component, everything that depends on it must rebuild: + +| Component Updated | Must Rebuild | +|---|---| +| relibc | prefix, then ALL C recipes (bison, flex, m4, etc.) | +| kernel | base, drivers, all recipes | +| base | everything in the image | +| bootloader | disk image creation | +| installer | disk image creation | +| redoxfs | disk image creation | +| userutils | base, login/session recipes | + +### Step 12: Full Image Build + +```bash +cd /home/kellito/Builds/RedBear-OS +pkill -9 -f "build-redbear\|repo cook" +rm -rf recipes/core//source recipes/core//target + +export CI=1 REDBEAR_ALLOW_PROTECTED_FETCH=1 +setsid nohup env CI=1 REDBEAR_ALLOW_PROTECTED_FETCH=1 \ + ./local/scripts/build-redbear.sh --upstream redbear-mini \ + > /tmp/build-V.log 2>&1 < /dev/null & +``` + +## Package-Specific Notes + +### relibc + +- **Upstream repo:** `https://gitlab.redox-os.org/redox-os/relibc.git` +- **Fork location:** `local/sources/relibc/` +- **Recipe:** `recipes/core/relibc/recipe.toml` +- **Red Bear features to preserve:** + - `eventfd` (eventfd_create, eventfd_read, eventfd_write) + - `signalfd` (signalfd_create/signalfd) + - `timerfd` (timerfd_create, timerfd_settime, timerfd_gettime) + - `__fseterr`, `__freadahead` (gnulib/m4/bison compatibility) + - `getprogname` (via `sys:exe` scheme) + - `getloadavg` (via `/proc/loadavg`) + - `clock_settime` (proper implementation) + - `getifaddrs`, `if_nameindex` (networking) + - `getrlimit`, `getrusage` (resource limits) + - `waitid` (POSIX waitid) + - `dup3`, `F_DUPFD_CLOEXEC` fallback + - `secure_getenv`, `getentropy` + - `clock_nanosleep` + - POSIX semaphores (`sem_open`/`sem_close`/`sem_unlink`) + - POSIX mqueue (`mq_*`) + - `DT_RPATH` support in ld.so + - `redox_syscall = "0.8.1"` (ABI migration) + - DRM ioctl definitions (GET_UNIQUE, SET_VERSION, MODE_ADD_FB2) + - `rpl_malloc`/`rpl_realloc` (gnulib compat) + - 8MB stack size +- **Features now in upstream (DROP from Red Bear):** + - `posix_spawn` (port from upstream → upstream now has it) + - VaList API adaptation (upstream fixed) + - Various doc/lint fixes (upstream applied) +- **Compile gotchas:** + - `core::arch::x86_64::__cpuid()` requires `unsafe { }` block (edition 2024) + - `Vec::into_raw_parts()` is unstable — use `mem::transmute` + `as_ptr`/`len`/`capacity` + - `-D unused_mut` in upstream Makefile flags + +### kernel + +- **Upstream repo:** `https://gitlab.redox-os.org/redox-os/kernel.git` +- **Fork location:** `local/sources/kernel/` +- **Red Bear features to preserve:** + - Any ACPI improvements specific to AMD + - IRQ routing changes + - MSI/MSI-X support + - IOMMU work +- **Check upstream first** — kernel is the fastest-moving Redox component + +### bootloader + +- **Upstream repo:** `https://gitlab.redox-os.org/redox-os/bootloader.git` +- **Fork location:** `local/sources/bootloader/` +- **Red Bear features to preserve:** + - Branding (Red Bear logo, colors) + - Any boot protocol changes + +### installer + +- **Upstream repo:** `https://gitlab.redox-os.org/redox-os/installer.git` +- **Fork location:** `local/sources/installer/` +- **Red Bear features to preserve:** + - ext4 filesystem support + - GRUB bootloader support + - CollisionTracker (file ownership conflict detection) + - Init service path validation + +### redoxfs + +- **Upstream repo:** `https://gitlab.redox-os.org/redox-os/redoxfs.git` +- **Fork location:** `local/sources/redoxfs/` +- **Red Bear features to preserve:** + - Any performance patches + - Any filesystem format changes + +### userutils + +- **Upstream repo:** `https://gitlab.redox-os.org/redox-os/userutils.git` +- **Fork location:** `local/sources/userutils/` +- **Red Bear features to preserve:** + - Custom `/etc/issue` (Red Bear login banner) + - Any Redox ABI changes for syscall 0.8.x + +### base + +- **Upstream repo:** Pinned via `recipes/core/base/recipe.toml` (path-based fork) +- **Fork location:** `local/sources/base/` +- **Red Bear features to preserve:** + - All Red Bear init.d services + - All Red Bear branding + - All Red Bear config overrides + - HID/I2C driver wiring + - D-Bus integration + - Session management (redbear-sessiond) + - ~100 individual P*.patch files in `local/patches/base/` + +## Rollback Procedure + +If the sync fails or produces a broken build: + +```bash +cd local/sources/ +git checkout master +git reset --hard backup-master-pre-upstream-update- +git push gitea master --force-with-lease +``` + +Then in the main repo: +```bash +git checkout -- recipes/core//recipe.toml +# Revert recipe changes if any +``` + +## Decision Matrix: Keep vs Drop vs Merge + +For each Red Bear commit when syncing: + +``` +Is the feature in upstream now? +├── YES → Does upstream's version work for us? +│ ├── YES → DROP our commit, use upstream +│ └── NO → MERGE: take upstream's structure, re-apply our specific changes +└── NO → Is it a Red Bear unique feature? + ├── YES → KEEP our commit + └── NO → Is it a toolchain/compat fix? + ├── Still needed? → KEEP + └── Fixed upstream? → DROP +``` + +## Checklist Per Package + +- [ ] Backup branch created +- [ ] Divergence analyzed (commits categorized) +- [ ] Sync branch created from latest upstream +- [ ] Squash-merge applied +- [ ] All conflicts resolved +- [ ] Commit made with sync message +- [ ] Compiles via `repo cook` +- [ ] Critical symbols verified (for relibc) +- [ ] Pushed to Gitea +- [ ] Recipe updated (if git+patches mode) +- [ ] Dependent components rebuilt +- [ ] Full image build succeeds +- [ ] QEMU boot test passes diff --git a/mk/prefix.mk b/mk/prefix.mk index 313a372775..40bd702f60 100644 --- a/mk/prefix.mk +++ b/mk/prefix.mk @@ -97,6 +97,16 @@ ifeq ($(PODMAN_BUILD),1) else rm -rf "$@" cp -r "$(PREFIX)/relibc-install/" "$@" +# Propagate fresh relibc artifacts (libc.a, crt*.o, headers) from the +# rebuilt sysroot back into gcc-install. The GCC driver's built-in +# library search path always looks inside gcc-install/ first; if that +# copy is stale (from the original GCC bootstrap) the linker silently +# picks up an outdated libc.a and we get spurious undefined-reference +# errors for symbols that exist only in the new relibc (e.g. __fseterr, +# __freadahead, eventfd helpers, …). This sync guarantees that every +# libc.a copy in the toolchain is identical to the one just compiled. + cp -rf "$@/$(GNU_TARGET)/lib/"* "$(PREFIX)/gcc-install/$(GNU_TARGET)/lib/" + cp -rf "$@/$(GNU_TARGET)/include/"* "$(PREFIX)/gcc-install/$(GNU_TARGET)/include/" # adapt path for libtoolize sed 's|/usr/share|$(ROOT)/$@/share|g' "$@/bin/libtoolize.orig" > "$@/bin/libtoolize" chmod 0755 "$@/bin/libtoolize" diff --git a/recipes/core/relibc/recipe.toml b/recipes/core/relibc/recipe.toml index 5829bb7678..aca0b76cda 100644 --- a/recipes/core/relibc/recipe.toml +++ b/recipes/core/relibc/recipe.toml @@ -1,74 +1,5 @@ [source] -git = "https://gitlab.redox-os.org/redox-os/relibc.git" -rev = "861bbb0" -patches = [ - # Bump relibc workspace redox_syscall pin to 0.8.1 (matches rest of build) - "P0-relibc-syscall-0.8.1.patch", - # Bump redox-ioctl's redox_syscall pin to 0.8.1 (matches rest of build) - "P3-eventfd-mod.patch", - # Add pub mod bits_eventfd to header/mod.rs - "P3-bits-eventfd-mod.patch", - # bits_eventfd module (eventfd_t type) - "P3-bits-eventfd.patch", - # sys_eventfd module — FULL eventfd() implementation (opens /scheme/event/eventfd) - "P3-eventfd-impl.patch", - # cbindgen.toml for sys/eventfd.h C header generation - "P3-eventfd-cbindgen.patch", - # eventfd_read() and eventfd_write() helpers - "P3-eventfd-readwrite.patch", - # sys_signalfd module (cbindgen.toml + mod.rs with cbindgen exports) - "P3-signalfd-header.patch", - # signalfd implementation (signal/signalfd.rs + signal/mod.rs wiring) - "P3-signalfd.patch", - # sys_timerfd module (creates mod.rs with timerfd_create/settime/gettime) - "P3-timerfd-impl.patch", - # timerfd: creates cbindgen.toml, reformats, adds relative time handling - "P3-timerfd-relative.patch", - # timerfd: fix cbindgen.toml to generate C (not C++) headers - "P3-timerfd-cbindgen-fix.patch", - - # sys_types_internal: include stdint.h so uint32_t/int32_t are available to - # all downstream headers (signal.h, sys_signalfd.h, etc.) via the - # signal.h -> sys/types.h -> sys/types/internal.h include chain - "P3-sys-types-stdint-include.patch", - - # open_memstream: creates stdio/open_memstream.rs - "P3-open-memstream-create.patch", - # open_memstream: wires mod into stdio/mod.rs - "P3-stdio-open-memstream-mod.patch", - # F_DUPFD_CLOEXEC fallback (try CLOEXEC, fall back to DUPFD + SETFD) - "P3-fcntl-dupfd-cloexec.patch", - # Non-conflicting individual patches - "P3-elf64-types.patch", - "P3-clock-nanosleep.patch", - "P3-exec-root-bypass.patch", - "P3-secure-getenv.patch", - "P3-getentropy.patch", - "P3-dup3.patch", - "P3-waitid-header.patch", - "P3-waitid.patch", - "P3-in6-pktinfo.patch", - "P3-inet6-pton-ntop.patch", - "P3-tcp-nodelay.patch", - "P3-tcp-sockopt-forward.patch", - # Named POSIX semaphores (sem_open/close/unlink) — comprehensive + refcount + va_list - "P3-semaphore-comprehensive.patch", - # semaphore.h: expose SEM_FAILED and real variadic sem_open() prototype/ABI - "P3-semaphore-varargs-header.patch", - # Reverse From<&syscall::TimeSpec> for timespec (needed by getrusage) - "P3-timespec-reverse-from.patch", - "P10-stack-size-8mb.patch", - "P11-getrlimit-getrusage.patch", - # Move #include after size_t/ptrdiff_t typedefs in stddef.h - # to prevent recursive include chain (stdint.h → sys/types.h → pthread.h) - # from seeing undefined size_t when gnulib wrappers are present - "P3-stddef-reorder.patch", - # ld.so: parse DT_RPATH in addition to DT_RUNPATH (RUNPATH takes precedence per gABI) - "P3-ldso-rpath-support.patch", - - # stdio_ext.h + __freadahead / __fseterr (m4/bison/flex gnulib compatibility) - "P3-stdio-fseterr.patch", -] +path = "../../../local/sources/relibc" [build] template = "custom" diff --git a/src/config.rs b/src/config.rs index 0b4d9d6e02..cb79f58802 100644 --- a/src/config.rs +++ b/src/config.rs @@ -104,33 +104,44 @@ pub fn init_config() { .unwrap_or(1), )); } - if config.cook_opt.logs.is_none() { - config.cook_opt.logs = Some(extract_env("COOKBOOK_LOGS", config.cook_opt.tui.unwrap())); - } - if config.cook_opt.offline.is_none() { - config.cook_opt.offline = Some(extract_env("COOKBOOK_OFFLINE", true)); - } - if config.cook_opt.compressed.is_none() { - config.cook_opt.compressed = Some(extract_env("COOKBOOK_COMPRESSED", false)); - } - if config.cook_opt.verbose.is_none() { - config.cook_opt.verbose = Some(extract_env("COOKBOOK_VERBOSE", true)); - } - if config.cook_opt.nonstop.is_none() { - config.cook_opt.nonstop = Some(extract_env("COOKBOOK_NONSTOP", false)); - } - if config.cook_opt.clean_build.is_none() { - config.cook_opt.clean_build = Some(extract_env("COOKBOOK_CLEAN_BUILD", false)); - } - if config.cook_opt.clean_target.is_none() { - config.cook_opt.clean_target = Some(extract_env("COOKBOOK_CLEAN_TARGET", false)); - } - if config.cook_opt.write_filetree.is_none() { - config.cook_opt.write_filetree = Some(extract_env( - "COOKBOOK_WRITE_FILETREE", + // --- Env-var override layer --- + // Environment variables ALWAYS override cookbook.toml values. The TOML + // value (or a sensible default) is used only when the env var is absent. + // This follows the standard "env > config-file > built-in-default" hierarchy. + config.cook_opt.logs = Some(extract_env( + "COOKBOOK_LOGS", + config.cook_opt.logs.unwrap_or(config.cook_opt.tui.unwrap()), + )); + config.cook_opt.offline = Some(extract_env( + "COOKBOOK_OFFLINE", + config.cook_opt.offline.unwrap_or(true), + )); + config.cook_opt.compressed = Some(extract_env( + "COOKBOOK_COMPRESSED", + config.cook_opt.compressed.unwrap_or(false), + )); + config.cook_opt.verbose = Some(extract_env( + "COOKBOOK_VERBOSE", + config.cook_opt.verbose.unwrap_or(true), + )); + config.cook_opt.nonstop = Some(extract_env( + "COOKBOOK_NONSTOP", + config.cook_opt.nonstop.unwrap_or(false), + )); + config.cook_opt.clean_build = Some(extract_env( + "COOKBOOK_CLEAN_BUILD", + config.cook_opt.clean_build.unwrap_or(false), + )); + config.cook_opt.clean_target = Some(extract_env( + "COOKBOOK_CLEAN_TARGET", + config.cook_opt.clean_target.unwrap_or(false), + )); + config.cook_opt.write_filetree = Some(extract_env( + "COOKBOOK_WRITE_FILETREE", + config.cook_opt.write_filetree.unwrap_or( config.cook_opt.clean_target.unwrap_or(false) || extract_env("COOKBOOK_WEB", false), - )); - } + ), + )); if config.mirrors.len() == 0 { // The GNU FTP mirror below is automatically inserted for convenience // You can choose other mirrors by setting it on cookbook.toml diff --git a/src/cook/script.rs b/src/cook/script.rs index ad79294f42..fce6894375 100644 --- a/src/cook/script.rs +++ b/src/cook/script.rs @@ -35,6 +35,14 @@ function DYNAMIC_INIT { # TODO: check paths for spaces export LDFLAGS="${USER_LDFLAGS}-Wl,-rpath-link,${COOKBOOK_SYSROOT}/lib -L${COOKBOOK_SYSROOT}/lib" + + # Re-add prefix toolchain sysroot libs for dynamic builds. + # DYNAMIC_INIT resets LDFLAGS from USER_LDFLAGS, so the -L added by + # BUILD_PRESCRIPT is lost. Add it back here. + REDBEAR_PREFIX_SYSROOT="${COOKBOOK_ROOT}/prefix/${TARGET}/sysroot" + if [ -d "${REDBEAR_PREFIX_SYSROOT}/${TARGET}/lib" ]; then + export LDFLAGS="${LDFLAGS} -L${REDBEAR_PREFIX_SYSROOT}/${TARGET}/lib" + fi export RUSTFLAGS="-C target-feature=-crt-static -L native=${COOKBOOK_SYSROOT}/lib -C link-arg=-Wl,-rpath-link,${COOKBOOK_SYSROOT}/lib" export COOKBOOK_DYNAMIC=1 @@ -106,6 +114,15 @@ export CPPFLAGS="${CPPFLAGS:+$CPPFLAGS }-I${COOKBOOK_SYSROOT}/include" USER_LDFLAGS="${LDFLAGS:+$LDFLAGS }" export LDFLAGS="${USER_LDFLAGS}-L${COOKBOOK_SYSROOT}/lib --static" +# Belt-and-suspenders: explicitly add the prefix toolchain sysroot to the +# linker search path so the fresh relibc libc.a is always found, regardless +# of which GCC binary the redoxer ends up invoking. See the companion sync +# step in mk/prefix.mk ($(PREFIX)/sysroot target) for the full explanation. +REDBEAR_PREFIX_SYSROOT="${COOKBOOK_ROOT}/prefix/${TARGET}/sysroot" +if [ -d "${REDBEAR_PREFIX_SYSROOT}/${TARGET}/lib" ]; then + export LDFLAGS="${LDFLAGS} -L${REDBEAR_PREFIX_SYSROOT}/${TARGET}/lib" +fi + # This reexport C variables into custom build script that can be consumed by cc crate function reexport_flags { target=${TARGET//-/_}