RedBear-OS/local/scripts/test-usb-error-recovery-qemu.sh

#!/usr/bin/env bash
# Validate bounded USB error recovery under hot-unplug mid-transfer (P8-C).
#
# Topology under test:
#   qemu-xhci root port → usb-storage (backed by a seeded raw image)
#
# Proof: hot-unplug the usb-storage device via the QEMU monitor (`device_del`)
# while usbscsid is actively serving reads, then verify xhcid and usbscsid
# handle the loss gracefully — a logged detach + a logged IO/scheme error is
# acceptable; a panic, abort, or RUST_BACKTRACE is not. This is the runtime
# error-injection half of P2-C (core complete) and the P8-C exit criterion:
# "hot-unplug during transfer, verify graceful error (not panic)".
#
# Why the QEMU monitor and not the python login harness:
# qemu-login-expect.py drives only the guest serial console (send/expect). It
# cannot toggle to the QEMU monitor to issue `device_del`, so this script uses
# the same `-chardev stdio,mux=on -mon chardev=debug` monitor-mux pattern as
# test-xhci-device-lifecycle-qemu.sh, switching between serial and monitor with
# the Ctrl-A c (`\001c`) escape.
#
# The guest-side transfer is driven by `redbear-usb-storage-check` (in the
# redbear-hwutils package), which performs STORAGE_DISCOVERY + WRITE + READBACK
# + RESTORE — a sequence of real bulk transfers that usbscsid is mid-way
# through when the unplug lands. No guest-side checker is invented; the
# existing one is reused, exactly as test-usb-storage-qemu.sh does.
#
# Graceful-handling markers (real driver source):
#   "Device on port <n> was detached"           (xhcid detach path)
#   "usbscsid: READ IO ERROR: ..."              (usbscsid main.rs:190)
#   "usbscsid: WRITE IO ERROR: ..."             (usbscsid main.rs:200)
#   "usbscsid: scheme tick error: ..."          (usbscsid main.rs:163)
#   "usbscsid: event error: ..."                (usbscsid main.rs:156)
# Failure markers (prove NOT graceful):
#   "panic" / "RUST_BACKTRACE" / "usbscsid: startup failed"

set -euo pipefail

seed_usb_image() {
    local image_path="$1"
    python3 - "$image_path" <<'PY'
import pathlib
import sys

path = pathlib.Path(sys.argv[1])
payload = (b"REDBEAR-USB-ERROR-RECOVERY-CHECK\0" * 32)[:512]
payload = payload.ljust(512, b'\0')

with path.open("r+b") as fh:
    fh.seek(0)
    fh.write(payload)
PY
}

find_uefi_firmware() {
    local candidates=(
        "/usr/share/ovmf/x64/OVMF.4m.fd"
        "/usr/share/OVMF/x64/OVMF.4m.fd"
        "/usr/share/ovmf/x64/OVMF_CODE.4m.fd"
        "/usr/share/OVMF/x64/OVMF_CODE.4m.fd"
        "/usr/share/ovmf/OVMF.fd"
        "/usr/share/OVMF/OVMF_CODE.fd"
        "/usr/share/qemu/edk2-x86_64-code.fd"
    )
    local path
    for path in "${candidates[@]}"; do
        if [[ -f "$path" ]]; then
            printf '%s\n' "$path"
            return 0
        fi
    done
    return 1
}

usage() {
    cat <<'USAGE'
Usage: test-usb-error-recovery-qemu.sh [--check] [config]

Boot a Red Bear image, attach a usb-storage device, drive an active transfer
via redbear-usb-storage-check, and hot-unplug the device mid-transfer through
the QEMU monitor. Verifies xhcid/usbscsid handle the loss gracefully (logged
detach + logged error, no panic). Defaults to redbear-mini (mapped to the
in-tree redbear-mini image).

The --check flag is accepted for runbook/matriz consistency with the other USB
proofs; the monitor-driven sequence runs identically in either mode (mirroring
test-xhci-device-lifecycle-qemu.sh, which treats --check the same way).
USAGE
}

config="redbear-mini"
for arg in "$@"; do
    case "$arg" in
        --help|-h|help)
            usage
            exit 0
            ;;
        --check)
            # Accepted for consistency; behavior is identical (see usage).
            ;;
        redbear-*)
            config="$arg"
            ;;
    esac
done

firmware="$(find_uefi_firmware)" || {
    echo "ERROR: no usable x86_64 UEFI firmware found" >&2
    exit 1
}

arch="${ARCH:-$(uname -m)}"
image="build/$arch/$config/harddrive.img"
extra="build/$arch/$config/extra.img"
usb_img="build/$arch/$config/usb-error-recovery-storage.img"
log_file="build/$arch/$config/usb-error-recovery.log"
session_tag="Red Bear OS USB Error Recovery Test $$"
session_image="/tmp/redbear-usb-error-recovery-$$-harddrive.img"
session_extra="/tmp/redbear-usb-error-recovery-$$-extra.img"
session_usb_img="/tmp/redbear-usb-error-recovery-$$-usb.img"

# Prefer the live ISO (canonical build-redbear.sh artifact); fall back to
# harddrive.img for older make-all flows. The lifecycle/error-recovery proofs
# drive the QEMU monitor, which requires a harddrive-style -drive (snapshot=on)
# rather than -cdrom, so when only the ISO is present we still boot the ISO but
# attach the USB image as a separate snapshot drive (the storage device under
# test is the usb-storage, not the boot disk).
iso="build/$arch/$config.iso"
if [[ ! -f "$image" && ! -f "$iso" ]]; then
    echo "ERROR: no bootable image found ($iso or $image)" >&2
    echo "Build it first with: ./local/scripts/build-redbear.sh $config" >&2
    exit 1
fi
# Resolve to an existing boot medium, preferring the ISO.
if [[ -f "$iso" ]]; then
    boot_src="iso"
else
    boot_src="harddrive"
fi

if [[ ! -f "$extra" ]]; then
    truncate -s 1g "$extra"
fi

if [[ ! -f "$usb_img" ]]; then
    truncate -s 64M "$usb_img"
fi
seed_usb_image "$usb_img"

image_real="$(realpath "$image" 2>/dev/null || echo "$image")"
iso_real="$(realpath "$iso" 2>/dev/null || echo "$iso")"
extra_real="$(realpath "$extra")"
usb_img_real="$(realpath "$usb_img")"
log_file="$(realpath -m "$log_file")"

ln -sf "$extra_real" "$session_extra"
ln -sf "$usb_img_real" "$session_usb_img"
if [[ "$boot_src" == "harddrive" ]]; then
    ln -sf "$image_real" "$session_image"
fi

pkill -f "qemu-system-x86_64.*$session_tag" 2>/dev/null || true
sleep 1

rm -f "$log_file"

# Assemble the boot-drive argument. The monitor-mux lifecycle pattern uses a
# snapshot harddrive drive; when we have an ISO we attach it via -cdrom instead
# (the usb-storage device under test is independent of the boot medium).
if [[ "$boot_src" == "harddrive" ]]; then
    boot_drive_args=(-drive "file=$session_image,format=raw,if=none,id=drv0,snapshot=on" -device nvme,drive=drv0,serial=NVME_SERIAL)
else
    boot_drive_args=(-cdrom "$iso_real")
fi

expect <<EOF
log_user 1
log_file -noappend $log_file
set timeout 1800
set send_slow {1 0.0}
spawn qemu-system-x86_64 -name {$session_tag} -device qemu-xhci,id=xhci -smp 4 -m 2048 -bios $firmware -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -nographic -vga none {*}"${boot_drive_args[*]}" -drive file=$session_extra,format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -drive file=$session_usb_img,format=raw,if=none,id=usbdisk,snapshot=on -enable-kvm -cpu host

# Wait for boot to the login prompt.
expect "login:"
send "root\r"
expect "assword:"
send "password\r"
expect -re {Type 'help' for available commands\.}
expect -re {# }

# Hotplug the usb-storage device under test, then wait for usbscsid to spawn
# and serve at least one transfer (DISCOVERY). This proves the device was
# attached and the bulk-transfer path was live before the unplug.
send "\001c"
expect "(qemu)"
send "device_add usb-storage,bus=xhci.0,drive=usbdisk,id=usbstore0\r"
expect "(qemu)"
send "\001c"
expect -re {USB SCSI driver spawned with scheme}
expect -re {xhcid: finished attach for port [0-9.]+}

# Drive an active transfer. redbear-usb-storage-check performs DISCOVERY +
# WRITE + READBACK + RESTORE, a sequence of real bulk transfers. We do not
# wait for it to finish: the device_del below lands while usbscsid is
# servicing these transfers (mid-transfer).
send "redbear-usb-storage-check\r"

# Allow the check to enter its transfer loop before unplugging. 3s is enough
# for DISCOVERY + the first WRITE/READBACK bulk transfers to be in flight on
# QEMU's emulated xHCI (characteristic of the storage proof's timing).
after 3000

# Hot-unplug mid-transfer via the QEMU monitor.
send "\001c"
expect "(qemu)"
send "device_del usbstore0\r"
expect "(qemu)"
send "\001c"

# Observe the graceful-detach path from xhcid. A bounded wait: if the detach
# marker does not appear, the unplug was not observed and the proof fails the
# post-hoc grep below.
expect {
    -re {Device on port [0-9.]+ was detached} { }
    timeout { }
}

# Give usbscsid a moment to log the transfer error from its tick loop, then
# return to a shell prompt to shut down cleanly.
after 2000
send "\r"
expect -re {# }
send "shutdown\r"
sleep 2
EOF

pkill -f "qemu-system-x86_64.*$session_tag" 2>/dev/null || true
rm -f "$session_image" "$session_extra" "$session_usb_img"

failures=0

echo "--- USB Error Recovery Validation: $config ---"

# 1. The usb-storage device attached and usbscsid spawned (precondition: there
#    was a live transfer to interrupt).
if grep -aq "USB SCSI driver spawned with scheme" "$log_file"; then
    echo "  [PASS] usbscsid spawned for the hotplugged usb-storage device"
else
    echo "  [FAIL] usbscsid did not spawn; no live transfer to interrupt" >&2
    failures=$((failures + 1))
fi

# 2. The hot-unplug was observed by xhcid (detach path fired).
if grep -Eaq 'Device on port [0-9.]+ was detached' "$log_file"; then
    echo "  [PASS] xhcid observed the mid-transfer hot-unplug"
else
    echo "  [FAIL] xhcid did not log the detach for the unplugged device" >&2
    failures=$((failures + 1))
fi

# 3. Graceful error handling: usbscsid logged an IO/scheme error from its
#    transfer/tick loop rather than hanging silently. Any one of the
#    documented error markers satisfies this (they are the catch-and-log
#    sites that replaced the panic! calls in P1-B).
if grep -Eaq 'usbscsid: (READ|WRITE) IO ERROR:|usbscsid: scheme tick error:|usbscsid: event error:' "$log_file"; then
    echo "  [PASS] usbscsid logged a graceful transfer error (no silent hang)"
else
    echo "  [WARN] no usbscsid transfer-error marker observed (transfer may have completed before the unplug landed)" >&2
fi

# 4. The critical P8-C guarantee: NO panic / abort / backtrace anywhere in the
#    boot. This is the single hard failure that invalidates the proof.
if grep -qiE "panic|RUST_BACKTRACE|usbscsid: startup failed" "$log_file"; then
    echo "  [FAIL] panic-class failure detected during mid-transfer hot-unplug" >&2
    grep -iE "panic|RUST_BACKTRACE|usbscsid: startup failed" "$log_file" | head -5 >&2
    failures=$((failures + 1))
else
    echo "  [PASS] no panic/abort/backtrace during mid-transfer hot-unplug"
fi

echo "--- Results: $failures failure(s), log: $log_file ---"

if [[ "$failures" -gt 0 ]]; then
    exit 1
fi

exit 0