tests: replace expect with stdlib python helper for QEMU login proofs

Drop the host expect dependency from the PS/2, timer, IOMMU, and USB
storage (BOT) runtime proofs. local/scripts/qemu-login-expect.py drives
the guest over serial-on-stdio with ordered expect/send steps, per-step
timeouts (matching expect's per-pattern semantics), and pass/fail markers.
It distinguishes premature QEMU death (exit 3) from genuine check failure
(exit 1) so the scripts retry external interruptions without retrying real
failures. Scripts use headless -display none -vga none (fbcond serial
mirror), a 3600s per-step timeout, and a retry loop (max 5 attempts).

local/scripts/qemu-login-expect.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Drop-in replacement for the `expect` tool used by the low-level proof
+scripts, using only the Python standard library (no external deps).
+
+Drives a QEMU guest over its serial-on-stdio chardev: spawns QEMU with
+stdin/stdout pipes, watches the guest output for expected patterns, sends
+replies, and reports pass/fail. This removes the host `expect` package
+dependency (which is documented but not always installed, and cannot be
+installed without sudo).
+
+Spec model: an ordered list of steps. Each step is `expect:<text>` (wait for
+the literal text in guest output) or `send:<text>` (write text + CR to the
+guest). After all steps, wait for `--pass <text>` to succeed, or `--fail
+<text>` / timeout to fail.
+"""
+import argparse
+import os
+import select
+import subprocess
+import sys
+import time
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(add_help=False)
+    p.add_argument("--timeout", type=int, default=240)
+    p.add_argument("--log", required=True)
+    p.add_argument("--step", action="append", default=[])
+    p.add_argument("--pass_marker", action="append", default=[])
+    p.add_argument("--fail_marker", action="append", default=[])
+    p.add_argument("qemu", nargs=argparse.REMAINDER)
+    args = p.parse_args()
+
+    qemu_cmd = args.qemu
+    if qemu_cmd and qemu_cmd[0] == "--":
+        qemu_cmd = qemu_cmd[1:]
+    if not qemu_cmd:
+        print("ERROR: no qemu command given", file=sys.stderr)
+        return 2
+
+    log_f = open(args.log, "wb", buffering=0)
+    try:
+        proc = subprocess.Popen(
+            qemu_cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            bufsize=0,
+        )
+    except FileNotFoundError as e:
+        print(f"ERROR: cannot spawn qemu: {e}", file=sys.stderr)
+        return 2
+
+    out = proc.stdout
+    assert out is not None, "qemu stdout pipe is required for serial interaction"
+
+    deadline = time.monotonic() + args.timeout
+    buf = bytearray()
+    state = {"failed": False, "passes": set(), "died_early": False}
+
+    def record(chunk: bytes) -> str:
+        buf.extend(chunk)
+        log_f.write(chunk)
+        text = buf.decode("utf-8", errors="replace")
+        for fm in args.fail_marker:
+            if fm in text and fm not in state["passes"]:
+                state["failed"] = True
+        for pm in args.pass_marker:
+            if pm in text:
+                state["passes"].add(pm)
+        return text
+
+    def send(text: str) -> None:
+        if proc.stdin:
+            try:
+                proc.stdin.write(text.encode() + b"\r")
+                proc.stdin.flush()
+            except (BrokenPipeError, OSError):
+                pass
+
+    steps = list(args.step)
+    i = 0
+    # Phase 1: walk expect/send steps in order. The deadline resets on every
+    # step advance so each expect step gets its own full timeout window,
+    # matching the per-pattern semantics of the expect tool's `set timeout`.
+    while i < len(steps):
+        if time.monotonic() > deadline or state["failed"]:
+            break
+        step = steps[i]
+        if step.startswith("send:"):
+            send(step[len("send:"):])
+            i += 1
+            deadline = time.monotonic() + args.timeout
+            continue
+        if step.startswith("expect:"):
+            want = step[len("expect:"):]
+            if want in buf.decode("utf-8", errors="replace"):
+                i += 1
+                deadline = time.monotonic() + args.timeout
+                continue
+        r, _, _ = select.select([out], [], [], 0.5)
+        if r:
+            chunk = os.read(out.fileno(), 65536)
+            if not chunk:
+                state["died_early"] = True
+                break
+            record(chunk)
+        if proc.poll() is not None and not r:
+            state["died_early"] = True
+            break
+
+    # Phase 2: wait for pass markers (or fail/timeout). Fresh window: markers
+    # that were also expect steps are usually already recorded by phase 1.
+    # Skipped when phase 1 broke early: the guest check that prints pass
+    # markers is only sent by a completed step sequence, so waiting would
+    # just double the failure time.
+    if i >= len(steps):
+        deadline = time.monotonic() + args.timeout
+        while not state["failed"]:
+            if state["passes"] and len(state["passes"]) == len(args.pass_marker):
+                break
+            if not args.pass_marker:
+                break
+            if time.monotonic() > deadline:
+                break
+            r, _, _ = select.select([out], [], [], 0.5)
+            if r:
+                chunk = os.read(out.fileno(), 65536)
+                if not chunk:
+                    state["died_early"] = True
+                    break
+                record(chunk)
+            if proc.poll() is not None and not r:
+                state["died_early"] = True
+                break
+
+    try:
+        proc.terminate()
+        try:
+            proc.wait(timeout=3)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+    except OSError:
+        pass
+    log_f.close()
+
+    if state["failed"]:
+        print(f"ERROR: a --fail marker appeared; see {args.log}", file=sys.stderr)
+        return 1
+    if args.pass_marker and len(state["passes"]) != len(args.pass_marker):
+        if state["died_early"]:
+            print(f"ERROR: qemu exited before the check completed; see {args.log}", file=sys.stderr)
+            return 3
+        print(f"ERROR: expected pass markers not all seen; see {args.log}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

local/scripts/test-iommu-qemu.sh

@@ -109,25 +109,39 @@ pkill -f "qemu-system-x86_64.*$config" 2>/dev/null || true
 sleep 1
 
 if [[ "$check_mode" -eq 1 ]]; then
-    expect <<EOF
-log_user 1
-set timeout 240
-spawn qemu-system-x86_64 -name {Red Bear OS x86_64} -device qemu-xhci -device amd-iommu -smp 4 -m 2048 -bios $firmware -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file=build/$arch/$config/network.pcap -nographic -vga none ${boot_args[*]} -drive file=$extra,format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -enable-kvm -cpu host $extra_qemu_args
-expect -re {PCI .*1022:1419}
-expect "login:"
-send "root\r"
-expect "assword:"
-send "password\r"
-expect "Type 'help' for available commands."
-send "redbear-phase-iommu-check\r"
-expect "Red Bear OS IOMMU Runtime Check"
-expect "units_detected="
-expect "units_initialized_now="
-expect "units_initialized_after="
-expect "events_drained="
-send "shutdown\r"
-sleep 2
-EOF
+    attempt=1
+    while true; do
+        rc=0
+        python3 local/scripts/qemu-login-expect.py \
+        --timeout 3600 \
+        --log "build/$arch/$config/iommu-check.log" \
+        --step "expect:1022:1419" \
+        --step "expect:login:" \
+        --step "send:root" \
+        --step "expect:assword:" \
+        --step "send:password" \
+        --step "expect:Type 'help' for available commands." \
+        --step "send:redbear-phase-iommu-check" \
+        --step "expect:Red Bear OS IOMMU Runtime Check" \
+        --step "expect:units_detected=" \
+        --step "expect:units_initialized_now=" \
+        --step "expect:units_initialized_after=" \
+        --step "expect:events_drained=" \
+        --step "send:shutdown" \
+        --pass_marker "events_drained=" \
+        -- \
+        qemu-system-x86_64 -name "Red Bear OS x86_64" -device qemu-xhci -device amd-iommu -smp 4 -m 2048 -bios "$firmware" -chardev stdio,id=debug,signal=off,mux=on -serial chardev=debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" -display none -vga none "${boot_args[@]}" -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -enable-kvm -cpu host $extra_qemu_args || rc=$?
+        if [[ $rc -eq 0 ]]; then
+            break
+        fi
+        if [[ ( $rc -eq 3 || $rc -eq 137 ) && $attempt -lt 5 ]]; then
+            attempt=$((attempt + 1))
+            echo "QEMU exited before completing the check; retrying (attempt $attempt/3)"
+            sleep 2
+            continue
+        fi
+        exit "$rc"
+    done
     pkill -f "qemu-system-x86_64.*$image" 2>/dev/null || true
     echo "IOMMU first-use validation path completed via guest runtime check"
     exit 0
@@ -148,9 +162,8 @@ exec qemu-system-x86_64 \
   -device virtio-net,netdev=net0 \
   -netdev user,id=net0 \
   -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" \
-  -nographic -vga none \
-  -drive file="$image",format=raw,if=none,id=drv0,snapshot=on \
-  -device nvme,drive=drv0,serial=NVME_SERIAL \
+  -display none -vga none \
+  "${boot_args[@]}" \
   -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on \
   -device nvme,drive=drv1,serial=NVME_EXTRA \
   -enable-kvm -cpu host \

local/scripts/test-ps2-qemu.sh

@@ -93,23 +93,37 @@ pkill -f "qemu-system-x86_64.*$config" 2>/dev/null || true
 sleep 1
 
 if [[ "$check_mode" -eq 1 ]]; then
-    expect <<EOF
-log_user 1
-set timeout 240
-spawn qemu-system-x86_64 -name {Red Bear OS x86_64} -device qemu-xhci -smp 4 -m 2048 -bios $firmware -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file=build/$arch/$config/network.pcap -nographic -vga none ${boot_args[*]} -drive file=$extra,format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -enable-kvm -cpu host $extra_qemu_args
-expect "login:"
-send "root\r"
-expect "assword:"
-send "password\r"
-expect "Type 'help' for available commands."
-send "redbear-phase-ps2-check\r"
-expect "Red Bear OS PS/2 Runtime Check"
-expect "present=/scheme/serio/0"
-expect "present=/scheme/serio/1"
-expect "phase3_input_check=ok"
-send "shutdown\r"
-sleep 2
-EOF
+    attempt=1
+    while true; do
+        rc=0
+        python3 local/scripts/qemu-login-expect.py \
+        --timeout 3600 \
+        --log "build/$arch/$config/ps2-check.log" \
+        --step "expect:login:" \
+        --step "send:root" \
+        --step "expect:assword:" \
+        --step "send:password" \
+        --step "expect:Type 'help' for available commands." \
+        --step "send:redbear-phase-ps2-check" \
+        --step "expect:Red Bear OS PS/2 Runtime Check" \
+        --step "expect:present=/scheme/serio/0" \
+        --step "expect:present=/scheme/serio/1" \
+        --step "expect:phase3_input_check=ok" \
+        --step "send:shutdown" \
+        --pass_marker "phase3_input_check=ok" \
+        -- \
+        qemu-system-x86_64 -name "Red Bear OS x86_64" -device qemu-xhci -smp 4 -m 2048 -bios "$firmware" -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" -display none -vga none "${boot_args[@]}" -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -enable-kvm -cpu host $extra_qemu_args || rc=$?
+        if [[ $rc -eq 0 ]]; then
+            break
+        fi
+        if [[ ( $rc -eq 3 || $rc -eq 137 ) && $attempt -lt 5 ]]; then
+            attempt=$((attempt + 1))
+            echo "QEMU exited before completing the check; retrying (attempt $attempt/3)"
+            sleep 2
+            continue
+        fi
+        exit "$rc"
+    done
     pkill -f "qemu-system-x86_64.*$image" 2>/dev/null || true
     echo "PS/2 serio runtime validation completed via guest runtime check"
     exit 0
@@ -129,9 +143,8 @@ exec qemu-system-x86_64 \
   -device virtio-net,netdev=net0 \
   -netdev user,id=net0 \
   -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" \
-  -nographic -vga none \
-  -drive file="$image",format=raw,if=none,id=drv0,snapshot=on \
-  -device nvme,drive=drv0,serial=NVME_SERIAL \
+  -display none -vga none \
+  "${boot_args[@]}" \
   -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on \
   -device nvme,drive=drv1,serial=NVME_EXTRA \
   -enable-kvm -cpu host \

local/scripts/test-timer-qemu.sh

@@ -93,21 +93,35 @@ pkill -f "qemu-system-x86_64.*$config" 2>/dev/null || true
 sleep 1
 
 if [[ "$check_mode" -eq 1 ]]; then
-    expect <<EOF
-log_user 1
-set timeout 240
-spawn qemu-system-x86_64 -name {Red Bear OS x86_64} -device qemu-xhci -smp 4 -m 2048 -bios $firmware -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file=build/$arch/$config/network.pcap -nographic -vga none ${boot_args[*]} -drive file=$extra,format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -enable-kvm -cpu host $extra_qemu_args
-expect "login:"
-send "root\r"
-expect "assword:"
-send "password\r"
-expect "Type 'help' for available commands."
-    send "if test -e /scheme/time/4; then ls /scheme/time/4; printf 'monotonic_progress=ok\\n'; elif test -e /scheme/time/CLOCK_MONOTONIC; then ls /scheme/time/CLOCK_MONOTONIC; printf 'monotonic_progress=ok\\n'; else echo missing_time; fi\r"
-    expect -re {/scheme/time/(4|CLOCK_MONOTONIC)}
-    expect "monotonic_progress=ok"
-    send "shutdown\r"
-    expect eof
-EOF
+    attempt=1
+    while true; do
+        rc=0
+        python3 local/scripts/qemu-login-expect.py \
+        --timeout 3600 \
+        --log "build/$arch/$config/timer-check.log" \
+        --step "expect:login:" \
+        --step "send:root" \
+        --step "expect:assword:" \
+        --step "send:password" \
+        --step "expect:Type 'help' for available commands." \
+        --step "send:if test -e /scheme/time/4; then ls /scheme/time/4; printf 'monotonic_progress=ok\\n'; elif test -e /scheme/time/CLOCK_MONOTONIC; then ls /scheme/time/CLOCK_MONOTONIC; printf 'monotonic_progress=ok\\n'; else echo missing_time; fi" \
+        --step "expect:/scheme/time/" \
+        --step "expect:monotonic_progress=ok" \
+        --step "send:shutdown" \
+        --pass_marker "monotonic_progress=ok" \
+        -- \
+        qemu-system-x86_64 -name "Red Bear OS x86_64" -device qemu-xhci -smp 4 -m 2048 -bios "$firmware" -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" -display none -vga none "${boot_args[@]}" -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -enable-kvm -cpu host $extra_qemu_args || rc=$?
+        if [[ $rc -eq 0 ]]; then
+            break
+        fi
+        if [[ ( $rc -eq 3 || $rc -eq 137 ) && $attempt -lt 5 ]]; then
+            attempt=$((attempt + 1))
+            echo "QEMU exited before completing the check; retrying (attempt $attempt/3)"
+            sleep 2
+            continue
+        fi
+        exit "$rc"
+    done
     echo "Timer runtime validation completed via guest runtime check"
     exit 0
 fi
@@ -126,9 +140,8 @@ exec qemu-system-x86_64 \
   -device virtio-net,netdev=net0 \
   -netdev user,id=net0 \
   -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" \
-  -nographic -vga none \
-  -drive file="$image",format=raw,if=none,id=drv0,snapshot=on \
-  -device nvme,drive=drv0,serial=NVME_SERIAL \
+  -display none -vga none \
+  "${boot_args[@]}" \
   -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on \
   -device nvme,drive=drv1,serial=NVME_EXTRA \
   -enable-kvm -cpu host \

local/scripts/test-usb-storage-qemu.sh

@@ -109,54 +109,46 @@ pkill -f "qemu-system-x86_64.*$config" 2>/dev/null || true
 sleep 1
 
 rm -f "$log_file"
-expect <<EOF
-log_user 1
-log_file -noappend $log_file
-set timeout 300
-spawn qemu-system-x86_64 -name {Red Bear OS x86_64} -device qemu-xhci,id=xhci -smp 4 -m 2048 -bios $firmware -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file=build/$arch/$config/network.pcap -nographic -vga none ${boot_args[*]} -drive file=$extra,format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -drive file=$usb_img,format=raw,if=none,id=usbdisk,snapshot=on -device usb-storage,bus=xhci.0,drive=usbdisk -enable-kvm -cpu host
-expect "USB SCSI driver spawned"
-expect "DISK CONTENT: $expected_sector_b64"
-expect "login:"
-send "root\r"
-expect "assword:"
-send "password\r"
-expect "Type 'help' for available commands."
-send "redbear-usb-storage-check\r"
-expect {
-    "[PASS] STORAGE_DISCOVERY:" { }
-    "[FAIL] STORAGE_DISCOVERY:" {
-        puts "ERROR: USB storage device discovery failed"
-        exit 1
-    }
-    timeout { exit 1 }
-}
-expect {
-    "[PASS] STORAGE_WRITE:" { }
-    "[FAIL] STORAGE_WRITE:" {
-        puts "ERROR: USB storage write failed"
-        exit 1
-    }
-    timeout { exit 1 }
-}
-expect {
-    "[PASS] STORAGE_READBACK:" { }
-    "[FAIL] STORAGE_READBACK:" {
-        puts "ERROR: USB storage readback failed"
-        exit 1
-    }
-    timeout { exit 1 }
-}
-expect {
-    "[PASS] STORAGE_RESTORE:" { }
-    "[FAIL] STORAGE_RESTORE:" {
-        puts "ERROR: USB storage sector restore failed"
-        exit 1
-    }
-    timeout { exit 1 }
-}
-send "shutdown\r"
-sleep 2
-EOF
+attempt=1
+while true; do
+    rc=0
+    python3 local/scripts/qemu-login-expect.py \
+    --timeout 3600 \
+    --log "$log_file" \
+    --step "expect:USB SCSI driver spawned" \
+    --step "expect:DISK CONTENT: $expected_sector_b64" \
+    --step "expect:login:" \
+    --step "send:root" \
+    --step "expect:assword:" \
+    --step "send:password" \
+    --step "expect:Type 'help' for available commands." \
+    --step "send:redbear-usb-storage-check" \
+    --step "expect:[PASS] STORAGE_DISCOVERY:" \
+    --step "expect:[PASS] STORAGE_WRITE:" \
+    --step "expect:[PASS] STORAGE_READBACK:" \
+    --step "expect:[PASS] STORAGE_RESTORE:" \
+    --step "send:shutdown" \
+    --pass_marker "[PASS] STORAGE_DISCOVERY:" \
+    --pass_marker "[PASS] STORAGE_WRITE:" \
+    --pass_marker "[PASS] STORAGE_READBACK:" \
+    --pass_marker "[PASS] STORAGE_RESTORE:" \
+    --fail_marker "[FAIL] STORAGE_DISCOVERY:" \
+    --fail_marker "[FAIL] STORAGE_WRITE:" \
+    --fail_marker "[FAIL] STORAGE_READBACK:" \
+    --fail_marker "[FAIL] STORAGE_RESTORE:" \
+    -- \
+    qemu-system-x86_64 -name "Red Bear OS x86_64" -device qemu-xhci,id=xhci -smp 4 -m 2048 -bios "$firmware" -chardev stdio,id=debug,signal=off,mux=on -serial chardev:debug -mon chardev=debug -machine q35 -device ich9-intel-hda -device hda-output -device virtio-net,netdev=net0 -netdev user,id=net0 -object filter-dump,id=f1,netdev=net0,file="build/$arch/$config/network.pcap" -display none -vga none "${boot_args[@]}" -drive file="$extra",format=raw,if=none,id=drv1,snapshot=on -device nvme,drive=drv1,serial=NVME_EXTRA -drive file="$usb_img",format=raw,if=none,id=usbdisk,snapshot=on -device usb-storage,bus=xhci.0,drive=usbdisk -enable-kvm -cpu host || rc=$?
+    if [[ $rc -eq 0 ]]; then
+        break
+    fi
+    if [[ ( $rc -eq 3 || $rc -eq 137 ) && $attempt -lt 5 ]]; then
+        attempt=$((attempt + 1))
+        echo "QEMU exited before completing the check; retrying (attempt $attempt/3)"
+        sleep 2
+        continue
+    fi
+    exit "$rc"
+done
 
 pkill -f "qemu-system-x86_64.*$image" 2>/dev/null || true