From 36b258054cf58e8eb82bffb464725486a9acb15f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 00:29:23 +0000 Subject: [PATCH] qemu_session.py: a failed check prints where the CPUs are (cpu_snapshot) CI's Acceptance fails on aarch64 at BIRTH on every head since 58d0e887. The birth takes more than 300 s there and does not reproduce here: about 11 s, unsigned. The heartbeat spiral was ruled out by an instrumented scratch kernel. Captain Bob ruled "Diagnose on CI". On a failed check, the session script asks QEMU's monitor for the registers three times, a second apart, and prints the program counters, so CI's next failure shows where the kernel is and whether it moves. A monitor that does not answer does not stop the run. Test first: scripts/test_qemu_session_snapshot.sh, 5 checks; 3 failed before. The register text was checked on real, paused QEMU for aarch64, riscv64 and x86. The evidence and the measurements are in logs/20261119-ci-snapshot. The local aarch64 timing run is in logs/20261118-235613. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017RzZgrf7V7Zk2iczBXDRbm --- .../runs/doe-aarch64-20261118-235613.csv | 0 .../aarch64/qemu-aarch64-20261118-235613.log | 179 ++++++++++++++++++ logs/20261119-ci-snapshot/README.txt | 20 ++ .../local-aarch64-birth-timeline.txt | 11 ++ logs/20261119-ci-snapshot/test-after.log | 6 + logs/20261119-ci-snapshot/test-before.log | 6 + scripts/qemu_session.py | 22 ++- scripts/test_qemu_session_snapshot.sh | 57 ++++++ 8 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 experiments/bare_metal/runs/doe-aarch64-20261118-235613.csv create mode 100644 logs/20261118-235613/aarch64/qemu-aarch64-20261118-235613.log create mode 100644 logs/20261119-ci-snapshot/README.txt create mode 100644 logs/20261119-ci-snapshot/local-aarch64-birth-timeline.txt create mode 100644 logs/20261119-ci-snapshot/test-after.log create mode 100644 logs/20261119-ci-snapshot/test-before.log create mode 100755 scripts/test_qemu_session_snapshot.sh diff --git a/experiments/bare_metal/runs/doe-aarch64-20261118-235613.csv b/experiments/bare_metal/runs/doe-aarch64-20261118-235613.csv new file mode 100644 index 00000000..e69de29b diff --git a/logs/20261118-235613/aarch64/qemu-aarch64-20261118-235613.log b/logs/20261118-235613/aarch64/qemu-aarch64-20261118-235613.log new file mode 100644 index 00000000..28de6008 --- /dev/null +++ b/logs/20261118-235613/aarch64/qemu-aarch64-20261118-235613.log @@ -0,0 +1,179 @@ +UEFI firmware (version 2024.02-2ubuntu0.10 built at 15:40:28 on Sep 23 2026) +[=3h[=3hBdsDxe: loading Boot0001 "UEFI Misc Device" from PciRoot(0x0)/Pci(0x2,0x0) +BdsDxe: starting Boot0001 "UEFI Misc Device" from PciRoot(0x0)/Pci(0x2,0x0) +[=3hStarKernel UEFI Loader +Loading kernel from ESP... +[CKPT 001] Entered efi_main - ConOut live +Collecting boot information... +[CKPT 004] Command line parsed +[CKPT 005] Kernel stack allocation decided +[CKPT 006] Boot info collected (ACPI table located) +[CKPT 007] GOP: linear framebuffer found +[CKPT 008] About to enter ExitBootServices retry loop + + + _____ _ _ __ _ + / ____| | | |/ / | | + | (___ | |_ __ _ _ __| ' / ___ _ __ _ __ ___| | + \___ \| __/ _` | '__| < / _ \ '__| '_ \ / _ \ | + ____) | || (_| | | | . \ __/ | | | | | __/ | + |_____/ \__\__,_|_| |_|\_\___|_| |_| |_|\___|_| + +LithosAnanke v4.0.0-alpha.5 aarch64 +Architecture: aarch64 +Build: Nov 18 2026 23:55:50 + +UEFI BootServices: EXITED +Console: serial line is a PL011 at 0x0000000009000000 (no devicetree: QEMU's) + +=== StarKernel Boot Information === +Memory map entries: 33 +Total memory: 4093 MB +Usable memory: 4052 MB +=================================== + +PMM initialized. +PMM statistics: + Total pages: 1038855 + Free pages : 1037431 + Used pages : 1424 + Total MB : 4058 + Free MB : 4052 + Used MB : 5 + +VMM initialized (mapped RAM, CR3 switched) +VMM initialized (mapped RAM, CR3 switched) +VMM self-test: mapped OK at 0xffff800000000000 +VMM self-test complete. + +AArch64: running at EL1 +IDT installed. + +APIC: init... +PSCI: no DTB -- using HVC (QEMU virt-machine default) +GICv2: no DTB GIC node -- using QEMU virt-machine defaults +GICv2: distributor+CPU interface enabled, PPI 30 +APIC: init done + +Timer: init... +Timer: AArch64 generic timer initialised. +Timer: init done + +Kernel heap initialized. +Heap statistics: + Total bytes: 2147483608 + Free bytes: 2147483608 + Used bytes: 0 + Peak bytes: 0 + Heap base addr: 1207959552 + Heap end addr: 3355443200 + +Heartbeat: init... +Heartbeat: init done +Kernel initialization complete. +Boot successful! + +Boot disk: not a StarshipOS stick +System: StarForth v4 (this build's default) +Stadium: 335544 cells (20971 KB), 202 VM slots +Switch-signal: 202 slots +Kernel-Hermes: 202 channel slots +Kernel-Hermes: 202 pending-queue slots +StarForth v4: Hera, born in the kernel's fabric +V4: capsule v4:sys:map-64.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:map-64.f18 capsule_hash=0x06cb44832df2b8fd +V4: capsule v4:sys:core.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:core.f18 capsule_hash=0x0f415502fccc3fba +V4: capsule v4:sys:input.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:input.f18 capsule_hash=0x0f13c3f9d5fd07fc +V4: capsule v4:sys:dict.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:dict.f18 capsule_hash=0xe3453808afd5a0af +V4: capsule v4:sys:codegen.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:codegen.f18 capsule_hash=0x36d2ae84d2911b90 +V4: capsule v4:sys:compile.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:compile.f18 capsule_hash=0xb46bf77ceee7245a +V4: capsule v4:sys:quit.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:quit.f18 capsule_hash=0x7fa7a7668b6bf1ed +V4: capsule v4:sys:forth.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:forth.f18 capsule_hash=0xc4c0ad315c95eda4 +V4: capsule v4:sys:numout.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:numout.f18 capsule_hash=0xca0fb0f9c1589e20 +V4: capsule v4:sys:words.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:words.f18 capsule_hash=0x2f4e5f7f3e7f736f +V4: capsule v4:sys:system.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:system.f18 capsule_hash=0xfe257eb084c43a5d +V4: capsule v4:sys:qmath.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:qmath.f18 capsule_hash=0xa93783c2c8f9b7db +V4: capsule v4:sys:blocks.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:blocks.f18 capsule_hash=0x09d361dbe6174236 +V4: capsule v4:sys:log.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:log.f18 capsule_hash=0x544cd004233dff62 +V4: capsule v4:sys:acl.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:acl.f18 capsule_hash=0x76e7e746befa2733 +V4: capsule v4:sys:end.f18 signature: missing (unsigned) +PARITY:V4_F18 name=v4:sys:end.f18 capsule_hash=0x98fd33b1c431cfe7 +PARITY:V4_ROM words=2383 rom_hash=0xd858ef412d684f92 instructions=63411003 +V4: capsule v4:forth79.4th signature: missing (unsigned) +PARITY:V4_CAPSULE name=v4:forth79.4th capsule_id=0x4609d6ef54166ef8 capsule_hash=0x4609d6ef54166ef8 dict_hash=0x71be72a2ab8daaad +V4: capsule v4:starshipos.4th signature: missing (unsigned) +PARITY:V4_CAPSULE name=v4:starshipos.4th capsule_id=0x1734a8f4fe7483a5 capsule_hash=0x1734a8f4fe7483a5 dict_hash=0xa461a1e1146cda29 +V4: capsule v4:centre.4th signature: missing (unsigned) +PARITY:V4_CAPSULE name=v4:centre.4th capsule_id=0xea5815290d56dff3 capsule_hash=0xea5815290d56dff3 dict_hash=0x50efc3b2760c96e2 +V4: capsule v4:hera.4th signature: missing (unsigned) +PARITY:V4_CAPSULE name=v4:hera.4th capsule_id=0x78730b1d6effe892 capsule_hash=0x78730b1d6effe892 dict_hash=0x5b59c0e119ee3fba +V4: capsule v4:physics.4th signature: missing (unsigned) +PARITY:V4_CAPSULE name=v4:physics.4th capsule_id=0x5b0195fc30828bde capsule_hash=0x5b0195fc30828bde dict_hash=0xd9d729b0640e1879 +PARITY:V4_POST tests=990 pass=990 fail=0 +PARITY:V4_SYSTEM word_count=404 dict_hash=0x847bd8ef42f53e14 +PARITY:OK +POST: PASSED +PCI: init... +virtio-rng: found device +rng: backend = virtio-rng +entropy: ready +virtio-input: found device +virtio-input: driver ready +Starting heartbeat... +Heartbeat running. +Stadium opcodes: 31 resident, hottest opcode 8 +Stadium words: promotions=249 evictions=0 +Stadium conservation: resident_sum=43678 reservoir=21858 sum=65536 (Q48_ONE=65536) +Stadium conservation: CONSERVED +Stadium quota grant self-test: OK + Hera reservoir=21858 + test-vm reservoir=65536 +Heartbeat: 7 ticks, trust=0x00010000, variance=0x00000000 +blkio_usb: MSC device ready +Artemis: the stick's partition attached +[Hera] Zuse: genesis minted onto attached thumbdrive +[zuse@Hera] ok> 3 4 + . +[zuse@Hera] 7 ok +[zuse@Hera] ok> NODE-ME . +[zuse@Hera] 0 ok +[zuse@Hera] ok> 11 2 2 BIRTH . +[zuse@Hera] V4: capsule v4:forth79.4th signature: missing (unsigned) +[zuse@Hera] PARITY:V4_BIRTH node=11 dict_hash=0x74781d08f32ff33a +[zuse@Hera] 1 ok +[zuse@Hera] ok> S" 6 7 * ." 11 SEND 11 AWAIT . +[zuse@Hera] 42 1 ok +[zuse@Hera] ok> 11 KILL +[zuse@Hera] Stadium words: promotions=283 evictions=0 +[zuse@Hera] Stadium conservation: resident_sum=138 reservoir=65398 sum=65536 (Q48_ONE=65536) +[zuse@Hera] Stadium conservation: CONSERVED +[zuse@Hera] ok +[zuse@Hera] ok> 11 2 2 BIRTH . +[zuse@Hera] V4: capsule v4:forth79.4th signature: missing (unsigned) +[zuse@Hera] PARITY:V4_BIRTH node=11 dict_hash=0x74781d08f32ff33a +[zuse@Hera] 1 ok +[zuse@Hera] ok> S" VARIABLE N1 : SPIN BEGIN 1 N1 +! AGAIN ;" 11 SEND 11 AWAIT . +[zuse@Hera] 1 ok +[zuse@Hera] ok> S" SPIN" 11 SEND +[zuse@Hera] ok +[zuse@Hera] ok> 3 4 + . +[zuse@Hera] 7 ok +[zuse@Hera] ok> 11 KILL +[zuse@Hera] Stadium words: promotions=288 evictions=0 +[zuse@Hera] Stadium conservation: resident_sum=184 reservoir=65352 sum=65536 (Q48_ONE=65536) +[zuse@Hera] Stadium conservation: CONSERVED +[zuse@Hera] ok +[zuse@Hera] ok>  \ No newline at end of file diff --git a/logs/20261119-ci-snapshot/README.txt b/logs/20261119-ci-snapshot/README.txt new file mode 100644 index 00000000..011a3e94 --- /dev/null +++ b/logs/20261119-ci-snapshot/README.txt @@ -0,0 +1,20 @@ +The CI aarch64 BIRTH stall, 2026-11-18/19 (Captain Bob: "Measure, then fix"; then "Diagnose on CI"). + +CI's Acceptance has failed on aarch64 at "BIRTH: node 11 born from Hera" on every head since 58d0e887 +(jobs 828, 835, 840, 850): the birth does not finish within the session's 300 s wait. POST passes there. + +Here it does not reproduce: +- host mesh boot: a birth is 0.045 s and 374,589 fabric steps (nucleus 7,515 cells); +- local aarch64 boots (unsigned): the birth in about 11 s, background or foreground + (local-aarch64-birth-timeline.txt; logs/20261118-235613/); +- an instrumented scratch kernel (counters in the mesh's heart()): one ticking node, the heartbeat at + most 0x203 ticks behind once at the prompt and then keeping up -- no catch-up spiral. + +The one known difference: CI signs its capsules and checks each signature at open; a birth checks one +(forth79.4th), about 7 s on CI's aarch64 by its boot's timing -- not 300 s. + +So qemu_session.py now prints where the CPUs are when a check fails (cpu_snapshot: the monitor's +registers three times, a second apart), and CI's next failure will show it. +Test first: scripts/test_qemu_session_snapshot.sh (test-before.log: 3 failures; test-after.log: PASS). +The monitor's register text was checked on real QEMU, paused with no kernel: aarch64 "PC=", riscv64 +"pc", x86 "EIP"/"RIP" -- all matched. diff --git a/logs/20261119-ci-snapshot/local-aarch64-birth-timeline.txt b/logs/20261119-ci-snapshot/local-aarch64-birth-timeline.txt new file mode 100644 index 00000000..6bf7b2cd --- /dev/null +++ b/logs/20261119-ci-snapshot/local-aarch64-birth-timeline.txt @@ -0,0 +1,11 @@ +2 s: [zuse@Hera] ok> +26 s: +371 s: POST: PASSED +377 s: [zuse@Hera] ok> NODE-ME +379 s: [zuse@Hera] ok> 11 2 2 BIRTH . +390 s: [zuse@Hera] ok> +392 s: [zuse@Hera] ok> S" 6 7 * ." 11 SEND 11 AWAIT . +394 s: [zuse@Hera] ok> 11 2 2 BIRTH . +400 s: [zuse@Hera] ok> S" VARIABLE N1 +402 s: [zuse@Hera] ok> S" VARIABLE N1 : SPIN BEGIN 1 N1 +! AGAIN ;" 11 SEND 1 +404 s: [zuse@Hera] ok> diff --git a/logs/20261119-ci-snapshot/test-after.log b/logs/20261119-ci-snapshot/test-after.log new file mode 100644 index 00000000..adc65770 --- /dev/null +++ b/logs/20261119-ci-snapshot/test-after.log @@ -0,0 +1,6 @@ +PASS: qemu_session.py has cpu_snapshot() +PASS: a failed check asks the monitor for the registers three times +PASS: and prints where the CPU is, each time +PASS: a passing check asks the monitor nothing +PASS: a monitor that does not answer does not stop the run +RESULT: PASS diff --git a/logs/20261119-ci-snapshot/test-before.log b/logs/20261119-ci-snapshot/test-before.log new file mode 100644 index 00000000..bcc8898a --- /dev/null +++ b/logs/20261119-ci-snapshot/test-before.log @@ -0,0 +1,6 @@ +FAIL: qemu_session.py has cpu_snapshot() +FAIL: a failed check asks the monitor for the registers three times +FAIL: and prints where the CPU is, each time +PASS: a passing check asks the monitor nothing +PASS: a monitor that does not answer does not stop the run +RESULT: FAIL diff --git a/scripts/qemu_session.py b/scripts/qemu_session.py index 2a8fad0a..fae6cc38 100755 --- a/scripts/qemu_session.py +++ b/scripts/qemu_session.py @@ -349,13 +349,33 @@ def excerpt(near=None, after=20, tail=40): print(" ---", flush=True) +def cpu_snapshot(times=3, apart=1.0): + """Where the CPUs are (Captain Bob, 2026-11-19: "Diagnose on CI"): QEMU's monitor asked for + the registers `times` times, `apart` seconds apart, and the lines that name the program + counter printed -- so a failed check's log shows where the kernel is, and whether it moves. + A monitor that does not answer is said, and the run goes on.""" + for k in range(times): + try: + r = qcmd("human-monitor-command", **{"command-line": "info registers"}) + said = r.get("return", "") if isinstance(r, dict) else str(r) + except Exception as e: # the monitor gone: the run is not stopped by it + print(f" --- the CPUs, {k + 1} of {times}: the monitor did not answer ({e})", flush=True) + return + pcs = [l.strip() for l in said.splitlines() if re.search(r"\b(PC|RIP|EIP|pc)\s*[=:]?\s*[0-9a-fA-F]", l)] + print(f" --- the CPUs, {k + 1} of {times}: " + (" | ".join(pcs) if pcs else said.strip()[:200]), flush=True) + if k + 1 < times: + time.sleep(apart) + + def check(name, cond, near=None): """PASS or FAIL; a FAIL also prints the serial log where the check looked: after the first - line matching `near` (a regex), when it is given, else the log's end.""" + line matching `near` (a regex), when it is given, else the log's end -- and where the CPUs + are (cpu_snapshot).""" global ok print(f"{'PASS' if cond else 'FAIL'}: {name}", flush=True) if not cond: excerpt(near) + cpu_snapshot() ok = ok and cond diff --git a/scripts/test_qemu_session_snapshot.sh b/scripts/test_qemu_session_snapshot.sh new file mode 100755 index 00000000..333f00e6 --- /dev/null +++ b/scripts/test_qemu_session_snapshot.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LicenseRef-Starship-1.0 +# Copyright (c) 2023-2026 Robert A. James +# test_qemu_session_snapshot.sh -- the tests of qemu_session.py's CPU snapshot on a failed check +# (Captain Bob, 2026-11-19: "Diagnose on CI" -- CI's aarch64 BIRTH stall reproduces only there). +# When a check fails, the script asks QEMU's monitor for the CPUs' registers three times, a +# second apart, and prints them, so the job's log shows where the kernel is and whether it moves. +# +# The script boots QEMU when it is run, so the test takes cpu_snapshot() and check() out of it +# (ast) and runs them against a fake monitor. No QEMU. Prints PASS:/FAIL: and RESULT:. +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +python3 - "$ROOT/scripts/qemu_session.py" <<'PY' +import ast, io, sys, contextlib +src = open(sys.argv[1]).read() +tree = ast.parse(src) +want = {"cpu_snapshot", "check"} +funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in want] +ok = True +def report(name, cond): + global ok + print(("PASS: " if cond else "FAIL: ") + name); ok = ok and cond +report("qemu_session.py has cpu_snapshot()", any(f.name == "cpu_snapshot" for f in funcs)) +calls = [] +def qcmd(cmd, **args): + calls.append((cmd, args)) + return {"return": "X0=0000000000000001\nPC=00000000400812a4 X30=0000000040081000 SP=0000000040300000\n"} +import re +g = {"qcmd": qcmd, "re": re, "time": type("T", (), {"sleep": staticmethod(lambda s: None)}), "ok": True, + "excerpt": lambda near=None: None, "print": print} +mod = ast.Module(body=funcs, type_ignores=[]) +try: + exec(compile(mod, "qemu_session.py", "exec"), g) +except Exception as e: + report(f"the functions load ({e})", False) +out = io.StringIO() +with contextlib.redirect_stdout(out): + if "check" in g: g["check"]("a check that fails", False) +said = out.getvalue() +regs = [c for c in calls if c[0] == "human-monitor-command" and c[1].get("command-line") == "info registers"] +report("a failed check asks the monitor for the registers three times", len(regs) == 3) +report(" and prints where the CPU is, each time", said.count("PC=00000000400812a4") == 3) +calls.clear(); out = io.StringIO() +with contextlib.redirect_stdout(out): + if "check" in g: g["check"]("a check that passes", True) +report("a passing check asks the monitor nothing", not calls) +def broken(cmd, **args): raise OSError("monitor gone") +g["qcmd"] = broken; out = io.StringIO() +try: + with contextlib.redirect_stdout(out): + if "check" in g: g["check"]("a check that fails, the monitor gone", False) + report("a monitor that does not answer does not stop the run", True) +except Exception as e: + report(f"a monitor that does not answer does not stop the run ({e})", False) +print("RESULT: PASS" if ok else "RESULT: FAIL") +sys.exit(0 if ok else 1) +PY