qemu_session.py: a failed check prints where the CPUs are (cpu_snapshot)

CI's Acceptance fails on aarch64 at BIRTH on every head since 58d0e887.
The birth takes more than 300 s there and does not reproduce here: about
11 s, unsigned. The heartbeat spiral was ruled out by an instrumented
scratch kernel.

Captain Bob ruled "Diagnose on CI". On a failed check, the session script
asks QEMU's monitor for the registers three times, a second apart, and
prints the program counters, so CI's next failure shows where the kernel
is and whether it moves. A monitor that does not answer does not stop
the run.

Test first: scripts/test_qemu_session_snapshot.sh, 5 checks; 3 failed
before. The register text was checked on real, paused QEMU for aarch64,
riscv64 and x86.

The evidence and the measurements are in logs/20261119-ci-snapshot. The
local aarch64 timing run is in logs/20261118-235613.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017RzZgrf7V7Zk2iczBXDRbm
This commit is contained in:
Claude
2026-11-19 00:29:23 +00:00
parent c2e42c3cba
commit 36b258054c
8 changed files with 300 additions and 1 deletions
+21 -1
View File
@@ -349,13 +349,33 @@ def excerpt(near=None, after=20, tail=40):
print(" ---", flush=True)
def cpu_snapshot(times=3, apart=1.0):
"""Where the CPUs are (Captain Bob, 2026-11-19: "Diagnose on CI"): QEMU's monitor asked for
the registers `times` times, `apart` seconds apart, and the lines that name the program
counter printed -- so a failed check's log shows where the kernel is, and whether it moves.
A monitor that does not answer is said, and the run goes on."""
for k in range(times):
try:
r = qcmd("human-monitor-command", **{"command-line": "info registers"})
said = r.get("return", "") if isinstance(r, dict) else str(r)
except Exception as e: # the monitor gone: the run is not stopped by it
print(f" --- the CPUs, {k + 1} of {times}: the monitor did not answer ({e})", flush=True)
return
pcs = [l.strip() for l in said.splitlines() if re.search(r"\b(PC|RIP|EIP|pc)\s*[=:]?\s*[0-9a-fA-F]", l)]
print(f" --- the CPUs, {k + 1} of {times}: " + (" | ".join(pcs) if pcs else said.strip()[:200]), flush=True)
if k + 1 < times:
time.sleep(apart)
def check(name, cond, near=None):
"""PASS or FAIL; a FAIL also prints the serial log where the check looked: after the first
line matching `near` (a regex), when it is given, else the log's end."""
line matching `near` (a regex), when it is given, else the log's end -- and where the CPUs
are (cpu_snapshot)."""
global ok
print(f"{'PASS' if cond else 'FAIL'}: {name}", flush=True)
if not cond:
excerpt(near)
cpu_snapshot()
ok = ok and cond
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: LicenseRef-Starship-1.0
# Copyright (c) 2023-2026 Robert A. James
# test_qemu_session_snapshot.sh -- the tests of qemu_session.py's CPU snapshot on a failed check
# (Captain Bob, 2026-11-19: "Diagnose on CI" -- CI's aarch64 BIRTH stall reproduces only there).
# When a check fails, the script asks QEMU's monitor for the CPUs' registers three times, a
# second apart, and prints them, so the job's log shows where the kernel is and whether it moves.
#
# The script boots QEMU when it is run, so the test takes cpu_snapshot() and check() out of it
# (ast) and runs them against a fake monitor. No QEMU. Prints PASS:/FAIL: and RESULT:.
set -uo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
python3 - "$ROOT/scripts/qemu_session.py" <<'PY'
import ast, io, sys, contextlib
src = open(sys.argv[1]).read()
tree = ast.parse(src)
want = {"cpu_snapshot", "check"}
funcs = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in want]
ok = True
def report(name, cond):
global ok
print(("PASS: " if cond else "FAIL: ") + name); ok = ok and cond
report("qemu_session.py has cpu_snapshot()", any(f.name == "cpu_snapshot" for f in funcs))
calls = []
def qcmd(cmd, **args):
calls.append((cmd, args))
return {"return": "X0=0000000000000001\nPC=00000000400812a4 X30=0000000040081000 SP=0000000040300000\n"}
import re
g = {"qcmd": qcmd, "re": re, "time": type("T", (), {"sleep": staticmethod(lambda s: None)}), "ok": True,
"excerpt": lambda near=None: None, "print": print}
mod = ast.Module(body=funcs, type_ignores=[])
try:
exec(compile(mod, "qemu_session.py", "exec"), g)
except Exception as e:
report(f"the functions load ({e})", False)
out = io.StringIO()
with contextlib.redirect_stdout(out):
if "check" in g: g["check"]("a check that fails", False)
said = out.getvalue()
regs = [c for c in calls if c[0] == "human-monitor-command" and c[1].get("command-line") == "info registers"]
report("a failed check asks the monitor for the registers three times", len(regs) == 3)
report(" and prints where the CPU is, each time", said.count("PC=00000000400812a4") == 3)
calls.clear(); out = io.StringIO()
with contextlib.redirect_stdout(out):
if "check" in g: g["check"]("a check that passes", True)
report("a passing check asks the monitor nothing", not calls)
def broken(cmd, **args): raise OSError("monitor gone")
g["qcmd"] = broken; out = io.StringIO()
try:
with contextlib.redirect_stdout(out):
if "check" in g: g["check"]("a check that fails, the monitor gone", False)
report("a monitor that does not answer does not stop the run", True)
except Exception as e:
report(f"a monitor that does not answer does not stop the run ({e})", False)
print("RESULT: PASS" if ok else "RESULT: FAIL")
sys.exit(0 if ok else 1)
PY