diff --git a/scripts/nightly.sh b/scripts/nightly.sh index 3fa7ad23..722bf42a 100755 --- a/scripts/nightly.sh +++ b/scripts/nightly.sh @@ -97,7 +97,7 @@ usage() { echo " protocol stages: $PROTOCOL_STAGES" >&2 exit 2 } -PROTO=build/protocol +PROTO=${NIGHTLY_PROTO:-build/protocol} # NIGHTLY_PROTO: the tests' scratch (scripts/test_nightly_protocol_report.sh) PROTOCOL_STAGES="accept-amd64 accept-aarch64 accept-riscv64 accept-stick-high test-32 proofs images docs" # The tree, but for the evidence the run writes (logs/, the DoE CSVs) and what builds leave behind # untracked: what scripts/check_protocol.sh compares. @@ -434,19 +434,21 @@ protocol-report) boots=() for a in amd64 aarch64 riscv64 stick-high; do log=$(sed -n 's/^LOG: //p' "$OUT/accept-$a.txt" 2>/dev/null | head -1) - post=$( [ -n "$log" ] && grep -aoE "PARITY:V4_POST [^[:cntrl:]]*" "$log" | head -1) - sys=$( [ -n "$log" ] && grep -aoE "PARITY:V4_SYSTEM [^[:cntrl:]]*" "$log" | head -1) + # each lookup may find nothing (a boot that stopped in POST): under set -e that must be an + # answer, "none", not the end of the report (2026-11-17) + post=$( { [ -n "$log" ] && [ -f "$log" ] && grep -aoE "PARITY:V4_POST [^[:cntrl:]]*" "$log" | head -1; } || true) + sys=$( { [ -n "$log" ] && [ -f "$log" ] && grep -aoE "PARITY:V4_SYSTEM [^[:cntrl:]]*" "$log" | head -1; } || true) boots+=("| $a | \`${log:-none}\` | \`${post:-none}\` | \`${sys:-none}\` |") done hashes=$(for a in amd64 aarch64 riscv64; do log=$(sed -n 's/^LOG: //p' "$OUT/accept-$a.txt" 2>/dev/null | head -1) - [ -n "$log" ] && grep -aoE "PARITY:V4_SYSTEM [^[:cntrl:]]*" "$log" | sed -n 's/.*dict_hash=//p' | head -1 + { [ -n "$log" ] && [ -f "$log" ] && grep -aoE "PARITY:V4_SYSTEM [^[:cntrl:]]*" "$log" | sed -n 's/.*dict_hash=//p' | head -1; } || true done) nh=$(grep -c . <<<"$hashes" || true); nu=$(sort -u <<<"$hashes" | grep -c . || true) if [ "$nh" = 3 ] && [ "$nu" = 1 ]; then same="the same on all three: \`$(head -1 <<<"$hashes")\`" else same="**not** one value on three architectures ($nh found, $nu distinct)"; result=FAIL; why+=("dict_hash"); fi - t32=$(grep -E '[0-9]+ checks, [0-9]+ failures' "$OUT/test-32.txt" 2>/dev/null | - awk '{c+=$(NF-3); f+=$(NF-1)} END {printf "%d programs, %d checks, %d failures", NR, c, f}') + t32=$( { [ -f "$OUT/test-32.txt" ] && grep -E '[0-9]+ checks, [0-9]+ failures' "$OUT/test-32.txt" | + awk '{c+=$(NF-3); f+=$(NF-1)} END {printf "%d programs, %d checks, %d failures", NR, c, f}'; } || true) isalog=$(find logs -path '*/isabelle/*.log' -newer "$PROTO/commit" 2>/dev/null | sort | tail -1) { echo "# Protocol for ${commit:0:8}" diff --git a/scripts/test_nightly_protocol_report.sh b/scripts/test_nightly_protocol_report.sh new file mode 100755 index 00000000..53225e49 --- /dev/null +++ b/scripts/test_nightly_protocol_report.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LicenseRef-Starship-1.0 +# Copyright (c) 2023-2026 Robert A. James +# test_nightly_protocol_report.sh -- the tests of `scripts/nightly.sh protocol-report`: it writes +# logs//PROTOCOL.md whatever the boots left, and a boot that stopped before POST's line (no +# PARITY:V4_POST, no PARITY:V4_SYSTEM -- an aarch64 boot out of its 570 s in POST, 2026-11-17) is +# reported as FAIL with "none", not left without a report. No QEMU: each case is a scratch +# NIGHTLY_OUT and NIGHTLY_PROTO with accept-.txt files and serial logs, as the stages leave them. +# +# Prints PASS:/FAIL: per check and RESULT:; exits 0 only if every check passes. +set -uo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NIGHTLY="$ROOT/scripts/nightly.sh" +T=$(mktemp -d) +TS="test-protocol-report-$$" +trap 'rm -rf "$T" "$ROOT/logs/$TS"' EXIT +ok=1 + +setup() { # setup + rm -rf "${T:?}"/*; mkdir -p "$T/out" "$T/proto" "$T/logs" + git -C "$ROOT" rev-parse HEAD > "$T/proto/commit"; echo "$TS" > "$T/proto/ts"; echo clean > "$T/proto/tree" + for a in amd64 aarch64 riscv64 stick-high; do + log="$T/logs/$a.log" + printf 'LOG: %s\nRESULT: PASS\n' "$log" > "$T/out/accept-$a.txt" + if [ "$a" = aarch64 ] && [ "$1" = none ]; then + printf 'PARITY:V4_CAPSULE name=v4:physics.4th\n' > "$log" # stopped in POST + else + printf 'PARITY:V4_POST tests=990 pass=990 fail=0\nPARITY:V4_SYSTEM word_count=403 dict_hash=0xab\nPOST: PASSED\n' > "$log" + fi + done + for s in accept-amd64 accept-aarch64 accept-riscv64 accept-stick-high test-32 proofs images docs; do + echo "output of $s" > "$T/proto/$s.txt"; echo 0 > "$T/proto/$s.rc" + done +} + +report() { (cd "$ROOT" && NIGHTLY_OUT="$T/out" NIGHTLY_PROTO="$T/proto" bash "$NIGHTLY" protocol-report > "$T/report.txt" 2>&1); } +check() { if eval "$2"; then echo "PASS: $1"; else echo "FAIL: $1"; sed 's/^/ /' "$T/report.txt" | head -4; ok=0; fi; } +P="$ROOT/logs/$TS/PROTOCOL.md" + +setup post; report +check "every boot reported: PROTOCOL.md written" '[ -f "$P" ]' +check " it names the four boots' POST lines" '[ "$(grep -c "tests=990 pass=990" "$P" 2>/dev/null)" = 4 ]' + +rm -rf "$ROOT/logs/$TS" +setup none; echo 124 > "$T/proto/accept-aarch64.rc"; report +check "a boot that stopped in POST: PROTOCOL.md still written" '[ -f "$P" ]' +check " it says Result: FAIL" 'grep -q "^- Result: FAIL" "$P" 2>/dev/null' +check " the stopped boot shows none for POST and the system" 'grep -q "^| aarch64 | .* | \`none\` | \`none\` |" "$P" 2>/dev/null' +check " the stage is FAIL (exit 124)" 'grep -q "accept-aarch64\` | FAIL (exit 124)" "$P" 2>/dev/null' +check " dict_hash is not one value on three" 'grep -q "not\*\* one value" "$P" 2>/dev/null' + +if [ $ok = 1 ]; then echo "RESULT: PASS"; else echo "RESULT: FAIL"; fi +[ $ok = 1 ]