diff --git a/Makefile b/Makefile index 510dccdb..67f1eff9 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,12 @@ # three, for the full acceptance. syft and # Isabelle/HOL, not packaged, are downloaded, # checked by SHA-256 and linked into -# /usr/local/bin (build/cache/tools) +# /usr/local/bin (build/cache/tools). +# INIT_DOCS=0: without the documents' and +# proofs' tools (TeX, Doxygen, Isabelle) +# scripts/nightly.sh boot | images | publish +# the nightly snapshot: boot-check, the +# board images, and Nexus # make goes to v3/Makefile (make help-hosted) # ============================================================================== @@ -252,8 +257,12 @@ write_stick: # tools in build/cache/tools, which make clean keeps; each is linked into /usr/local/bin. syft's # sums are its release's checksums.txt; Isabelle publishes none, so its sums are of the archives # as downloaded on 2026-11-06 -- a download that differs is refused. -INIT_COMMON := build-essential make git git-lfs python3 curl ca-certificates xorriso mtools gdisk dosfstools fdisk \ - pandoc latexmk texlive-xetex texlive-latex-base texlive-latex-recommended \ +# INIT_DOCS=0 leaves out what only the documents and the proofs need (the TeX and Doxygen +# packages, and Isabelle): what a build of the images needs, as the nightly (scripts/nightly.sh). +INIT_DOCS ?= 1 +INIT_COMMON := build-essential make git git-lfs python3 curl ca-certificates xz-utils \ + xorriso mtools gdisk dosfstools fdisk +INIT_DOCPKGS := pandoc latexmk texlive-xetex texlive-latex-base texlive-latex-recommended \ texlive-latex-extra texlive-pictures texlive-fonts-recommended lmodern \ fonts-dejavu fonts-dejavu-extra doxygen graphviz INIT_amd64 := qemu-system-x86 ovmf @@ -301,20 +310,26 @@ init: echo "init: needs apt-get (Debian/Ubuntu). The packages are listed in the Makefile, at init."; exit 1; } $(SUDO) env DEBIAN_FRONTEND=noninteractive apt-get update -qq $(SUDO) env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ - $(INIT_COMMON) $(foreach a,$(INIT_ARCHES),$(INIT_$(a))) + $(INIT_COMMON) $(if $(filter 0,$(INIT_DOCS)),,$(INIT_DOCPKGS)) $(foreach a,$(INIT_ARCHES),$(INIT_$(a))) git lfs install --local $(call fetch,$(SYFT_TGZ),$(SYFT_URL),$(SYFT_SHA_$(SYFT_ARCH))) @mkdir -p $(INIT_TOOLS)/syft-$(SYFT_VERSION) tar -xzf $(INIT_DL)/$(SYFT_TGZ) -C $(INIT_TOOLS)/syft-$(SYFT_VERSION) syft $(SUDO) ln -sf $(abspath $(INIT_TOOLS))/syft-$(SYFT_VERSION)/syft /usr/local/bin/syft + @syft version | sed -n 's/^Version: */ syft /p' +ifneq ($(INIT_DOCS),0) $(call fetch,$(ISABELLE_TGZ),$(ISABELLE_URL),$(ISABELLE_SHA_$(patsubst $(ISABELLE)_%.tar.gz,%,$(ISABELLE_TGZ)))) @[ -x $(INIT_TOOLS)/$(ISABELLE)/bin/isabelle ] || { mkdir -p $(INIT_TOOLS) && tar -xzf $(INIT_DL)/$(ISABELLE_TGZ) -C $(INIT_TOOLS); } $(SUDO) ln -sf $(abspath $(INIT_TOOLS))/$(ISABELLE)/bin/isabelle /usr/local/bin/isabelle - @syft version | sed -n 's/^Version: */ syft /p' @echo " isabelle $$(isabelle version)" @echo "" @echo "Ready for $(INIT_ARCHES): make ARCH= clean qemu, make -C v4 test, make docs," @echo " make sbom, isabelle build -D proof/" +else + @echo "" + @echo "Ready for $(INIT_ARCHES), without the documents' and proofs' tools: make ARCH= clean qemu," + @echo " make boot_image TARGET=, make sbom" +endif help: @sed -n '/^# TARGET is/,/^# =====/p' Makefile | sed '$$d; s/^# \{0,1\}//' diff --git a/scripts/nightly.sh b/scripts/nightly.sh new file mode 100755 index 00000000..8d6a6b6c --- /dev/null +++ b/scripts/nightly.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# nightly.sh -- the nightly snapshot: boot-check each ISA, build the board images, publish them. +# Three stages, run in this order (.gitea/workflows/nightly.yml runs each as a step): +# +# scripts/nightly.sh boot +# Boots that ISA headless in QEMU (scripts/qemu_session.py headless: POST: PASSED, and no +# prompt with no Zuse drive), in the foreground; one QEMU at a time, the acceptance order +# amd64, aarch64, riscv64. A failure ends the nightly. amd64 also keeps the ISO it booted, +# byte for byte, as build/nightly/StarshipOS.iso. +# scripts/nightly.sh images +# The board images, into build/nightly/: +# raspi5.img.xz Raspberry Pi 5 (make boot_image TARGET=RASPI), for Raspberry Pi Imager +# MilkV.img.xz Milk-V Mars (make boot_image TARGET=MILKV) +# SHA256SUMS of StarshipOS.iso and both images, before and after compression +# (Imager's extract_sha256 is the uncompressed one) +# QEMU boots the aarch64 and riscv64 kernels, not these images: they are built from the +# same sources, for hardware QEMU does not model. +# scripts/nightly.sh publish +# Deploys the three to a Maven repository (Nexus's starshipos-snapshots) as +# org.starship:kernelImage:-SNAPSHOT, one classifier per board -- +# pc (iso), raspi5 (img.xz), milkv (img.xz) -- in one deploy, so they share a timestamp. +# NEXUS_USER (default ci-publisher) and NEXUS_PASSWORD. Needs mvn and a JRE. +# +# NIGHTLY_VERSION overrides the version. The serial logs are logs/// and the DoE CSVs +# experiments/bare_metal/runs/, as for any boot (.claude/CLAUDE.md). Needs what +# make ARCH=amd64,aarch64,riscv64 init INIT_DOCS=0 installs. +set -euo pipefail +cd "$(dirname "$0")/.." + +OUT=build/nightly +LITHOS=$(sed -n 's/^LITHOS_VERSION *?= *//p' kernel/Makefile) +VERSION=${NIGHTLY_VERSION:-$LITHOS-SNAPSHOT} +usage() { echo "Usage: scripts/nightly.sh boot | images | publish " >&2; exit 2; } + +case ${1:-} in +boot) + arch=${2:-} + case $arch in amd64|aarch64|riscv64) ;; *) usage ;; esac + [ "$arch" != amd64 ] || { rm -rf "$OUT"; mkdir -p "$OUT"; } # amd64 is the first stage + echo "=== nightly: org.starship:kernelImage:$VERSION from $(git rev-parse --short HEAD); boot check $arch" + git checkout -q -- disk/zuse-scratch # the committed Zuse pair, as before each acceptance boot + python3 scripts/qemu_session.py headless "$arch" --label "nightly-$arch" --make "QEMU_DISPLAY=none" + [ "$arch" != amd64 ] || cp build/amd64/kernel/starkernel.iso "$OUT/StarshipOS.iso" + ;; +images) + [ -f "$OUT/StarshipOS.iso" ] || { echo "nightly: no $OUT/StarshipOS.iso -- run boot amd64 first" >&2; exit 1; } + for b in "RASPI raspi/lithos-raspi.img raspi5.img" "MILKV milkv/lithos-milkv.img MilkV.img"; do + set -- $b + echo "=== nightly: image $1" + make --no-print-directory boot_image TARGET="$1" + cp "build/boards/$2" "$OUT/$3" + done + # boot_image regenerates these (the source SBOM, and what qemu_session.py also puts back); + # the nightly does not change the tree + git checkout -q -- sbom.spdx sbom.spdx.json capsules/BLOCK_MAP.md v3/include/version.h + (cd "$OUT" && rm -f raspi5.img.xz MilkV.img.xz && sha256sum StarshipOS.iso raspi5.img MilkV.img > SHA256SUMS) + xz -T0 -6 "$OUT/raspi5.img" "$OUT/MilkV.img" + (cd "$OUT" && sha256sum raspi5.img.xz MilkV.img.xz >> SHA256SUMS && cat SHA256SUMS) + ;; +publish) + url=${2:-} + [ -n "$url" ] || usage + : "${NEXUS_PASSWORD:?publish needs NEXUS_PASSWORD}" + command -v mvn >/dev/null || { echo "nightly: publish needs mvn (apt-get install maven)" >&2; exit 1; } + for f in StarshipOS.iso raspi5.img.xz MilkV.img.xz; do + [ -f "$OUT/$f" ] || { echo "nightly: no $OUT/$f -- run boot and images first" >&2; exit 1; } + done + settings=$(mktemp) + trap 'rm -f "$settings"' EXIT + cat > "$settings" <<'EOF' + + starship-nexus + ${env.NEXUS_USER} + ${env.NEXUS_PASSWORD} + +EOF + export NEXUS_USER=${NEXUS_USER:-ci-publisher} + echo "=== nightly: publish org.starship:kernelImage:$VERSION -> $url" + mvn -B -q -s "$settings" org.apache.maven.plugins:maven-deploy-plugin:3.1.2:deploy-file \ + -Durl="$url" -DrepositoryId=starship-nexus \ + -DgroupId=org.starship -DartifactId=kernelImage -Dversion="$VERSION" \ + -Dfile="$OUT/StarshipOS.iso" -Dpackaging=iso -Dclassifier=pc \ + -Dfiles="$OUT/raspi5.img.xz,$OUT/MilkV.img.xz" -Dclassifiers=raspi5,milkv -Dtypes=img.xz,img.xz \ + -DgeneratePom=true + echo "=== nightly: published" + ;; +*) usage ;; +esac