Files
LithosAnanke/v4/tools/mkdictionary.py
T
Claude 6dcfbe6740 Capsule blocks steps 3+4 for v4: capsules are placed in RAM and load their lowest block
Captain Bob ruled: Block n lands at n-R; the lowest block is
interpreted; chaining is the developer's; together, v4 first.
- v4_blocks_place_capsule copies each Block n into RAM block n - R as 16
  lines of 64, space-padded, in any file order. A later block of the
  same number overwrites. It refuses a block below R.
- The mesh's load(), its capsule reader (CAPSULE-OPEN/(LINES), so a
  child's forth79.4th via BIRTH too), boot.c and test_host_unit's model
  each place a capsule, then interpret "<lowest> LOAD".
- mkdictionary.py skips --> when it types capsules at the hosted prompt.
- The v4 capsules are renumbered into R..2R-1, landing at RAM 1900 and
  up, away from POST's blocks 0-5. Each is chained with -->, and
  centre.4th's gap at 8103 is closed.

Test first:
- test_host_blocks.c gained 6 placement checks; it failed to build
  before.
- test_mesh_boot.c checks forth79's lowest block in RAM block 1900; it
  failed before.

After: POST is 990/990; all 52 tests at 64 bits, test-32, hosted-check
and ASan/UBSan pass; the amd64 kernel builds; the dictionary has 404
words.

Logs are in logs/20261118-capsule-place.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017RzZgrf7V7Zk2iczBXDRbm
2026-11-18 20:33:21 +00:00

184 lines
8.8 KiB
Python

#!/usr/bin/env python3
# SPDX-License-Identifier: LicenseRef-Starship-1.0
# Copyright (c) 2023-2026 Robert A. James
"""mkdictionary.py -- the dictionary of v4 (#157): every word the system boots with, alphabetical,
with its stack notation, a description, and where it is defined.
v4/tools/mkdictionary.py words the live word list, one per line
v4/tools/mkdictionary.py draft TSV a table for every live word not in TSV yet, appended,
its stack and description taken from the source where
the source has them (to be read and finished by hand)
v4/tools/mkdictionary.py check TSV every live word has an entry with a stack and a
description, and every entry is a live word: 0, or 1
with what is wrong
v4/tools/mkdictionary.py md TSV OUT.md the dictionary as Markdown, alphabetical, for the book
The live word list is the system's own: the hosted v4 (`make -C v4 hosted`) booted and asked
`WORDS` with the kernel's own capsules loaded (`sk_v4_capsules[]`), plus the kernel's words
(`sk_v4_words[]`), both in kernel/src/v4/sk_v4.c -- the words a booted StarshipOS has. The table (docs/reference/v4-dictionary.tsv) holds the stack
notation and the description, which only a person can write; the names are never typed by hand.
"""
import os, re, subprocess, sys
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
HOSTED = os.path.join(ROOT, "v4", "build", "starforth4-amd64")
SOURCES = ([os.path.join("capsules/v4/sys", f) for f in sorted(os.listdir(os.path.join(ROOT, "capsules/v4/sys")))
if f.endswith(".f18")] +
[os.path.join("capsules/v4", f) for f in sorted(os.listdir(os.path.join(ROOT, "capsules/v4")))
if f.endswith(".4th")])
KERNEL_WORDS_SRC = "kernel/src/v4/sk_v4.c"
def kernel_capsules():
"""The kernel's own capsules (sk_v4_capsules[] in kernel/src/v4/sk_v4.c), as files of capsules/v4."""
src = open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read()
m = re.search(r"sk_v4_capsules\[\]\s*=\s*\{([^}]*)\}", src)
return [os.path.join("capsules/v4", n.split(":", 1)[1]) for n in re.findall(r'"([^"]+)"', m.group(1) if m else "")]
def live_words():
"""The hosted system's WORDS, with the kernel's own capsules loaded first, as the kernel loads
them (#189: centre.4th, hera.4th); then the kernel's own words not among them."""
if not os.path.exists(HOSTED):
sys.exit(f"mkdictionary: {HOSTED} is missing: make -C v4 hosted")
feed = []
for path in kernel_capsules():
for l in open(os.path.join(ROOT, path)):
# every block, in order, typed at the prompt: a block's header is not FORTH, and its `-->`
# (which chains to the next block when the kernel LOADs it) is an error at the terminal
if not re.match(r"^Block \d+\s*$", l) and l.strip() != "-->":
feed.append(l.rstrip("\n"))
feed.append('." [[WORDS]]" CR WORDS')
out = subprocess.run([HOSTED], input="\n".join(feed) + "\n", capture_output=True, text=True, timeout=120).stdout
if "ERROR" in out.split("[[WORDS]]")[0]:
sys.exit("mkdictionary: a kernel capsule did not load in the hosted system")
if "[[WORDS]]" not in out:
sys.exit("mkdictionary: the hosted system printed no WORDS")
text = out.split("[[WORDS]]", 1)[1].replace("ok>", " ")
if " ok" in text:
text = text[:text.rfind(" ok")]
words = text.split()
src = open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read()
m = re.search(r"sk_v4_words\[\]\s*=\s*\{([^}]*)\}", src)
for k in re.findall(r'"([^"]+)"', m.group(1) if m else ""):
if k not in words:
words.append(k)
return words
def definitions():
"""Where each name is defined, and what its source says of it: {name: (file, stack, text)}."""
found = {}
for rel in SOURCES:
lines = open(os.path.join(ROOT, rel), errors="replace").read().split("\n")
for i, l in enumerate(lines):
names = []
if rel.endswith(".f18"):
names = re.findall(r'",\s+(\S+)\s+#,', l)
else:
names = re.findall(r"(?:^|\s):\s+(\S+)", l)
names += re.findall(r"\b(?:CONSTANT|VARIABLE|DEFER|CREATE|VOCABULARY)\s+(\S+)", l)
for n in names:
if n in found:
continue
stack, text = "", ""
inline = re.search(r":\s+" + re.escape(n) + r"\s+(\([^)]*\))", l)
if inline:
stack = inline.group(1)
# the comment block just above
j, block = i - 1, []
while j >= 0 and lines[j].startswith("\\"):
block.insert(0, lines[j].lstrip("\\").strip())
j -= 1
comment = " ".join(b for b in block if b)
cm = re.match(r"(\([^)]*--[^)]*\))\s*(.*)", comment)
if cm and not stack:
stack, comment = cm.group(1), cm.group(2)
text = comment
found[n] = (rel, stack, text)
for k in re.findall(r'"([^"]+)"', re.search(r"sk_v4_words\[\]\s*=\s*\{([^}]*)\}",
open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read()).group(1)):
found.setdefault(k, (KERNEL_WORDS_SRC, "", ""))
return found
def read_table(path):
table = {}
if os.path.exists(path):
for l in open(path).read().split("\n"):
# a comment is a line beginning with # that has no tab: a row always has tabs, and
# some words begin with # (#, #>, #S)
if not l.strip() or (l.startswith("#") and "\t" not in l):
continue
f = l.split("\t")
while len(f) < 3:
f.append("")
table[f[0]] = (f[1].strip(), f[2].strip())
return table
def main():
if len(sys.argv) < 2:
sys.exit(__doc__)
mode = sys.argv[1]
if mode == "words":
print("\n".join(live_words()))
return
path = sys.argv[2]
table = read_table(path)
words = live_words()
if mode == "draft":
defs = definitions()
new = [w for w in words if w not in table]
with open(path, "a") as f:
if not table:
f.write("# docs/reference/v4-dictionary.tsv -- the dictionary of v4 (#157): name, stack notation,\n"
"# description, tab-separated. The names are the live system's (v4/tools/mkdictionary.py\n"
"# check refuses a missing or a stale one). A row whose description starts TODO is a draft.\n")
for w in sorted(new):
rel, stack, text = defs.get(w, ("", "", ""))
desc = ("TODO " + text) if text else "TODO"
f.write(f"{w}\t{stack}\t{desc}\n")
print(f"mkdictionary: {len(words)} live words; {len(new)} drafted into {path}")
return
if mode == "check":
bad = []
for w in words:
if w not in table:
bad.append(f"no entry: {w}")
elif not table[w][0] or not table[w][1] or table[w][1].startswith("TODO"):
bad.append(f"unfinished: {w}")
for w in table:
if w not in words:
bad.append(f"not a live word: {w}")
if bad:
print("\n".join(bad))
print(f"mkdictionary: {len(bad)} problem(s) in {path} ({len(words)} live words)")
sys.exit(1)
print(f"mkdictionary: {path} covers all {len(words)} live words")
return
if mode == "md":
defs = definitions()
out = sys.argv[3]
with open(out, "w") as f:
f.write("# The dictionary of v4\n\n")
f.write(f"Every word StarForth v4 boots with -- {len(words)} of them -- in alphabetical order: its "
"stack notation (what it takes and leaves: `( before -- after )`), what it does, and "
"where it is defined.\n\n")
f.write("| Word | Stack | Description | Defined in |\n|---|---|---|---|\n")
for w in sorted(words, key=lambda x: (x.upper(), x)):
stack, desc = table.get(w, ("", ""))
rel = defs.get(w, ("", "", ""))[0]
cell = lambda x: x.replace("|", "\\|")
# a token with a quote in it is FORTH (." text" S" ABORT"): set as code, so its
# quotes print straight, not as typographic ones
desc = re.sub(r'(\S*"\S*)', lambda mm: "`" + mm.group(1) + "`", desc)
f.write(f"| `{cell(w)}` | `{cell(stack)}` | {cell(desc)} | `{rel}` |\n")
print(f"mkdictionary: {out}, {len(words)} words")
return
sys.exit(__doc__)
if __name__ == "__main__":
main()