LithosAnanke/v4/tools/mkdictionary.py

#!/usr/bin/env python3
# SPDX-License-Identifier: LicenseRef-Starship-1.0
# Copyright (c) 2023-2026 Robert A. James
"""mkdictionary.py -- the dictionary of v4 (#157): every word the system boots with, alphabetical,
with its stack notation, a description, and where it is defined.

    v4/tools/mkdictionary.py words              the live word list, one per line
    v4/tools/mkdictionary.py draft TSV          a table for every live word not in TSV yet, appended,
                                                its stack and description taken from the source where
                                                the source has them (to be read and finished by hand)
    v4/tools/mkdictionary.py check TSV          every live word has an entry with a stack and a
                                                description, and every entry is a live word: 0, or 1
                                                with what is wrong
    v4/tools/mkdictionary.py md TSV OUT.md      the dictionary as Markdown, alphabetical, for the book

The live word list is the system's own: the hosted v4 (`make -C v4 hosted`) booted and asked
`WORDS`, plus the kernel's words of the lone node (`sk_v4_words[]` in kernel/src/v4/sk_v4.c) --
the words a booted StarshipOS has. The table (docs/reference/v4-dictionary.tsv) holds the stack
notation and the description, which only a person can write; the names are never typed by hand.
"""
import os, re, subprocess, sys

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
HOSTED = os.path.join(ROOT, "v4", "build", "starforth4-amd64")
SOURCES = ([os.path.join("capsules/v4/sys", f) for f in sorted(os.listdir(os.path.join(ROOT, "capsules/v4/sys")))
            if f.endswith(".f18")] +
           [os.path.join("capsules/v4", f) for f in sorted(os.listdir(os.path.join(ROOT, "capsules/v4")))
            if f.endswith(".4th")])
KERNEL_WORDS_SRC = "kernel/src/v4/sk_v4.c"


def live_words():
    """The hosted system's WORDS, then the kernel's own words not among them."""
    if not os.path.exists(HOSTED):
        sys.exit(f"mkdictionary: {HOSTED} is missing: make -C v4 hosted")
    out = subprocess.run([HOSTED], input="WORDS\n", capture_output=True, text=True, timeout=120).stdout
    lines = out.split("\n")
    try:
        start = next(k for k, l in enumerate(lines) if l.startswith("ok> ") and l[4:].strip())
    except StopIteration:
        sys.exit("mkdictionary: the hosted system printed no WORDS")
    got = []
    for l in lines[start:]:
        if l.startswith("ok> ") and got:
            break
        got.append(l[4:] if l.startswith("ok> ") else l)
    text = " ".join(got)
    if " ok" in text:
        text = text[:text.rfind(" ok")]
    words = text.split()
    src = open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read()
    m = re.search(r"sk_v4_words\[\]\s*=\s*\{([^}]*)\}", src)
    for k in re.findall(r'"([^"]+)"', m.group(1) if m else ""):
        if k not in words:
            words.append(k)
    return words


def definitions():
    """Where each name is defined, and what its source says of it: {name: (file, stack, text)}."""
    found = {}
    for rel in SOURCES:
        lines = open(os.path.join(ROOT, rel), errors="replace").read().split("\n")
        for i, l in enumerate(lines):
            names = []
            if rel.endswith(".f18"):
                names = re.findall(r'",\s+(\S+)\s+#,', l)
            else:
                names = re.findall(r"(?:^|\s):\s+(\S+)", l)
                names += re.findall(r"\b(?:CONSTANT|VARIABLE|DEFER|CREATE|VOCABULARY)\s+(\S+)", l)
            for n in names:
                if n in found:
                    continue
                stack, text = "", ""
                inline = re.search(r":\s+" + re.escape(n) + r"\s+(\([^)]*\))", l)
                if inline:
                    stack = inline.group(1)
                # the comment block just above
                j, block = i - 1, []
                while j >= 0 and lines[j].startswith("\\"):
                    block.insert(0, lines[j].lstrip("\\").strip())
                    j -= 1
                comment = " ".join(b for b in block if b)
                cm = re.match(r"(\([^)]*--[^)]*\))\s*(.*)", comment)
                if cm and not stack:
                    stack, comment = cm.group(1), cm.group(2)
                text = comment
                found[n] = (rel, stack, text)
    for k in re.findall(r'"([^"]+)"', re.search(r"sk_v4_words\[\]\s*=\s*\{([^}]*)\}",
                                                open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read()).group(1)):
        found.setdefault(k, (KERNEL_WORDS_SRC, "", ""))
    return found


def read_table(path):
    table = {}
    if os.path.exists(path):
        for l in open(path).read().split("\n"):
            # a comment is a line beginning with # that has no tab: a row always has tabs, and
            # some words begin with # (#, #>, #S)
            if not l.strip() or (l.startswith("#") and "\t" not in l):
                continue
            f = l.split("\t")
            while len(f) < 3:
                f.append("")
            table[f[0]] = (f[1].strip(), f[2].strip())
    return table


def main():
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    mode = sys.argv[1]
    if mode == "words":
        print("\n".join(live_words()))
        return
    path = sys.argv[2]
    table = read_table(path)
    words = live_words()
    if mode == "draft":
        defs = definitions()
        new = [w for w in words if w not in table]
        with open(path, "a") as f:
            if not table:
                f.write("# docs/reference/v4-dictionary.tsv -- the dictionary of v4 (#157): name, stack notation,\n"
                        "# description, tab-separated. The names are the live system's (v4/tools/mkdictionary.py\n"
                        "# check refuses a missing or a stale one). A row whose description starts TODO is a draft.\n")
            for w in sorted(new):
                rel, stack, text = defs.get(w, ("", "", ""))
                desc = ("TODO " + text) if text else "TODO"
                f.write(f"{w}\t{stack}\t{desc}\n")
        print(f"mkdictionary: {len(words)} live words; {len(new)} drafted into {path}")
        return
    if mode == "check":
        bad = []
        for w in words:
            if w not in table:
                bad.append(f"no entry: {w}")
            elif not table[w][0] or not table[w][1] or table[w][1].startswith("TODO"):
                bad.append(f"unfinished: {w}")
        for w in table:
            if w not in words:
                bad.append(f"not a live word: {w}")
        if bad:
            print("\n".join(bad))
            print(f"mkdictionary: {len(bad)} problem(s) in {path} ({len(words)} live words)")
            sys.exit(1)
        print(f"mkdictionary: {path} covers all {len(words)} live words")
        return
    if mode == "md":
        defs = definitions()
        out = sys.argv[3]
        with open(out, "w") as f:
            f.write("# The dictionary of v4\n\n")
            f.write(f"Every word StarForth v4 boots with -- {len(words)} of them -- in alphabetical order: its "
                    "stack notation (what it takes and leaves: `( before -- after )`), what it does, and "
                    "where it is defined.\n\n")
            f.write("| Word | Stack | Description | Defined in |\n|---|---|---|---|\n")
            for w in sorted(words, key=lambda x: (x.upper(), x)):
                stack, desc = table.get(w, ("", ""))
                rel = defs.get(w, ("", "", ""))[0]
                cell = lambda x: x.replace("|", "\\|")
                # a token with a quote in it is FORTH (." text" S" ABORT"): set as code, so its
                # quotes print straight, not as typographic ones
                desc = re.sub(r'(\S*"\S*)', lambda mm: "`" + mm.group(1) + "`", desc)
                f.write(f"| `{cell(w)}` | `{cell(stack)}` | {cell(desc)} | `{rel}` |\n")
        print(f"mkdictionary: {out}, {len(words)} words")
        return
    sys.exit(__doc__)


if __name__ == "__main__":
    main()