admin/LithosAnanake
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage S-1a: parity hashing covers the whole dictionary; failures are logged, counted and marked INCOMPLETE (FABRIC-3.9.0 §8.1)

Browse Source 
- MAX_DICT_ENTRIES (2048) removed: the entry list is allocated to the
  dictionary's size; a cyclic chain is detected (Floyd) instead of capped.
- Colon-body walk bounded by vm->here, not a 1024-cell cap.
- Every failure prints PARITY:INCOMPLETE #n with its full reason, logs at
  LOG_ERROR, bumps sk_parity_incomplete_count(), and the record reads
  hash=INCOMPLETE / PARITY:FAIL code=4 rather than a plausible hash.
- Pre-existing #GP fixed: the walker dereferenced inline body data (S"
  bytes, offsets); it now follows only known dictionary entries.
- LIT operands that are entry pointers are hashed by word_id, not address,
  so a mature dictionary hashes identically across ISAs.

Probe (reverted): 563 -> 2563 words hashed, identical on amd64/aarch64/
riscv64; cyclic chain -> INCOMPLETE, counter 2. Boot parity unchanged
(531, 0x572daac83b797332). 3-ISA acceptance passes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Robert Allan James
2026-09-24 12:51:34 -04:00
co-authored by Claude Opus 5.5
parent 9bf2e8ec95
commit ee3db96ea8
35 changed files with 164134 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/starkernel/vm/parity.c
+196 -31
View File
@@ -54,12 +54,19 @@
#include "starkernel/vm/parity.h"
#include "vm.h"
#include "log.h" /* log_message(LOG_ERROR) -- S-1a: failures are never silent */
#ifdef __STARKERNEL__
#include "console.h"
#include "starkernel/hal/hal.h"
#include "starkernel/kmalloc.h" /* S-1a: the entry list is sized to the dictionary */
#define PARITY_ALLOC(n) kmalloc(n)
#define PARITY_FREE(p) kfree(p)
#else
#include <stdio.h>
#include <stdlib.h>
#define PARITY_ALLOC(n) malloc(n)
#define PARITY_FREE(p) free(p)
#endif
#include <stdbool.h>
@@ -118,8 +125,61 @@ static void sk_parity_debug_check_ptr(struct VM *vm,
do { (void)(vm); (void)(label); (void)(entry); (void)(idx); (void)(hptr); (void)(xt); (void)(ptr); } while (0)
#endif
/* Maximum dictionary entries for traversal array */
#define MAX_DICT_ENTRIES 2048
/* FABRIC-3.9.0.md §8.1 (Stage S-1a, 2026-09-24): MAX_DICT_ENTRIES (2048) is
* gone. The walk used to stop at 2048 entries -- a dictionary larger than
* that got a canonical hash that silently ignored the rest. The walk now
* covers the whole chain; a cyclic (corrupt) chain, the guard the cap stood
* in for, is detected directly (dict_walk_count() below), and every failure
* is logged, counted and marked INCOMPLETE rather than returned as a hash. */
static uint32_t g_parity_incomplete_count;
uint32_t sk_parity_incomplete_count(void) { return g_parity_incomplete_count; }
static void parity_incomplete(const char *why) {
g_parity_incomplete_count++;
/* The full reason, straight to the console: log_message()'s own line
* buffer is LOG_LINE_MAX (64) bytes, which cut every reason here off
* mid-sentence. The log line stays too, for the persisted record. */
print_str("PARITY:INCOMPLETE #");
print_u32(g_parity_incomplete_count);
print_str(" -- ");
print_str(why);
print_nl();
log_message(LOG_ERROR, "PARITY: INCOMPLETE -- %s", why);
}
/* S-1a finding (2026-09-24): the colon-body walker used to treat EVERY
* cell of a body as a DictEntry* and dereference it. Only LIT's operand
* was known to be data; S" string bytes, branch and loop offsets and other
* inline operands were dereferenced too. Harmless while parity only ever
* ran at early boot (no colon bodies yet), but the first hash of a mature
* dictionary took a #GP -- reproduced on the ORIGINAL code
* (logs/20260924-095730/amd64/), so it predates S-1a. The walker now
* dereferences a cell only if it is a known dictionary entry, looked up in
* a sorted copy of the entry list; any other cell is hashed as its raw
* value and never followed. */
static void parity_sort_entries(DictEntry **a, uint32_t n) {
uint32_t gap, i, j;
for (gap = n / 2; gap > 0; gap /= 2) { /* shell sort -- no libc qsort */
for (i = gap; i < n; i++) {
DictEntry *t = a[i];
for (j = i; j >= gap && (uintptr_t)a[j - gap] > (uintptr_t)t; j -= gap) {
a[j] = a[j - gap];
}
a[j] = t;
}
}
}
static int parity_is_known_entry(DictEntry * const *known, uint32_t n, const void *p) {
uint32_t lo = 0, hi = n;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
if ((uintptr_t)known[mid] == (uintptr_t)p) return 1;
if ((uintptr_t)known[mid] < (uintptr_t)p) lo = mid + 1; else hi = mid;
}
return 0;
}
#if SK_PARITY_DEBUG
#define SK_PARITY_CANONICAL_MASK 0xffff800000000000ULL
@@ -482,20 +542,40 @@ static uint64_t fnv1a_64_u32(uint32_t val, uint64_t hash) {
* @brief Count the number of entries in the VM dictionary.
*
* Walks the linked list starting at @c vm->latest, following each entry's
* @c link pointer until @c NULL. Stops early and returns @c MAX_DICT_ENTRIES
* if the list exceeds that limit, preventing runaway traversal on a corrupt
* dictionary.
* @c link pointer until @c NULL -- the whole chain, no cap (S-1a). A
* corrupt, cyclic chain is detected (dict_walk_count()), logged, counted,
* and reported as 0 rather than walked forever.
*
* @param vm Active VM whose dictionary will be counted; returns 0 if @c NULL.
* @return Number of dictionary entries found, capped at @c MAX_DICT_ENTRIES.
* @return Number of dictionary entries found (0 on a cyclic chain).
*/
uint32_t sk_dict_word_count(struct VM *vm) {
if (!vm) return 0;
/* S-1a: length of the whole link chain from vm->latest, with Floyd's
* cycle check (a second pointer moving two links per step) standing in for
* the old fixed cap. Returns 0 and sets *out, or -1 on a cyclic chain. */
static int dict_walk_count(struct VM *vm, uint32_t *out) {
DictEntry *slow = vm->latest, *fast = vm->latest;
uint32_t count = 0;
for (DictEntry *e = vm->latest; e != NULL; e = e->link) {
while (slow) {
count++;
if (count >= MAX_DICT_ENTRIES) break;
slow = slow->link;
if (fast && fast->link) {
fast = fast->link->link;
if (fast && fast == slow) return -1;
} else {
fast = (DictEntry *)0;
}
}
*out = count;
return 0;
}
uint32_t sk_dict_word_count(struct VM *vm) {
uint32_t count = 0;
if (!vm) return 0;
if (dict_walk_count(vm, &count) != 0) {
parity_incomplete("dictionary link chain is cyclic (corrupt)");
return 0;
}
return count;
}
@@ -515,7 +595,9 @@ uint32_t sk_dict_word_count(struct VM *vm) {
* - A @c NULL cell → traversal stops (end-of-body sentinel).
*
* Returns @p hash unchanged if @p entry lacks the @c WORD_COMPILED flag or
* has no valid data-field. Capped at 1024 cells per body as a safety limit.
* has no valid data-field. Bounded by vm->here (the end of compiled code),
* not a cell cap (S-1a); a body that reaches HERE without EXIT or a NULL
* cell sets *complete to 0 (logged, counted).
*
* @param vm Active VM instance.
* @param entry Dictionary entry whose body will be hashed; must have
@@ -524,7 +606,8 @@ uint32_t sk_dict_word_count(struct VM *vm) {
* @param hash Running FNV-1a hash state from the caller.
* @return Updated hash state after processing the colon body.
*/
static uint64_t hash_colon_body(struct VM *vm, DictEntry *entry, uint32_t word_index, uint64_t hash) {
static uint64_t hash_colon_body(struct VM *vm, DictEntry *entry, uint32_t word_index, uint64_t hash,
int *complete, DictEntry * const *known, uint32_t nknown) {
if (!vm || !entry || !(entry->flags & WORD_COMPILED)) {
return hash;
}
@@ -573,9 +656,19 @@ static uint64_t hash_colon_body(struct VM *vm, DictEntry *entry, uint32_t word_i
sk_parity_debug_check_ptr(vm, "colon_body_ip", entry, word_index, entry, entry->func, ip);
#endif
/* Safety limit on body length */
size_t max_cells = (VM_MEMORY_SIZE - body_addr) / sizeof(cell_t);
if (max_cells > 1024) max_cells = 1024;
/* FABRIC-3.9.0.md §8.1 (Stage S-1a): no fixed cell cap (was 1024 -- a
* longer body was silently cut off). A body cannot extend past the end
* of compiled code, vm->here, so that is the bound; reaching it without
* EXIT or a NULL cell means the body is not a well-formed colon body,
* and the hash is marked INCOMPLETE rather than quietly truncated. */
size_t max_cells;
int terminated = 0;
if ((size_t)vm->here <= (size_t)body_addr) {
parity_incomplete("colon body starts at or past HERE");
*complete = 0;
return hash;
}
max_cells = ((size_t)vm->here - (size_t)body_addr) / sizeof(cell_t);
for (size_t i = 0; i < max_cells; i++) {
DictEntry *w = (DictEntry *)(uintptr_t)(ip[i]);
@@ -584,12 +677,25 @@ static uint64_t hash_colon_body(struct VM *vm, DictEntry *entry, uint32_t word_i
#endif
if (w == NULL) {
terminated = 1;
break; /* End of body */
}
/* Not a dictionary entry: inline data (string bytes, an offset, an
* operand). Hash the raw cell and never dereference it. */
if (!parity_is_known_entry(known, nknown, w)) {
cell_t raw = ip[i];
for (int b = 0; b < 8; b++) {
hash = fnv1a_64_u8((uint8_t)(raw & 0xFF), hash);
raw >>= 8;
}
continue;
}
if (w == exit_entry) {
/* Hash EXIT's word_id and stop */
hash = fnv1a_64_u32(w->word_id, hash);
terminated = 1;
break;
}
@@ -601,15 +707,31 @@ static uint64_t hash_colon_body(struct VM *vm, DictEntry *entry, uint32_t word_i
i++;
if (i < max_cells) {
cell_t lit_val = ip[i];
/* Hash as 64-bit value */
for (int b = 0; b < 8; b++) {
hash = fnv1a_64_u8((uint8_t)(lit_val & 0xFF), hash);
lit_val >>= 8;
/* S-1a finding (2026-09-24): `['] WORD` compiles LIT <DictEntry*>
* -- a native address that differs per ISA (seen: ACL-BOOT's
* ACL-PIN sequences), which made the hash of a mature
* dictionary differ across ISAs. Rule 2 (word_id, never
* address) applies to a LIT operand that is an entry too:
* hash that entry's word_id, tagged so it can't collide with
* a plain number. Any other literal is hashed as its value. */
if (parity_is_known_entry(known, nknown, (void *)(uintptr_t)lit_val)) {
hash = fnv1a_64_u8(0xE7, hash);
hash = fnv1a_64_u32(((DictEntry *)(uintptr_t)lit_val)->word_id, hash);
} else {
/* Hash as 64-bit value */
for (int b = 0; b < 8; b++) {
hash = fnv1a_64_u8((uint8_t)(lit_val & 0xFF), hash);
lit_val >>= 8;
}
}
}
}
}
if (!terminated) {
parity_incomplete("colon body reaches HERE without EXIT");
*complete = 0;
}
return hash;
}
@@ -633,35 +755,68 @@ static uint64_t hash_colon_body(struct VM *vm, DictEntry *entry, uint32_t word_i
* the hash is stable across different build addresses and address-space
* layouts.
*
* Traversal is capped at @c MAX_DICT_ENTRIES (2048) entries to guard
* against corruption.
* Traversal covers the whole dictionary (S-1a): the entry list is
* allocated to its actual size, and a cyclic chain, an allocation failure
* or an unterminated colon body sets *complete to 0 instead of returning a
* hash over part of it.
*
* @param vm Active VM whose dictionary will be hashed; returns 0 if @c NULL.
* @return 64-bit FNV-1a canonical hash of the dictionary structure.
*/
uint64_t sk_dict_canonical_hash(struct VM *vm) {
uint64_t sk_dict_canonical_hash(struct VM *vm, int *complete) {
*complete = 0;
if (!vm) return 0;
/* FABRIC-3.9.0.md §8.1 (Stage S-1a): the entry list is sized to the
* dictionary actually present -- no fixed 2048-entry array, no silent
* stop. The chain is measured first (cycle-checked), then collected. */
uint32_t total = 0;
if (dict_walk_count(vm, &total) != 0) {
parity_incomplete("dictionary link chain is cyclic (corrupt)");
return 0;
}
DictEntry **entries = (DictEntry **)0;
if (total > 0) {
entries = (DictEntry **)PARITY_ALLOC((size_t)total * sizeof(DictEntry *));
if (!entries) {
parity_incomplete("could not allocate the dictionary entry list");
return 0;
}
}
*complete = 1;
/* Build array of entries in reverse order (latest to oldest) */
DictEntry *entries[MAX_DICT_ENTRIES];
uint32_t count = 0;
#if SK_PARITY_DEBUG
sk_parity_debug_log_msg("sk_dict_canonical_hash:walk_latest");
sk_parity_debug_check_ptr(vm, "latest_head", vm->latest, 0, vm->latest, vm->latest ? vm->latest->func : NULL, vm->latest);
#endif
for (DictEntry *e = vm->latest; e != NULL; e = e->link) {
for (DictEntry *e = vm->latest; e != NULL && count < total; e = e->link) {
#if SK_PARITY_DEBUG
sk_parity_debug_check_ptr(vm, "dict_entry", e, count, e, e->func, e);
#endif
if (count < MAX_DICT_ENTRIES) {
entries[count++] = e;
}
entries[count++] = e;
#if SK_PARITY_DEBUG
sk_parity_debug_check_ptr(vm, "dict_link", e->link, count, e, e->func, e->link);
#endif
}
/* A sorted copy of the entry list: colon-body cells are dereferenced
* only if they are one of these (see parity_sort_entries()). */
DictEntry **known = (DictEntry **)0;
if (count > 0) {
known = (DictEntry **)PARITY_ALLOC((size_t)count * sizeof(DictEntry *));
if (!known) {
PARITY_FREE(entries);
*complete = 0;
parity_incomplete("could not allocate the sorted entry list");
return 0;
}
memcpy(known, entries, (size_t)count * sizeof(DictEntry *));
parity_sort_entries(known, count);
}
/* Hash in creation order (oldest to newest = reverse of array) */
uint64_t hash = FNV1A_64_OFFSET_BASIS;
@@ -710,10 +865,12 @@ uint64_t sk_dict_canonical_hash(struct VM *vm) {
/* 6. For colon definitions: hash body as word_id sequence */
if (e->flags & WORD_COMPILED) {
hash = hash_colon_body(vm, e, ordinal, hash);
hash = hash_colon_body(vm, e, ordinal, hash, complete, known, count);
}
}
if (known) PARITY_FREE(known);
if (entries) PARITY_FREE(entries);
return hash;
}
@@ -748,6 +905,7 @@ void sk_parity_collect(struct VM *vm, ParityPacket *out) {
out->here_offset = 0;
out->latest_word_id = 0;
out->header_hash64 = 0;
out->hash_complete = 0;
out->tests_total = 0;
out->tests_passed = 0;
out->tests_failed = 0;
@@ -765,7 +923,10 @@ void sk_parity_collect(struct VM *vm, ParityPacket *out) {
out->word_count = sk_dict_word_count(vm);
out->here_offset = (uint32_t)vm->here;
out->latest_word_id = vm->latest ? vm->latest->word_id : 0;
out->header_hash64 = sk_dict_canonical_hash(vm);
out->header_hash64 = sk_dict_canonical_hash(vm, &out->hash_complete);
if (!out->hash_complete && out->bootstrap_result == SK_BOOTSTRAP_OK) {
out->bootstrap_result = SK_BOOTSTRAP_PARITY_INCOMPLETE; /* S-1a: never a quiet wrong hash */
}
/* M7.1b fields - from global test stats (if tests were run) */
#ifdef STARFORTH_ENABLE_TESTS
@@ -896,7 +1057,11 @@ void sk_parity_print(const ParityPacket *pkt) {
print_str(" latest_id=");
print_u32(pkt->latest_word_id);
print_str(" hash=");
print_hex64(pkt->header_hash64);
if (pkt->hash_complete) {
print_hex64(pkt->header_hash64);
} else {
print_str("INCOMPLETE"); /* S-1a: never print a hash over part of the dictionary */
}
print_nl();
/* M7.1b line (only if tests were run) */