Stage K-1: real kernel realloc; word-metrics defects closed (FABRIC-3.10.8 §4.1)

- Kernel sf_realloc() is a real realloc (copy min(old,new) via new
  kmalloc_usable_size(), free old, NULL-on-0 frees). It allocated fresh
  and nothing else: every growth leaked, and vm_state_push()'s call-state
  stack lost every saved state past depth 8 (found here).
- Shim realloc() copies min(old,new) (over-read when growing).
- Defect 9: FC bucket growth no longer leaks or patches a copy; no
  sf_xrealloc() (freed a block the index still held on failure).
- Defect 5: heat snapshot heap-sized (was heats[1024], truncating).
- Defect 6: word refused whole when its metrics can't be allocated.
- Defect 7: vm_cleanup() (both) frees metrics' inner arrays.
- Defect 8: most-likely-next kept incrementally; no full id scan.
- Probe (removed): realloc keeps contents, heap returns exactly; call
  stack depth 20 intact; all three ISAs. Churn criterion NOT met: ~7 KB
  retained per churned VM (quota slots/word rows -- K-4 re-measures); a
  post-churn CONSOLE-CMD refusal attributed to the quota leak, K-4 retests.
- Standard sequence on amd64/aarch64/riscv64: hashes, G, ledger unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/dictionary_heat_optimization.c

@@ -59,6 +59,7 @@
 #include "dictionary_heat_optimization.h"
 #include "log.h"
 #include "platform_time.h"
+#include "platform_alloc.h" /* sf_malloc()/sf_free() -- the heat snapshot (K-1) */
 
 /* struct SfFcIndex (vm->fc_index, per VM since MC-0c) is declared in dictionary_heat_optimization.h */
 
@@ -119,13 +120,23 @@ static int compare_heat_ascending(const void *a, const void *b) {
 void dict_update_heat_percentiles(VM *vm) {
     if (!vm) return;
 
-    /* Collect all heats from dictionary */
-    cell_t heats[DICTIONARY_SIZE];
-    int count = 0;
+    /* Collect all heats from dictionary. FABRIC-3.10.8 K-1: sized to the
+     * dictionary, allocated per call -- was heats[DICTIONARY_SIZE] on the
+     * stack, silently dropping every word past the 1024th. */
+    cell_t *heats;
+    int count = 0, n = 0;
 
     /* Protect traversal against concurrent heartbeat decay sweeps */
     sf_mutex_lock(&vm->dict_lock);
-    for (DictEntry *w = vm->latest; w && count < DICTIONARY_SIZE; w = w->link) {
+    for (DictEntry *w = vm->latest; w; w = w->link) n++;
+    heats = n ? (cell_t *)sf_malloc((size_t)n * sizeof(cell_t)) : (cell_t *)0;
+    if (n && !heats) {
+        sf_mutex_unlock(&vm->dict_lock);
+        /* Refused whole: the previous thresholds stand. */
+        log_message(LOG_ERROR, "heat percentiles: can't allocate a snapshot of %d words", n);
+        return;
+    }
+    for (DictEntry *w = vm->latest; w && count < n; w = w->link) {
         heats[count++] = w->execution_heat;
     }
     sf_mutex_unlock(&vm->dict_lock);
@@ -134,6 +145,7 @@ void dict_update_heat_percentiles(VM *vm) {
         vm->heat_threshold_25th = 0;
         vm->heat_threshold_50th = 0;
         vm->heat_threshold_75th = 0;
+        sf_free(heats);
         return;
     }
 
@@ -144,6 +156,7 @@ void dict_update_heat_percentiles(VM *vm) {
     vm->heat_threshold_25th = heats[(count * 25) / 100];
     vm->heat_threshold_50th = heats[(count * 50) / 100];
     vm->heat_threshold_75th = heats[(count * 75) / 100];
+    sf_free(heats);
 
     log_message(LOG_DEBUG,
                 "Heat percentiles: 25th=%ld, 50th=%ld, 75th=%ld (from %d words)",

src/dictionary_management.c

@@ -73,6 +73,8 @@ static uint32_t g_fc_index_alloc_failures; /* atomic: any core can count one */
  * to run silently outside the physics. */
 #define WORD_ID_TABLE_MIN 64u
 
+/* Words vm_create_word() refused whole: no stable id (S-1b), or, since K-1,
+ * no transition metrics. */
 static uint32_t g_word_id_refusals;
 
 uint32_t vm_dictionary_word_id_refusals(void) { return __atomic_load_n(&g_word_id_refusals, __ATOMIC_RELAXED); }
@@ -230,15 +232,6 @@ int vm_dict_entry_ok(VM *vm, DictEntry *candidate) {
 
 /* --- helpers ------------------------------------------------------------- */
 
-static inline void *sf_xrealloc(void *p, size_t nbytes) {
-    void *r = sf_realloc(p, nbytes);
-    if (!r) {
-        sf_free(p);
-        return NULL;
-    }
-    return r;
-}
-
 struct SfFcIndex *vm_dictionary_index(VM *vm) {
     if (!vm) return NULL;
     if (!vm->fc_index) {
@@ -271,19 +264,15 @@ static int sf_fc_reserve(struct SfFcIndex *ix, size_t i, size_t need) {
     if (ix->cap[i] >= need) return 1;
     size_t newcap = ix->cap[i] ? ix->cap[i] : 8;
     while (newcap < need) newcap <<= 1;
-    DictEntry **old = ix->list[i];
-    DictEntry **newv = (DictEntry **) sf_xrealloc(old, newcap * sizeof(*newv));
+    /* sf_realloc(), not sf_xrealloc(): on failure the bucket must stay as it
+     * was (sf_xrealloc() frees the old block, which this index still
+     * points at). */
+    DictEntry **newv = (DictEntry **) sf_realloc(ix->list[i], newcap * sizeof(*newv));
     if (!newv) return 0;
-    /* Kernel sf_realloc is a bump allocator that does not copy; patch that up.
-     * On hosted Linux, realloc already copied before freeing old — do NOT memcpy
-     * from freed memory (UB; glibc overwrites freed block with heap metadata). */
-#ifdef __STARKERNEL__
-    {
-        size_t old_count = ix->count[i];
-        if (newv != old && old && old_count > 0)
-            memcpy(newv, old, old_count * sizeof(*newv));
-    }
-#endif
+    /* FABRIC-3.10.8 K-1: the kernel sf_realloc() is a real realloc now
+     * (copies, frees the old block), so the kernel-only copy that patched
+     * its old allocate-only behaviour is gone -- it would read freed memory.
+     * The old block was never freed before: every bucket growth leaked it. */
     ix->list[i] = newv;
     ix->cap[i] = newcap;
     return 1;
@@ -549,7 +538,16 @@ DictEntry *vm_create_word(VM *vm, const char *name, size_t len, word_func_t func
     if (entry->transition_metrics) {
         transition_metrics_init(entry->transition_metrics);
     } else {
-        log_message(LOG_WARN, "vm_create_word: transition metrics malloc failed for '%.*s'", (int)len, name);
+        /* FABRIC-3.10.8 K-1: refused whole, like a word that can't get an
+         * id (below) -- was logged at WARN and defined anyway, a word
+         * running outside the transition physics. */
+        __atomic_add_fetch(&g_word_id_refusals, 1, __ATOMIC_RELAXED);
+        log_message(LOG_ERROR, "vm_create_word: transition metrics can't be allocated for '%.*s': refused",
+                    (int)len, name);
+        vm->error = 1;
+        sf_free(entry);
+        entry = NULL;
+        goto create_cleanup;
     }
 
     memcpy(entry->name, name, len);

src/physics_pipelining_metrics.c

@@ -251,6 +251,20 @@ int transition_metrics_record(WordTransitionMetrics *metrics, uint32_t next_word
     metrics->transition_heat[next_word_id]++;
     metrics->total_transitions++;
 
+    /* FABRIC-3.10.8 K-1: keep the most-likely-next word current here, so
+     * transition_metrics_update_cache() needn't scan the whole id space on
+     * every transition. Counts only ever rise by one, so the leader changes
+     * only when this word overtakes it (ties: the lower id, as the scan
+     * picked). */
+    {
+        uint32_t best = metrics->most_likely_next_word_id;
+        uint64_t bc   = (best < metrics->transition_heat_cap) ? metrics->transition_heat[best] : 0;
+        uint64_t nc   = metrics->transition_heat[next_word_id];
+        if (nc > bc || (nc == bc && next_word_id < best)) {
+            metrics->most_likely_next_word_id = next_word_id;
+        }
+    }
+
     return 0;
 }
 
@@ -328,30 +342,30 @@ int64_t transition_metrics_get_probability_q48(const WordTransitionMetrics *metr
  *                   must match the size used at allocation time.
  */
 void transition_metrics_update_cache(WordTransitionMetrics *metrics, uint32_t dict_size) {
-    if (!metrics || !metrics->transition_heat || metrics->total_transitions == 0) {
+    if (!metrics) return; /* K-1: was dereferenced in the branch below when NULL */
+    if (!metrics->transition_heat || metrics->total_transitions == 0) {
         metrics->max_transition_probability_q48 = 0;
         metrics->most_likely_next_word_id = 0;
         return;
     }
 
-    int64_t max_prob_q48 = 0;
-    uint32_t max_idx = 0;
-
-    /* Find word with highest transition count. Bounded by the array's own
-     * length (S-1b): it may lag dict_size until the next grow. */
+    /* FABRIC-3.10.8 K-1: the leader (highest transition count, lowest id on
+     * a tie) is kept current by transition_metrics_record(); this only
+     * refreshes its probability. It used to scan every id up to dict_size
+     * on every transition. One difference, by construction: the scan
+     * compared probabilities, which are floored Q48.16 quotients, so once
+     * total_transitions passes 65536 two different counts could floor
+     * equal and the scan kept the lower id; the leader is by count.
+     * Bounded by the array's own length (S-1b): it may lag dict_size. */
     uint32_t limit = (dict_size < metrics->transition_heat_cap) ? dict_size : metrics->transition_heat_cap;
-    for (uint32_t i = 0; i < limit; i++) {
-        if (metrics->transition_heat[i] > 0) {
-            int64_t prob_q48 = transition_metrics_get_probability_q48(metrics, i);
-            if (prob_q48 > max_prob_q48) {
-                max_prob_q48 = prob_q48;
-                max_idx = i;
-            }
-        }
-    }
+    uint32_t best  = metrics->most_likely_next_word_id;
 
-    metrics->max_transition_probability_q48 = max_prob_q48;
-    metrics->most_likely_next_word_id = max_idx;
+    if (best >= limit || metrics->transition_heat[best] == 0) {
+        metrics->max_transition_probability_q48 = 0;
+        metrics->most_likely_next_word_id = 0;
+        return;
+    }
+    metrics->max_transition_probability_q48 = transition_metrics_get_probability_q48(metrics, best);
 }
 
 /**

src/starkernel/memory/kmalloc.c

@@ -332,6 +332,25 @@ void kfree(void *ptr)
     sk_spin_unlock(&g_lock_heap);
 }
 
+/* FABRIC-3.10.8 K-1: the size `ptr` was allocated with (what kmalloc() or
+ * kmalloc_aligned() was asked for), or 0 if `ptr` isn't a live allocation
+ * from this heap. What a real realloc() needs to copy min(old, new). */
+size_t kmalloc_usable_size(const void *ptr)
+{
+    size_t n = 0;
+    if (!heap_initialized || !ptr || !pointer_within_heap(ptr)) {
+        return 0;
+    }
+    const kmalloc_prefix_t *prefix =
+        (const kmalloc_prefix_t *)((const uint8_t *)ptr - sizeof(kmalloc_prefix_t));
+    sk_spin_lock(&g_lock_heap);
+    if (prefix->magic == KMALLOC_MAGIC && prefix->block && !prefix->block->free) {
+        n = prefix->block->requested;
+    }
+    sk_spin_unlock(&g_lock_heap);
+    return n;
+}
+
 kmalloc_stats_t kmalloc_get_stats(void)
 {
     kmalloc_stats_t s;

src/starkernel/vm/alloc_kernel.c

@@ -157,22 +157,20 @@ void* sf_calloc(size_t count, size_t size)
 }
 
 /*
- * @brief Resize an allocation in the kernel heap.
+ * @brief Resize an allocation in the kernel heap (a real realloc).
  *
- * Unchanged from this file's previous implementation: `kmalloc.c` has no
- * realloc-equivalent, so a fresh block of @p new_size bytes is allocated
- * and returned; the original block at @p ptr is left for the caller to
- * free explicitly (this function does not free it, matching this file's
- * prior documented behavior exactly -- not a regression introduced by
- * the kmalloc.c switch).
+ * FABRIC-3.10.8 K-1: allocates @p new_size bytes, copies min(old, new) bytes
+ * (the old size from kmalloc_usable_size()), frees the old block, and
+ * returns the new one. On failure the old block is left untouched and NULL
+ * is returned, as in C. This used to allocate a fresh block and do nothing
+ * else -- no copy, no free -- while its callers (sf_realloc()'s users in the
+ * vendored VM: the dictionary index buckets, the call-state stack) assumed
+ * standard realloc(): every growth leaked the old block, and the call-state
+ * stack lost every saved state past its first capacity.
  *
  * Special cases match the C standard:
  * - @p ptr == @c NULL → equivalent to @c sf_malloc(@p new_size).
- * - @p new_size == 0 → returns @c NULL (caller treats old block as freed).
- *
- * The caller is responsible for copying content from the old block
- * before discarding the old pointer; this function does not perform the
- * copy.
+ * - @p new_size == 0 → frees @p ptr and returns @c NULL.
  *
  * @param ptr      Pointer to the existing allocation (may be @c NULL).
  * @param new_size Desired size of the new block in bytes.
@@ -181,10 +179,28 @@ void* sf_calloc(size_t count, size_t size)
  */
 void* sf_realloc(void* ptr, size_t new_size)
 {
-    if (!ptr) return sf_malloc(new_size);
-    if (new_size == 0) return (void*)0;
+    size_t old_size;
+    void  *np;
 
-    return sf_malloc(new_size);
+    if (!ptr) return sf_malloc(new_size);
+    if (new_size == 0) {
+        sf_free(ptr);
+        return (void*)0;
+    }
+
+    np = sf_malloc(new_size);
+    if (!np) return (void*)0; /* old block untouched, as in C */
+
+    old_size = kmalloc_usable_size(ptr);
+    {
+        const uint8_t *src = (const uint8_t *)ptr;
+        uint8_t       *dst = (uint8_t *)np;
+        size_t n = (old_size < new_size) ? old_size : new_size;
+        size_t i;
+        for (i = 0; i < n; i++) dst[i] = src[i];
+    }
+    sf_free(ptr);
+    return np;
 }
 
 /*

src/starkernel/vm/host/shim.c

@@ -147,10 +147,8 @@ void *calloc(size_t n, size_t size) {
  * @brief Kernel @c realloc() stub: resize a heap allocation.
  *
  * Implements standard @c realloc() semantics using @c malloc() + @c memcpy() +
- * @c free(). Because the old allocation size is not tracked by @c kmalloc(),
- * the copy uses @p size as the byte count — meaning this truncates the content
- * if @p size < old size, and does not guarantee the tail bytes are zero if
- * @p size > old size. Suitable for rare realloc calls in shared VM code.
+ * @c free(), copying min(old, new) bytes (the old size from
+ * @c kmalloc_usable_size()). Bytes past the old size are not zeroed, as in C.
  *
  * Special cases:
  * - @p ptr == @c NULL → equivalent to @c malloc(@p size).
@@ -166,11 +164,14 @@ void *realloc(void *ptr, size_t size) {
         free(ptr);
         return NULL;
     }
-    /* Simple realloc: allocate new, copy min(old,new), free old */
+    /* Allocate new, copy min(old, new), free old. FABRIC-3.10.8 K-1: the old
+     * size is now known (kmalloc_usable_size()); this used to copy `size`
+     * bytes regardless -- reading past the end of the old block whenever it
+     * grew. On failure the old block is left untouched, as in C. */
     void *newp = malloc(size);
     if (newp) {
-        /* Unknown old size; copy conservatively */
-        memcpy(newp, ptr, size);
+        size_t old = kmalloc_usable_size(ptr);
+        memcpy(newp, ptr, old < size ? old : size);
         free(ptr);
     }
     return newp;

src/starkernel/vm/vm_core.c

@@ -352,6 +352,9 @@ void vm_cleanup(VM* vm)
         while (dict_entry) {
             DictEntry *next_entry = dict_entry->link;
             if (dict_entry->transition_metrics) {
+                /* FABRIC-3.10.8 K-1: its own arrays first (transition_heat,
+                 * context_window, context_transitions) -- they leaked. */
+                transition_metrics_cleanup(dict_entry->transition_metrics);
                 sf_free(dict_entry->transition_metrics);
             }
             sf_free(dict_entry);

src/vm_bootstrap.c

@@ -62,6 +62,7 @@
 #include "../include/dictionary_heat_optimization.h"
 #include "../include/ssm_jacquard.h"
 #include "../include/platform_alloc.h"
+#include "../include/physics_pipelining_metrics.h" /* transition_metrics_cleanup() -- K-1 */
 #include "../include/vm_api.h" /* vm_input_ensure() -- S-1c */
 #include "word_source/include/vocabulary_words.h"
 
@@ -425,6 +426,8 @@ void vm_cleanup(VM* vm)
         while (dict_entry) {
             DictEntry *next_entry = dict_entry->link;
             if (dict_entry->transition_metrics) {
+                /* FABRIC-3.10.8 K-1: its own arrays first -- they leaked. */
+                transition_metrics_cleanup(dict_entry->transition_metrics);
                 sf_free(dict_entry->transition_metrics);
             }
             sf_free(dict_entry);