FABRIC-3.10.8 K-11: check the lock order at run time

Every lock carries a rank (hermes < switch < registry < VM dict_lock <
VM tuning_lock < stadium < slot < doe < log < block < console < heap <
vmm < pmm); each core counts held locks per rank and an acquisition not
above the highest held rank counts an order violation, reported in full
at the idle point. Probe inversion reported exactly once; O-A, a 28-VM
churn and the standard sequence show 0 violations on all three ISAs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

FABRIC-3.10.8.md

@@ -774,3 +774,33 @@ Only these two FABRIC docs record the old hashes; nothing in code or scripts che
 | riscv64 | `logs/20260926-102708` | same as amd64 |
 
 **K-10 closes the three stale Hermes-VM references found at K-3.** `ARTEMIS-READY-ACK` (no caller) and the `switch_hermes_readiness` CSV column name (always 0, kept with its two siblings pending FABRIC-3.8.md Stage I) remain, as noted in §4.9.
+
+### 4.12 K-11 — the lock order checked at run time (2026-09-26)
+
+**Change.**
+- **Ranks** (`spinlock.h`, `sk_lock_rank_t`): hermes < switch < registry < VM `dict_lock` < VM `tuning_lock` < stadium < Stadium slot < doe < log < block < console < heap < vmm < pmm.
+  - `sk_spinlock_t` carries its rank (`SK_SPINLOCK_INIT(rank)`).
+  - The raw-lockword calls take one: `sk_lockword_acquire/release(word, rank)` for Stadium slots and `sf_mutex`.
+  - The kernel `sf_mutex_t` gains a `rank`. `sf_mutex_init()` sets the dict rank, and the kernel `vm_init()` re-ranks `tuning_lock`.
+- **The check** (`spinlock.c`).
+  - Each core counts the locks it holds per rank (`sk_percpu_t.held_rank[]`).
+  - Taking a lock whose rank is not above the highest held rank counts an order violation, recording both ranks and the call site. A same-rank pair counts too.
+  - A recursion is still counted as a recursion only.
+  - New `sk_lock_order_violations()`.
+- **The report** (`sk_lock_report()`, at the idle point) prints the full line to the console. The log line keeps only the count, because `log_message()` cuts at `LOG_LINE_MAX` (64 bytes): the first probe run showed `...while holding he` (`logs/20260926-103947`). This follows parity.c's INCOMPLETE report.
+
+**Evidence.**
+- **Probe** (temporary `K11-PROBE` word: heap, then registry; amd64, `logs/20260926-104239`). Nothing is reported through boot. After the probe, exactly one line: `LOCK: order violation x1 (first: took registry while holding heap, from 0x000000003d71a62c)`.
+- **Clean** (probe reverted; the tree is byte-identical to the saved stage diff). O-A, then a 28-VM churn (32 births, 28 kills), then the standard sequence:
+
+| ISA | log | order violations / any `LOCK:` line | result |
+|---|---|---|---|
+| amd64 | `logs/20260926-104446` | **0 / none** | K-10 hashes, G A=40 B=40, HHH, all ok `3`, 0 B idle, 0 / 0 |
+| aarch64 | `logs/20260926-110203` | **0 / none** | same |
+| riscv64 | `logs/20260926-112011` | **0 / none** | same; 0 `timed out` |
+
+The ledger reads pulled 319488 / returned 316703 on all three. That is the K-3 baseline plus 28 × 2048: the churn's 28 SENDs, each a message cell pulled and returned. Held and consumed are unchanged, and conservation holds.
+
+**Found, not fixed:** MC-0b's three older lock reports (recursion, bad release, held at switch) go through `log_message()` only, so they are cut at 64 bytes and lose the lock name and site. None fired in any run here.
+
+**K-11 closes the "lock order checked by inspection only" finding.** With K-9..K-11, every ruling from K-8's report is carried out.

include/platform_lock.h

@@ -54,6 +54,7 @@
 typedef struct sf_mutex
 {
     volatile unsigned int state;
+    unsigned int          rank;  /* kernel lock-order rank (spinlock.h, FABRIC-3.10.8 K-11) */
 } sf_mutex_t;
 #else
 #include <pthread.h>

include/starkernel/percpu.h

@@ -42,6 +42,7 @@
 #define STARKERNEL_PERCPU_H
 
 #include <stdint.h>
+#include "starkernel/spinlock.h" /* SK_LOCK_RANK_COUNT */
 
 struct VM;
 
@@ -57,6 +58,9 @@ typedef struct sk_percpu {
     uint64_t          saved_irq;
     uint64_t          lock_acquired; /* every acquire on this core, ever */
     uint64_t          lock_released; /* every release on this core, ever */
+    /* K-11 (spinlock.c): locks this core holds right now, per rank -- the
+     * lock-order check compares a new lock's rank with the highest held. */
+    uint32_t          held_rank[SK_LOCK_RANK_COUNT];
 } sk_percpu_t;
 
 /* Installs the boot core's block in this core's per-CPU register. Call

include/starkernel/spinlock.h

@@ -38,7 +38,8 @@
  *     on the same core, so it is never treated as legitimate. The call is
  *     let through (spinning would hang the core) and counted;
  *   - bad release: releasing a lock this core doesn't hold;
- *   - held at switch: a switch or idle point reached with a lock held.
+ *   - held at switch: a switch or idle point reached with a lock held;
+ *   - order violation: see the rank check below.
  *
  * sf_mutex_t (the vendored VM's dict_lock/tuning_lock) uses the same lock
  * word through sk_lockword_acquire()/sk_lockword_release() (shim.c).
@@ -72,22 +73,50 @@
  * decays cells) and below a VM's dict_lock (vm_core.c's heat bump dispatches
  * the word under it), and above block (MIGRATE calls blk_flush()); nothing
  * in the block path calls into Stadium.
+ *
+ * The order is checked at run time (FABRIC-3.10.8 K-11): every lock has a
+ * rank (sk_lock_rank_t, same order as above, with a VM's dict_lock and
+ * tuning_lock between the registry and Stadium), each core counts the
+ * locks it holds per rank, and taking a lock whose rank is not above every
+ * rank this core already holds counts an order violation. Two locks of one
+ * rank (two Stadium slots, two VMs' dict_locks) count as one too. Counted
+ * and logged like the other failures; never refused.
  */
 #ifndef STARKERNEL_SPINLOCK_H
 #define STARKERNEL_SPINLOCK_H
 
 #include <stdint.h>
 
+typedef enum sk_lock_rank {
+    SK_LOCK_RANK_NONE = 0,     /* unranked: neither checked nor counted */
+    SK_LOCK_RANK_HERMES,
+    SK_LOCK_RANK_SWITCH,
+    SK_LOCK_RANK_REGISTRY,
+    SK_LOCK_RANK_VM_DICT,      /* a VM's dict_lock (sf_mutex_t) */
+    SK_LOCK_RANK_VM_TUNING,    /* a VM's tuning_lock (sf_mutex_t) */
+    SK_LOCK_RANK_STADIUM,
+    SK_LOCK_RANK_STADIUM_SLOT,
+    SK_LOCK_RANK_DOE,
+    SK_LOCK_RANK_LOG,
+    SK_LOCK_RANK_BLOCK,
+    SK_LOCK_RANK_CONSOLE,
+    SK_LOCK_RANK_HEAP,
+    SK_LOCK_RANK_VMM,
+    SK_LOCK_RANK_PMM,
+    SK_LOCK_RANK_COUNT
+} sk_lock_rank_t;
+
 typedef struct sk_spinlock {
     volatile uint32_t owner;    /* 0 = free, else core_id + 1 */
     uint64_t          acquired; /* updated while held, so exact */
     uint64_t          released;
+    uint32_t          rank;     /* sk_lock_rank_t */
 } sk_spinlock_t;
 
-#define SK_SPINLOCK_INIT { 0, 0, 0 }
+#define SK_SPINLOCK_INIT(rank) { 0, 0, 0, (rank) }
 
-void sk_lockword_acquire(volatile uint32_t *word);
-void sk_lockword_release(volatile uint32_t *word);
+void sk_lockword_acquire(volatile uint32_t *word, uint32_t rank);
+void sk_lockword_release(volatile uint32_t *word, uint32_t rank);
 
 void sk_spin_lock(sk_spinlock_t *l);
 void sk_spin_unlock(sk_spinlock_t *l);
@@ -107,6 +136,7 @@ void sk_lock_report(void);
 uint64_t sk_lock_recursions(void);
 uint64_t sk_lock_bad_releases(void);
 uint64_t sk_lock_held_at_switch(void);
+uint64_t sk_lock_order_violations(void);
 
 extern sk_spinlock_t g_lock_registry;
 extern sk_spinlock_t g_lock_stadium;

src/starkernel/hal/spinlock.c

@@ -28,25 +28,31 @@
 #include "starkernel/spinlock.h"
 #include "starkernel/percpu.h"
 #include "starkernel/arch.h"
+#include "starkernel/console.h"  /* K-11: the order report's full line */
+#include "freestanding/stdio.h"  /* snprintf() */
 #include "log.h"
 
-sk_spinlock_t g_lock_registry = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_stadium  = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_switch   = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_hermes   = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_doe      = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_log      = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_block    = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_console  = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_heap     = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_pmm      = SK_SPINLOCK_INIT;
-sk_spinlock_t g_lock_vmm      = SK_SPINLOCK_INIT;
+sk_spinlock_t g_lock_registry = SK_SPINLOCK_INIT(SK_LOCK_RANK_REGISTRY);
+sk_spinlock_t g_lock_stadium  = SK_SPINLOCK_INIT(SK_LOCK_RANK_STADIUM);
+sk_spinlock_t g_lock_switch   = SK_SPINLOCK_INIT(SK_LOCK_RANK_SWITCH);
+sk_spinlock_t g_lock_hermes   = SK_SPINLOCK_INIT(SK_LOCK_RANK_HERMES);
+sk_spinlock_t g_lock_doe      = SK_SPINLOCK_INIT(SK_LOCK_RANK_DOE);
+sk_spinlock_t g_lock_log      = SK_SPINLOCK_INIT(SK_LOCK_RANK_LOG);
+sk_spinlock_t g_lock_block    = SK_SPINLOCK_INIT(SK_LOCK_RANK_BLOCK);
+sk_spinlock_t g_lock_console  = SK_SPINLOCK_INIT(SK_LOCK_RANK_CONSOLE);
+sk_spinlock_t g_lock_heap     = SK_SPINLOCK_INIT(SK_LOCK_RANK_HEAP);
+sk_spinlock_t g_lock_pmm      = SK_SPINLOCK_INIT(SK_LOCK_RANK_PMM);
+sk_spinlock_t g_lock_vmm      = SK_SPINLOCK_INIT(SK_LOCK_RANK_VMM);
 
 /* Failure counters (spinlock.h). Written with atomics: a failure can be
  * counted on any core. The first site of each kind is kept for the log. */
 static uint64_t g_recursions;
 static uint64_t g_bad_releases;
 static uint64_t g_held_at_switch;
+static uint64_t g_order_violations;
+static uint32_t g_order_new_rank, g_order_held_rank;
+static void    *g_order_site;
+static uint64_t g_reported_order;
 static const volatile uint32_t *g_recursion_word;
 static void       *g_recursion_site;
 static const volatile uint32_t *g_bad_release_word;
@@ -82,9 +88,21 @@ static inline void irq_restore(uint64_t f)
 #endif
 }
 
+/* ---- the rank check (K-11) -------------------------------------------- */
+
+/* Highest rank this core holds, or SK_LOCK_RANK_NONE. */
+static uint32_t highest_held_rank(const sk_percpu_t *pc)
+{
+    uint32_t r;
+    for (r = SK_LOCK_RANK_COUNT - 1u; r > SK_LOCK_RANK_NONE; r--) {
+        if (pc->held_rank[r]) return r;
+    }
+    return SK_LOCK_RANK_NONE;
+}
+
 /* ---- the lock word ---------------------------------------------------- */
 
-static void lockword_acquire(volatile uint32_t *w, void *site)
+static void lockword_acquire(volatile uint32_t *w, uint32_t rank, void *site)
 {
     sk_percpu_t *pc;
     uint32_t     me;
@@ -97,6 +115,21 @@ static void lockword_acquire(volatile uint32_t *w, void *site)
 
     if (pc->lock_depth++ == 0) pc->saved_irq = f;
     pc->lock_acquired++;
+    if (rank >= SK_LOCK_RANK_COUNT) rank = SK_LOCK_RANK_NONE;
+
+    /* Counted per rank even on recursion, so its release stays symmetric. */
+    if (rank != SK_LOCK_RANK_NONE) {
+        if (*w != me) {
+            uint32_t held = highest_held_rank(pc);
+            if (held != SK_LOCK_RANK_NONE && rank <= held &&
+                __atomic_fetch_add(&g_order_violations, 1, __ATOMIC_RELAXED) == 0) {
+                g_order_new_rank  = rank;
+                g_order_held_rank = held;
+                g_order_site      = site;
+            }
+        }
+        pc->held_rank[rank]++;
+    }
 
     if (*w == me) {
         /* Already ours: spinning would hang this core for good. Let it
@@ -117,7 +150,7 @@ static void lockword_acquire(volatile uint32_t *w, void *site)
     }
 }
 
-static void lockword_release(volatile uint32_t *w, void *site)
+static void lockword_release(volatile uint32_t *w, uint32_t rank, void *site)
 {
     sk_percpu_t *pc;
     uint32_t     me;
@@ -126,6 +159,10 @@ static void lockword_release(volatile uint32_t *w, void *site)
     pc = sk_percpu();
     me = pc->core_id + 1u;
 
+    if (rank < SK_LOCK_RANK_COUNT && rank != SK_LOCK_RANK_NONE && pc->held_rank[rank]) {
+        pc->held_rank[rank]--;
+    }
+
     if (*w == me) {
         __atomic_store_n(w, 0u, __ATOMIC_RELEASE);
     } else if (__atomic_fetch_add(&g_bad_releases, 1, __ATOMIC_RELAXED) == 0) {
@@ -137,26 +174,26 @@ static void lockword_release(volatile uint32_t *w, void *site)
     if (--pc->lock_depth == 0) irq_restore(pc->saved_irq);
 }
 
-void sk_lockword_acquire(volatile uint32_t *word)
+void sk_lockword_acquire(volatile uint32_t *word, uint32_t rank)
 {
-    lockword_acquire(word, __builtin_return_address(0));
+    lockword_acquire(word, rank, __builtin_return_address(0));
 }
 
-void sk_lockword_release(volatile uint32_t *word)
+void sk_lockword_release(volatile uint32_t *word, uint32_t rank)
 {
-    lockword_release(word, __builtin_return_address(0));
+    lockword_release(word, rank, __builtin_return_address(0));
 }
 
 void sk_spin_lock(sk_spinlock_t *l)
 {
-    lockword_acquire(&l->owner, __builtin_return_address(0));
+    lockword_acquire(&l->owner, l->rank, __builtin_return_address(0));
     l->acquired++;
 }
 
 void sk_spin_unlock(sk_spinlock_t *l)
 {
     l->released++;
-    lockword_release(&l->owner, __builtin_return_address(0));
+    lockword_release(&l->owner, l->rank, __builtin_return_address(0));
 }
 
 int sk_spin_held_here(const sk_spinlock_t *l)
@@ -175,6 +212,7 @@ void sk_lock_check_none_held(const char *where)
 uint64_t sk_lock_recursions(void)     { return g_recursions; }
 uint64_t sk_lock_bad_releases(void)   { return g_bad_releases; }
 uint64_t sk_lock_held_at_switch(void) { return g_held_at_switch; }
+uint64_t sk_lock_order_violations(void) { return g_order_violations; }
 
 /* ---- reporting -------------------------------------------------------- */
 
@@ -194,6 +232,16 @@ static const char *lock_name(const volatile uint32_t *w)
     return "vm sf_mutex or Stadium slot";
 }
 
+static const char *rank_name(uint32_t rank)
+{
+    static const char *const names[SK_LOCK_RANK_COUNT] = {
+        "unranked", "hermes", "switch", "registry", "vm dict_lock",
+        "vm tuning_lock", "stadium", "Stadium slot", "doe", "log", "block",
+        "console", "heap", "vmm", "pmm"
+    };
+    return rank < SK_LOCK_RANK_COUNT ? names[rank] : "?";
+}
+
 /* Freestanding kvsnprintf has no %llx: hex by hand. */
 static void hex64(char out[19], uint64_t v)
 {
@@ -228,4 +276,18 @@ void sk_lock_report(void)
                     (unsigned long long)g_held_at_switch,
                     g_held_at_switch_where ? g_held_at_switch_where : "?");
     }
+    if (g_order_violations != g_reported_order) {
+        char line[160];
+        g_reported_order = g_order_violations;
+        hex64(a, (uint64_t)(uintptr_t)g_order_site);
+        /* The full line to the console: log_message() cuts at LOG_LINE_MAX
+         * (64 bytes), which loses the lock names. The log line stays for
+         * the persisted record (same as parity.c's INCOMPLETE report). */
+        snprintf(line, sizeof line, "LOCK: order violation x%llu (first: took %s while holding %s, from %s)",
+                 (unsigned long long)g_order_violations,
+                 rank_name(g_order_new_rank), rank_name(g_order_held_rank), a);
+        console_println(line);
+        log_message(LOG_ERROR, "LOCK: order violation x%llu",
+                    (unsigned long long)g_order_violations);
+    }
 }

src/starkernel/vm/host/shim.c

@@ -402,6 +402,9 @@ void log_set_vm(struct VM *vm) { (void)vm; }
 int sf_mutex_init(sf_mutex_t *mutex) {
     if (!mutex) return -1;
     mutex->state = 0;
+    /* FABRIC-3.10.8 K-11: a VM's dict_lock rank; vm_init() re-ranks its
+     * tuning_lock (the only two kernel sf_mutexes). */
+    mutex->rank  = SK_LOCK_RANK_VM_DICT;
     return 0;
 }
 
@@ -434,7 +437,7 @@ uint64_t sf_mutex_destroyed_held_count(void) { return g_mutex_destroyed_held; }
  */
 void sf_mutex_lock(sf_mutex_t *mutex) {
     if (!mutex) return;
-    sk_lockword_acquire(&mutex->state);
+    sk_lockword_acquire(&mutex->state, mutex->rank);
 }
 
 /**
@@ -444,7 +447,7 @@ void sf_mutex_lock(sf_mutex_t *mutex) {
  */
 void sf_mutex_unlock(sf_mutex_t *mutex) {
     if (!mutex) return;
-    sk_lockword_release(&mutex->state);
+    sk_lockword_release(&mutex->state, mutex->rank);
 }
 
 /* -----------------------------------------------------------------------------

src/starkernel/vm/stadium.c

@@ -152,8 +152,8 @@ typedef struct {
 static StadiumVMQuota *stadium_quotas = (StadiumVMQuota *)0;
 
 /* MC-0d: one quota slot's lock (see StadiumVMQuota.lock). */
-static void slot_lock(int slot)   { sk_lockword_acquire(&stadium_quotas[slot].lock); }
-static void slot_unlock(int slot) { sk_lockword_release(&stadium_quotas[slot].lock); }
+static void slot_lock(int slot)   { sk_lockword_acquire(&stadium_quotas[slot].lock, SK_LOCK_RANK_STADIUM_SLOT); }
+static void slot_unlock(int slot) { sk_lockword_release(&stadium_quotas[slot].lock, SK_LOCK_RANK_STADIUM_SLOT); }
 
 /* Doubly-linked resident-list threading, one entry per cell, parallel to
  * stadium_owner[] -- kmalloc'd at stadium_boot_init() to stadium_ncells

src/starkernel/vm/vm_bootstrap.c

@@ -58,6 +58,7 @@
 #ifdef __STARKERNEL__
 #include "starkernel/vm/arena.h"
 #include "starkernel/console.h"
+#include "starkernel/spinlock.h" /* FABRIC-3.10.8 K-11 -- SK_LOCK_RANK_VM_TUNING */
 #endif
 
 #include <string.h>
@@ -184,6 +185,7 @@ void vm_init_with_host(VM* vm, const VMHostServices *host)
         vm->error = 1;
         return;
     }
+    vm->tuning_lock.rank = SK_LOCK_RANK_VM_TUNING; /* FABRIC-3.10.8 K-11 (spinlock.h) */
 
     vm->memory = (uint8_t*)vm_host_alloc(vm, VM_MEMORY_SIZE, sizeof(void*));
     if (!vm->memory)