Robert Allan JamesandClaude Opus 5.5
71e0be96b0
Each worker core sets up its own IPI receiver before going online (amd64 LAPIC + vector 0x22 with its own stub, aarch64 banked GIC SGI 0, riscv64 sie.SSIE); core 0 sends by ICR / GICD_SGIR / SBI IPI. Park lets exactly one interrupt in without a lost-wake window. Boot self-test: IPI round trips 3/3 and 'woke x3, IPIs x3' on all three ISAs; worker vCPU threads 0.0% on the host. 3-ISA acceptance clean. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
590 lines
25 KiB
C
590 lines
25 KiB
C
/*
|
||
StarForth — Steady-State Virtual Machine Runtime
|
||
Copyright (c) 2023–2025 Robert A. James. All rights reserved.
|
||
Licensed under the StarForth License, Version 1.0.
|
||
*/
|
||
|
||
/**
|
||
* apic.c (aarch64) - Minimal GICv2 driver + ARM Generic Timer (items 0.6/0.7/
|
||
* 4.3.5d).
|
||
*
|
||
* "Minimal" is deliberate and stays deliberate: item 0.6 enabled exactly one
|
||
* PPI (the timer); item 4.3.5d adds the ability to enable exactly one SPI
|
||
* (for virtio-keyboard-pci, item 4.3.5e), still by explicit per-INTID call,
|
||
* not a general GIC driver -- no SGI support, no GICv3/ITS, no SPI beyond
|
||
* the one 4.3.5e will actually enable.
|
||
*
|
||
* GICD/GICC base addresses -- DTB-discovered when a DTB is present, QEMU
|
||
* virt-machine constants otherwise (FABRIC-3.md §IV.3 item 7, 2026-09-04
|
||
* fix; originally recorded as a deliberate hardcode, FABRIC-0.md item 0.6
|
||
* GAP-B1 follow-up, because `fdt_valid(boot_info->dtb)` fails on this
|
||
* system's QEMU aarch64 firmware -- qemu-efi-aarch64 2025.11-3ubuntu7 does
|
||
* not forward a devicetree to the guest, confirmed live rather than
|
||
* assumed). That reasoning covered the QEMU/UEFI path correctly but never
|
||
* covered the native Pi 5 boot path (`rpi5_native_boot.c`), which receives
|
||
* a real DTB directly and calls this same, unmodified `apic_init()` --
|
||
* audited 2026-09-04, found still hardcoding the QEMU addresses
|
||
* unconditionally, which blocks reaching `ok>` on real BCM2712 hardware
|
||
* (its real GIC-400 sits at `0x10_7fff9000`, nowhere near QEMU's
|
||
* `0x08000000`). `gic_bases_from_dtb()` below closes that gap: tries
|
||
* DTB discovery first, falls back to the QEMU constants only when no DTB
|
||
* is present or the GIC node isn't found there -- so the QEMU/UEFI path's
|
||
* existing, working behaviour is unchanged.
|
||
*
|
||
* QEMU's own values were not recalled from memory either -- they were read
|
||
* out of QEMU 10.2.1's own internal devicetree
|
||
* (`qemu-system-aarch64 -machine virt,dumpdtb=...`, decoded with this
|
||
* tree's own fdt.c reader) and are therefore correct for this exact QEMU
|
||
* version, though not guaranteed stable across others. Register *offsets*
|
||
* within each block (GICD_CTLR, GICC_IAR, etc.) are fixed by the GICv2
|
||
* architecture, not the board, and were cross-checked against Linux's own
|
||
* driver header (linux/irqchip/arm-gic.h, from the linux-headers package
|
||
* installed on the build host) rather than recalled either.
|
||
*/
|
||
|
||
#include "apic.h"
|
||
#include "uefi.h"
|
||
#include "console.h"
|
||
#include "timer.h"
|
||
#include "starkernel/fdt.h"
|
||
#include <stdint.h>
|
||
|
||
/* Defined in arch.c (item 0.4). Same extern-in-place convention as
|
||
* interrupts.c: "exception level" has no cross-ISA meaning. */
|
||
extern int aarch64_current_el(void);
|
||
|
||
/* Defined in arch.c (FABRIC-3.md §IV.3 item 7 fix, 2026-09-04) --
|
||
* discovers the real PSCI conduit for arch_cold_reset() from the DTB.
|
||
* Collocated with its consumer there, not defined here, even though this
|
||
* is the one point in boot with boot_info->dtb already in hand for GIC
|
||
* discovery -- see arch.c's own doc comment on this function for why. */
|
||
extern void aarch64_psci_conduit_init(const void *dtb);
|
||
|
||
/* ---- QEMU virt machine constants (fallback when no DTB GIC node is
|
||
* found -- see file header) ----------------------------------------------- */
|
||
#define GICD_BASE_PA_QEMU_DEFAULT 0x08000000UL
|
||
#define GICC_BASE_PA_QEMU_DEFAULT 0x08010000UL
|
||
|
||
/* Runtime GIC base addresses -- set once by apic_init(), defaulting to the
|
||
* QEMU constants above until/unless gic_bases_from_dtb() overrides them.
|
||
* uintptr_t, not a #define, precisely because these are no longer
|
||
* compile-time constants on this path. */
|
||
static uintptr_t s_gicd_base = GICD_BASE_PA_QEMU_DEFAULT;
|
||
static uintptr_t s_gicc_base = GICC_BASE_PA_QEMU_DEFAULT;
|
||
|
||
/* Non-secure EL1 physical timer PPI = 30 (INTID 16+14); EL2 hypervisor
|
||
* timer PPI = 26 (INTID 16+10). Both read directly out of QEMU's own DT
|
||
* dump for the `timer` node's four-entry `interrupts` property (secure,
|
||
* non-secure, virtual, hypervisor, in that architected order) -- not the
|
||
* architecture-generic PPI numbers assumed from memory. */
|
||
#define TIMER_PPI_EL1 30u
|
||
#define TIMER_PPI_EL2 26u
|
||
|
||
/* GICv2 Distributor register byte offsets (ARM IHI 0048B; confirmed against
|
||
* Linux's arm-gic.h, see file header). */
|
||
#define GICD_CTLR 0x000
|
||
#define GICD_ISENABLER0 0x100
|
||
#define GICD_IPRIORITYR 0x400
|
||
#define GICD_ITARGETSR 0x800 /* item 4.3.5d -- SPI target-CPU byte array */
|
||
#define GICD_ICFGR 0xC00
|
||
#define GICD_ISPENDR 0x200 /* item 4.3.5d -- software set-pending, SPIs only */
|
||
|
||
/* GICv2 CPU Interface register byte offsets. */
|
||
#define GICC_CTLR 0x00
|
||
#define GICC_PMR 0x04
|
||
#define GICC_IAR 0x0C
|
||
#define GICC_EOIR 0x10
|
||
|
||
#define GICD_CTLR_ENABLE_GRP0 (1u << 0)
|
||
#define GICC_CTLR_ENABLE_GRP0 (1u << 0)
|
||
|
||
/* Priority: GICv2 compares as "interrupt allowed if priority < PMR", lower
|
||
* value = more urgent. One interrupt is configured, so the specific value
|
||
* only has to be numerically below the PMR mask set in apic_init(). */
|
||
#define TIMER_PRIORITY 0x80u
|
||
#define PMR_ALLOW_ALL 0xFFu
|
||
|
||
/* MMIO accessed as identity/physical addresses: arch_mmu_init() for aarch64
|
||
* is still the M3 stub (page-table bring-up deferred), so whatever mapping
|
||
* EDK2 left active at ExitBootServices governs -- the same assumption
|
||
* amd64/timer.c already documents and relies on for its own MMIO (HPET). */
|
||
static inline void mmio_write32(uintptr_t base, uint32_t offset, uint32_t val)
|
||
{
|
||
*(volatile uint32_t *)(base + offset) = val;
|
||
}
|
||
|
||
static inline uint32_t mmio_read32(uintptr_t base, uint32_t offset)
|
||
{
|
||
return *(volatile uint32_t *)(base + offset);
|
||
}
|
||
|
||
/* Set once by apic_init(), read by aarch64_irq_handler() to know which PPI
|
||
* to expect -- computed once (EL cannot change post-boot, same rationale
|
||
* as item 0.5's el2_mode_flag) rather than re-derived per interrupt. */
|
||
static uint32_t s_timer_ppi = TIMER_PPI_EL1;
|
||
|
||
static uint64_t s_timer_period_tsc = 0;
|
||
static uint64_t s_counter_hz_apic = 0; /* CNTFRQ_EL0 (item 0.8/§26) */
|
||
|
||
/**
|
||
* @brief Read one big-endian 32-bit cell from a devicetree property blob.
|
||
*
|
||
* `fdt.c` keeps its own `be32()` helper file-local (freestanding, no shared
|
||
* byte-swap utility to reuse) -- duplicated here rather than exposing it,
|
||
* same "a few lines is simpler than a new shared dependency" precedent
|
||
* `rpi5_dtb.c`'s own `reg_first_cell_be32()` already set for this exact
|
||
* situation.
|
||
*/
|
||
static uint32_t be32_cell(const unsigned char *b)
|
||
{
|
||
return ((uint32_t) b[0] << 24) | ((uint32_t) b[1] << 16) |
|
||
((uint32_t) b[2] << 8) | (uint32_t) b[3];
|
||
}
|
||
|
||
/**
|
||
* @brief Read one two-cell (64-bit) big-endian address from a `reg` entry.
|
||
*
|
||
* BCM2712's `gicv2` node (see file header) uses two address cells and two
|
||
* size cells per `reg` entry -- confirmed directly against `bcm2712.dtsi`'s
|
||
* `axi`/`gicv2` nodes (`#address-cells = <2>; #size-cells = <2>;`), unlike
|
||
* `rpi5_dtb.c`'s UART/mailbox lookups which read a single 32-bit cell
|
||
* because their `soc` parent node declares `#address-cells = <1>`. Reads
|
||
* only the two address cells; the caller advances past the two size cells
|
||
* itself via @c GIC_REG_ENTRY_BYTES to reach the next entry.
|
||
*/
|
||
static uint64_t be64_addr_cell_pair(const unsigned char *b)
|
||
{
|
||
return ((uint64_t) be32_cell(b) << 32) | (uint64_t) be32_cell(b + 4);
|
||
}
|
||
|
||
/* One gicv2 `reg` entry is 4 cells (2 address + 2 size) * 4 bytes = 16
|
||
* bytes. Entry 0 = GICD (distributor), entry 1 = GICC (CPU interface 0) --
|
||
* confirmed against bcm2712.dtsi's own `reg` list order (GICD, GICC, GICH,
|
||
* GICV, the standard arm,gic-400 binding order), not assumed. */
|
||
#define GIC_REG_ENTRY_BYTES 16u
|
||
|
||
/**
|
||
* @brief Try to discover the real GICD/GICC base addresses from the DTB.
|
||
*
|
||
* Looks up the `"arm,gic-400"` node and reads its `reg` property's first
|
||
* two entries. Returns 0 (leaving @p gicd_out / @p gicc_out untouched) if
|
||
* @p dtb is not a valid FDT, no matching node exists, or the `reg`
|
||
* property is shorter than two entries -- callers must keep their own
|
||
* QEMU-constant default in that case, exactly as before this function
|
||
* existed (FABRIC-3.md §IV.3 item 7 fix, 2026-09-04; see file header for
|
||
* why this is needed on the native Pi 5 path specifically).
|
||
*
|
||
* @param dtb Candidate devicetree blob (@c BootInfo->dtb); NULL-safe.
|
||
* @param gicd_out Receives the distributor base address on success.
|
||
* @param gicc_out Receives the CPU interface 0 base address on success.
|
||
* @return 1 on success, 0 if discovery failed for any reason.
|
||
*/
|
||
static int gic_bases_from_dtb(const void *dtb, uintptr_t *gicd_out, uintptr_t *gicc_out)
|
||
{
|
||
const void *node;
|
||
const unsigned char *reg;
|
||
uint32_t len;
|
||
|
||
if (!fdt_valid(dtb)) return 0;
|
||
|
||
node = fdt_find_node_by_compatible(dtb, "arm,gic-400");
|
||
if (!node) return 0;
|
||
|
||
reg = (const unsigned char *) fdt_find_prop_in_node(dtb, node, "reg", &len);
|
||
if (!reg || len < 2u * GIC_REG_ENTRY_BYTES) return 0;
|
||
|
||
*gicd_out = (uintptr_t) be64_addr_cell_pair(reg);
|
||
*gicc_out = (uintptr_t) be64_addr_cell_pair(reg + GIC_REG_ENTRY_BYTES);
|
||
return 1;
|
||
}
|
||
|
||
/**
|
||
* @brief Initialise the GICv2 distributor and CPU interface (M4 milestone).
|
||
*
|
||
* Enables exactly one interrupt source: the physical timer PPI for the
|
||
* exception level detected in item 0.4 (30 at EL1, 26 at EL2 — see
|
||
* @c TIMER_PPI_EL1 / @c TIMER_PPI_EL2 above). Sequence:
|
||
*
|
||
* 0. Try @c gic_bases_from_dtb() against @p boot_info->dtb; on success,
|
||
* @c s_gicd_base / @c s_gicc_base are overridden from the real
|
||
* hardware addresses. On failure (no DTB, e.g. this system's QEMU/UEFI
|
||
* firmware — see file header) they keep the QEMU constant defaults,
|
||
* exactly the behaviour this function had before this step existed.
|
||
* 1. Set this interrupt's priority (@c GICD_IPRIORITYR) below the CPU
|
||
* interface's priority mask, so it is never itself masked out.
|
||
* 2. Enable it in @c GICD_ISENABLER0 (PPIs 16–31 live in the first word).
|
||
* 3. Enable the distributor, Group 0 only (@c GICD_CTLR) — this deployment
|
||
* has no GICv2 security-extension split in use.
|
||
* 4. Set the CPU interface priority mask to allow everything through
|
||
* (@c GICC_PMR = 0xFF) and enable the CPU interface (@c GICC_CTLR).
|
||
*
|
||
* Deliberately not configured: @c GICD_ICFGR (trigger sensitivity). The
|
||
* ARM generic timer's PPI is architecturally level-sensitive; this is not
|
||
* a general GIC driver and item 0.6's own scope does not call for
|
||
* reconfiguring it.
|
||
*
|
||
* @param boot_info Kernel @c BootInfo; @c dtb is consulted for real-hardware
|
||
* GIC base discovery as of the 2026-09-04 fix (previously
|
||
* unused — see file header), and, as of the same fix, for
|
||
* @c aarch64_psci_conduit_init()'s PSCI conduit discovery
|
||
* (arch.c) — bundled into this call for the same reason
|
||
* GIC discovery is: this is the one point in boot with
|
||
* @c boot_info->dtb already in hand.
|
||
* @return 0 always.
|
||
*/
|
||
int apic_init(BootInfo *boot_info)
|
||
{
|
||
aarch64_psci_conduit_init(boot_info ? boot_info->dtb : (void *) 0);
|
||
|
||
if (boot_info && gic_bases_from_dtb(boot_info->dtb, &s_gicd_base, &s_gicc_base)) {
|
||
console_puts("GICv2: base addresses discovered from DTB (GICD=0x");
|
||
for (int s = 60; s >= 0; s -= 4)
|
||
console_putc("0123456789abcdef"[(s_gicd_base >> s) & 0xF]);
|
||
console_puts(", GICC=0x");
|
||
for (int s = 60; s >= 0; s -= 4)
|
||
console_putc("0123456789abcdef"[(s_gicc_base >> s) & 0xF]);
|
||
console_println(")");
|
||
} else {
|
||
console_println("GICv2: no DTB GIC node -- using QEMU virt-machine defaults");
|
||
}
|
||
|
||
s_timer_ppi = (aarch64_current_el() == 2) ? TIMER_PPI_EL2 : TIMER_PPI_EL1;
|
||
|
||
/* IPRIORITYR is byte-indexed, one byte per INTID. */
|
||
{
|
||
uint32_t reg_off = GICD_IPRIORITYR + (s_timer_ppi & ~3u);
|
||
uint32_t shift = (s_timer_ppi & 3u) * 8u;
|
||
uint32_t val = mmio_read32(s_gicd_base, reg_off);
|
||
val = (val & ~(0xFFu << shift)) | (TIMER_PRIORITY << shift);
|
||
mmio_write32(s_gicd_base, reg_off, val);
|
||
}
|
||
|
||
mmio_write32(s_gicd_base, GICD_ISENABLER0, 1u << s_timer_ppi);
|
||
mmio_write32(s_gicd_base, GICD_CTLR, GICD_CTLR_ENABLE_GRP0);
|
||
|
||
mmio_write32(s_gicc_base, GICC_PMR, PMR_ALLOW_ALL);
|
||
mmio_write32(s_gicc_base, GICC_CTLR, GICC_CTLR_ENABLE_GRP0);
|
||
|
||
console_puts("GICv2: distributor+CPU interface enabled, PPI ");
|
||
console_putc((char)('0' + s_timer_ppi / 10));
|
||
console_putc((char)('0' + s_timer_ppi % 10));
|
||
console_println("");
|
||
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* @brief Enable and route one SPI (item 4.3.5d).
|
||
*
|
||
* Extends item 0.6's PPI-only init just far enough to bring up one Shared
|
||
* Peripheral Interrupt -- still not a general GIC driver, still one
|
||
* explicit INTID per call, same bounded-scope posture 0.6 used.
|
||
*
|
||
* Programs, for @p intid (expected range: SPIs, INTID >= 32):
|
||
* 1. @c GICD_IPRIORITYR (byte-indexed, same pattern @c apic_init() already
|
||
* uses for the timer PPI) -- @c TIMER_PRIORITY, below @c PMR_ALLOW_ALL.
|
||
* 2. @c GICD_ISENABLER at @c 0x100 + 4*(intid/32) -- PPIs/SGIs (0-31) live
|
||
* in word 0 (@c GICD_ISENABLER0); SPIs beyond use word @c intid/32, same
|
||
* architecture layout, not board-specific.
|
||
* 3. @c GICD_ITARGETSR (byte-indexed like IPRIORITYR) -- routes to CPU 0
|
||
* only; this target has no @c -smp, so no other bit is ever valid.
|
||
* 4. @c GICD_ICFGR is read back, not written, unless the readback disagrees
|
||
* with the expected level-triggered configuration -- FABRIC-0.md §27.5.1's
|
||
* decoded QEMU `interrupt-map` says PCI legacy INTx on this board already
|
||
* is level-triggered by default, so this keeps item 0.6's "don't touch
|
||
* ICFGR unless forced to" posture rather than writing it unconditionally.
|
||
*
|
||
* Does **not** touch @c GICD_CTLR or @c GICC_CTLR/@c PMR -- @c apic_init()
|
||
* already enabled the distributor and CPU interface for the timer PPI, and
|
||
* those are global, not per-INTID; nothing here needs to repeat that.
|
||
*
|
||
* @param intid Target SPI's INTID (32 + GIC SPI number).
|
||
*/
|
||
void apic_spi_enable(uint32_t intid)
|
||
{
|
||
{
|
||
uint32_t reg_off = GICD_IPRIORITYR + (intid & ~3u);
|
||
uint32_t shift = (intid & 3u) * 8u;
|
||
uint32_t val = mmio_read32(s_gicd_base, reg_off);
|
||
val = (val & ~(0xFFu << shift)) | (TIMER_PRIORITY << shift);
|
||
mmio_write32(s_gicd_base, reg_off, val);
|
||
}
|
||
|
||
{
|
||
uint32_t reg_off = GICD_ISENABLER0 + 4u * (intid / 32u);
|
||
uint32_t bit = intid % 32u;
|
||
mmio_write32(s_gicd_base, reg_off, 1u << bit);
|
||
}
|
||
|
||
{
|
||
uint32_t reg_off = GICD_ITARGETSR + (intid & ~3u);
|
||
uint32_t shift = (intid & 3u) * 8u;
|
||
uint32_t val = mmio_read32(s_gicd_base, reg_off);
|
||
val = (val & ~(0xFFu << shift)) | (0x01u << shift); /* CPU 0 only */
|
||
mmio_write32(s_gicd_base, reg_off, val);
|
||
}
|
||
|
||
{
|
||
uint32_t reg_off = GICD_ICFGR + 4u * (intid / 16u);
|
||
uint32_t bitpos = (intid % 16u) * 2u;
|
||
uint32_t val = mmio_read32(s_gicd_base, reg_off);
|
||
uint32_t level_expected = 0u; /* level-triggered = the "trigger mode"
|
||
* bit (bit 1 of the 2-bit field) clear */
|
||
if (((val >> bitpos) & 0x2u) != level_expected) {
|
||
val = (val & ~(0x3u << bitpos)) | (level_expected << bitpos);
|
||
mmio_write32(s_gicd_base, reg_off, val);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @brief Read the GIC CPU interface's Interrupt Acknowledge Register.
|
||
*
|
||
* Reading @c GICC_IAR returns the INTID of the highest-priority pending
|
||
* interrupt and simultaneously moves it to the active state — this is the
|
||
* GICv2 acknowledgement step. Called from @c aarch64_irq_handler() in
|
||
* @c interrupts.c to identify which interrupt fired; the matching
|
||
* @c apic_eoi_intid() call below completes it.
|
||
*
|
||
* @return INTID (10 bits used by GICv2).
|
||
*/
|
||
uint32_t apic_read_iar(void)
|
||
{
|
||
return mmio_read32(s_gicc_base, GICC_IAR);
|
||
}
|
||
|
||
/**
|
||
* @brief Return the timer PPI this build is watching for.
|
||
*
|
||
* Set once in @c apic_init() from @c aarch64_current_el() (30 at EL1, 26 at
|
||
* EL2). @c aarch64_irq_handler() compares @c apic_read_iar()'s result
|
||
* against this rather than a hardcoded constant.
|
||
*/
|
||
uint32_t apic_timer_ppi(void)
|
||
{
|
||
return s_timer_ppi;
|
||
}
|
||
|
||
/**
|
||
* @brief Signal End of Interrupt to the GIC CPU interface.
|
||
*
|
||
* Writes @p intid back to @c GICC_EOIR, matching the value @c GICC_IAR
|
||
* returned in @c aarch64_irq_handler(). Required after every acknowledged
|
||
* interrupt or the GIC considers it permanently active and will not
|
||
* re-signal it.
|
||
*
|
||
* @param intid INTID as read from @c GICC_IAR (10 bits used by GICv2).
|
||
*/
|
||
void apic_eoi_intid(uint32_t intid)
|
||
{
|
||
mmio_write32(s_gicc_base, GICC_EOIR, intid);
|
||
}
|
||
|
||
/* FABRIC-3.10.8 MC-3b: the wake SGI. */
|
||
#define GICD_SGIR 0xF00
|
||
#define GIC_WAKE_SGI 0u
|
||
|
||
/**
|
||
* @brief On a secondary core: its banked GIC state, for the wake SGI only.
|
||
*
|
||
* SGI/PPI enables and priorities (GICD_ISENABLER0, GICD_IPRIORITYR 0-31)
|
||
* and the whole CPU interface are banked per core, so each core sets its
|
||
* own; the distributor itself is already enabled by core 0's apic_init().
|
||
* Its timer PPI stays disabled.
|
||
*
|
||
* @return This core's CPU-interface target mask, read from its own banked
|
||
* GICD_ITARGETSR0 (byte 0 names the reading core).
|
||
*/
|
||
uint32_t apic_secondary_init_gic(void)
|
||
{
|
||
uint32_t val = mmio_read32(s_gicd_base, GICD_IPRIORITYR); /* SGIs 0-3 */
|
||
val = (val & ~0xFFu) | TIMER_PRIORITY; /* SGI 0 */
|
||
mmio_write32(s_gicd_base, GICD_IPRIORITYR, val);
|
||
mmio_write32(s_gicd_base, GICD_ISENABLER0, 1u << GIC_WAKE_SGI);
|
||
mmio_write32(s_gicc_base, GICC_PMR, PMR_ALLOW_ALL);
|
||
mmio_write32(s_gicc_base, GICC_CTLR, GICC_CTLR_ENABLE_GRP0);
|
||
return mmio_read32(s_gicd_base, GICD_ITARGETSR) & 0xFFu;
|
||
}
|
||
|
||
/** @brief Send the wake SGI to the cores in @p target_mask. */
|
||
void apic_send_wake_sgi(uint32_t target_mask)
|
||
{
|
||
__asm__ volatile ("dsb ishst" ::: "memory");
|
||
mmio_write32(s_gicd_base, GICD_SGIR, ((target_mask & 0xFFu) << 16) | GIC_WAKE_SGI);
|
||
}
|
||
|
||
/**
|
||
* @brief Signal End of Interrupt (common cross-ISA signature — see below).
|
||
*
|
||
* The shared @c apic.h interface has no way to pass which interrupt is
|
||
* being completed, but GICv2's @c GICC_EOIR needs the exact INTID @c GICC_IAR
|
||
* returned, not just "an" EOI. @c aarch64_irq_handler() calls
|
||
* @c apic_eoi_intid() directly with that value instead; this form exists
|
||
* only to satisfy the common call site and is not used on the interrupt
|
||
* path GIC brings up.
|
||
*/
|
||
void apic_eoi(void)
|
||
{
|
||
}
|
||
|
||
/**
|
||
* @brief Compute the expected system-counter ticks per heartbeat period.
|
||
*
|
||
* Computes @c s_timer_period_tsc (the expected number of @c CNTPCT_EL0
|
||
* ticks between heartbeats), used both by @c heartbeat_init() to seed the
|
||
* rolling-window @c expected_delta and by @c apic_timer_start() /
|
||
* @c apic_timer_rearm() (item 0.7) as the @c TVAL step written on every
|
||
* arm and re-arm.
|
||
*
|
||
* If either @p tsc_hz or @p tick_hz is zero, a fallback of 10,000,000
|
||
* ticks is stored (approximately 160 ms at 62.5 MHz, a safe non-zero
|
||
* sentinel that prevents division-by-zero in the heartbeat path).
|
||
*
|
||
* @param tsc_hz System counter frequency from @c timer_init() (Hz).
|
||
* @param tick_hz Desired heartbeat rate (Hz); 0 → uses fallback.
|
||
* @return 0 always.
|
||
*/
|
||
int apic_timer_init(uint64_t tsc_hz, uint32_t tick_hz)
|
||
{
|
||
if (tick_hz > 0 && tsc_hz > 0) {
|
||
s_timer_period_tsc = tsc_hz / tick_hz;
|
||
} else {
|
||
s_timer_period_tsc = 10000000;
|
||
}
|
||
s_counter_hz_apic = tsc_hz;
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* @brief Arm the ARM Generic Timer and enable its interrupt (item 0.7).
|
||
*
|
||
* Writes @c CNTP_TVAL_EL0 (or @c CNTHP_TVAL_EL2 at EL2 — @c
|
||
* aarch64_current_el(), same EL-aware discipline as items 0.4–0.6) to
|
||
* @c s_timer_period_tsc, then sets @c CTL.ENABLE=1, @c CTL.IMASK=0.
|
||
*
|
||
* @c TVAL is a signed 32-bit *relative* countdown — writing it sets
|
||
* "fire @c s_timer_period_tsc ticks from this instant," unlike riscv64's
|
||
* SBI interface (item 0.3), which takes an absolute deadline the caller
|
||
* must compute. No "now" reading is needed here at all.
|
||
*
|
||
* The @c ISB after the @c CTL write is required, not decorative: per the
|
||
* architecture, and confirmed against Linux's own
|
||
* @c arch_timer_reg_write_cp15() (arch/arm64/include/asm/arch_timer.h),
|
||
* only the *control* register write needs synchronising before the
|
||
* enable/mask state is guaranteed visible to the interrupt pipeline —
|
||
* @c TVAL/@c CVAL writes do not carry the same requirement, which is why
|
||
* @c apic_timer_rearm() below omits it.
|
||
*
|
||
* @c s_timer_period_tsc must already be set by @c apic_timer_init(). If
|
||
* it is still 0, @c TVAL=0 fires immediately and continuously until
|
||
* @c apic_timer_rearm() runs — a correctly-functioning if oddly-paced
|
||
* timer, not a hang.
|
||
*/
|
||
void apic_timer_start(void)
|
||
{
|
||
/* CNTx_TVAL_EL is architecturally 32-bit (bits[63:32] RES0 on write),
|
||
* but MSR to a system register is always a 64-bit (Xn) instruction
|
||
* form -- there is no 32-bit encoding to match a narrower operand.
|
||
* Truncate to 32 bits ourselves, then zero-extend back to 64 for the
|
||
* operand: this supplies explicit, provably-correct zeros in the RES0
|
||
* field rather than depending on how the hardware treats whatever
|
||
* upper bits a narrower operand happened to leave there -- a
|
||
* distinction we could not confirm against a local authoritative
|
||
* source (unlike the GICv2 offsets and CTL bit layout above,
|
||
* cross-checked against Linux headers on this system; TVAL is not,
|
||
* since Linux's own driver only ever uses the 64-bit CVAL form and
|
||
* never exercises this path). */
|
||
uint64_t tval = (uint64_t)(uint32_t)s_timer_period_tsc;
|
||
|
||
if (aarch64_current_el() == 2) {
|
||
__asm__ volatile ("msr cnthp_tval_el2, %0" :: "r"(tval));
|
||
__asm__ volatile ("msr cnthp_ctl_el2, %0" :: "r"((uint64_t)1) : "memory");
|
||
} else {
|
||
__asm__ volatile ("msr cntp_tval_el0, %0" :: "r"(tval));
|
||
__asm__ volatile ("msr cntp_ctl_el0, %0" :: "r"((uint64_t)1) : "memory");
|
||
}
|
||
__asm__ volatile ("isb" ::: "memory");
|
||
}
|
||
|
||
/**
|
||
* @brief Re-arm the ARM Generic Timer for the next period (item 0.7).
|
||
*
|
||
* Writes a fresh @c TVAL — the entirety of a periodic re-arm, since @c TVAL
|
||
* is relative to "now" rather than an absolute deadline. Unlike riscv64's
|
||
* SBI re-arm (item 0.3), there is no drift-correction bookkeeping: each
|
||
* write means "@c s_timer_period_tsc ticks from this instant," so time
|
||
* spent servicing the interrupt before this call simply lengthens that one
|
||
* period rather than accumulating error across ticks.
|
||
*
|
||
* **This is the one call that must never be skipped.** The ARM Generic
|
||
* Timer does not auto-reload: once @c TVAL crosses zero, @c CTL.ISTATUS
|
||
* latches and stays asserted — a level condition — until software writes
|
||
* a new @c TVAL or clears @c CTL.ENABLE. A return path that skips this
|
||
* leaves the interrupt permanently pending, which the GIC would deliver
|
||
* again the instant it is EOI'd: a real interrupt storm, not a
|
||
* hypothetical one (this is exactly the failure this driver was checked
|
||
* against during item 0.6's own verification, and confirmed absent there
|
||
* only because nothing armed the timer yet).
|
||
*
|
||
* Called from @c aarch64_irq_handler() (interrupts.c) before
|
||
* @c heartbeat_tick() — same ordering discipline as
|
||
* @c riscv64_timer_rearm() (item 0.3): re-arm first, so a fault in the
|
||
* rest of the handler cannot also cost the next tick.
|
||
*
|
||
* The step written is @c heartbeat_next_period_ns() (item 0.8, §26)
|
||
* converted to @c CNTPCT_EL0 ticks via @c s_counter_hz_apic, not the fixed
|
||
* @c s_timer_period_tsc @c apic_timer_start() used for the very first arm
|
||
* -- that value remains @c apic_timer_period_tsc()'s return for
|
||
* @c heartbeat_init()'s initial @c expected_delta, unchanged.
|
||
*/
|
||
void apic_timer_rearm(void)
|
||
{
|
||
uint64_t period_ticks = (s_counter_hz_apic > 0)
|
||
? (heartbeat_next_period_ns() * s_counter_hz_apic) / 1000000000ULL
|
||
: s_timer_period_tsc;
|
||
if (period_ticks == 0) {
|
||
period_ticks = 1;
|
||
}
|
||
/* uint64_t, zero-extended -- see apic_timer_start() for why. */
|
||
uint64_t tval = (uint64_t)(uint32_t)period_ticks;
|
||
|
||
if (aarch64_current_el() == 2) {
|
||
__asm__ volatile ("msr cnthp_tval_el2, %0" :: "r"(tval));
|
||
} else {
|
||
__asm__ volatile ("msr cntp_tval_el0, %0" :: "r"(tval));
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @brief Stop timer delivery by clearing CTL.ENABLE (item 0.7).
|
||
*
|
||
* Per the architecture, @c ENABLE=0 unconditionally stops the count and
|
||
* de-asserts the interrupt condition regardless of @c IMASK — a single
|
||
* write is sufficient, no separate mask step needed.
|
||
*/
|
||
void apic_timer_stop(void)
|
||
{
|
||
if (aarch64_current_el() == 2) {
|
||
__asm__ volatile ("msr cnthp_ctl_el2, %0" :: "r"((uint64_t)0) : "memory");
|
||
} else {
|
||
__asm__ volatile ("msr cntp_ctl_el0, %0" :: "r"((uint64_t)0) : "memory");
|
||
}
|
||
__asm__ volatile ("isb" ::: "memory");
|
||
}
|
||
|
||
/**
|
||
* @brief Return the expected system-counter ticks per heartbeat period.
|
||
*
|
||
* Returns @c s_timer_period_tsc, set by @c apic_timer_init() as
|
||
* @c tsc_hz / @c tick_hz. This value is consumed by @c heartbeat_init()
|
||
* to seed the rolling-window @c expected_delta so that inter-tick
|
||
* deviation measurements are relative to the correct nominal interval.
|
||
*
|
||
* @return Expected @c CNTPCT_EL0 ticks per heartbeat period; 10,000,000
|
||
* if frequencies were not available at init time.
|
||
*/
|
||
uint64_t apic_timer_period_tsc(void)
|
||
{
|
||
return s_timer_period_tsc;
|
||
}
|