StarForth/experiments/bare_metal/analysis/analyse_multiuser_doe.R

#!/usr/bin/env Rscript
# analyse_multiuser_doe.R
# StarForth LithosAnanke — Multiuser/Multitasking DoE Analysis
#
# Analyses trial-summary records from the multiuser/multitasking campaign
# (multiuser-doe.4th / per-isa-doe.4th, FABRIC-3.md §XXXIII/§XXXV) -- one
# row per completed trial: concurrency level (nw: 2/4/8) x mode (0=uniform,
# 1=heterogeneous), fail count, and trial-end fleet physics (fleet_k_q48,
# fleet_conserved, switch_count_cumulative).
#
# Unlike analyse_multirep.R/analyse_bare_metal.R (which analyse dense
# per-tick HB-ON streams from a fixed manifest of completed runs), this
# script auto-discovers whatever CSV(s) are present and is designed to be
# re-run repeatedly WHILE a campaign is still in progress -- it degrades
# gracefully to a "no data yet" placeholder rather than erroring when a
# campaign has produced 0 trials so far, and re-summarises from scratch
# each run against however many trials currently exist. Input format is
# extract_doe_region.py's own CSV schema, or (for smoke-testing this
# pipeline before real campaign data exists) any CSV in this shape.

suppressPackageStartupMessages({
  library(ggplot2)
  library(svglite)
  library(dplyr)
  library(scales)
})

# sys.frames()[[1]]$ofile is unreliable depending on how Rscript was
# invoked (silently falls back to getwd(), which is wrong whenever this
# is called from a different working directory -- e.g. an orchestration
# script that cd's to the repo root first). commandArgs()'s "--file="
# entry is the robust Rscript idiom and is what's used here.
get_script_dir <- function() {
  raw_args <- commandArgs(trailingOnly = FALSE)
  file_arg <- grep("^--file=", raw_args, value = TRUE)
  if (length(file_arg) == 1) {
    return(dirname(normalizePath(sub("^--file=", "", file_arg))))
  }
  tryCatch(dirname(normalizePath(sys.frames()[[1]]$ofile)), error = function(e) getwd())
}
SCRIPT_DIR <- get_script_dir()
BASE_DIR    <- normalizePath(file.path(SCRIPT_DIR, ".."))
OUT_CHARTS  <- file.path(SCRIPT_DIR, "charts")
OUT_TABLES  <- file.path(SCRIPT_DIR, "tables")
OUT_SECTION <- file.path(SCRIPT_DIR, "report", "sections", "multiuser_summary_autogen.tex")
dir.create(OUT_CHARTS, showWarnings = FALSE, recursive = TRUE)
dir.create(OUT_TABLES, showWarnings = FALSE, recursive = TRUE)
dir.create(dirname(OUT_SECTION), showWarnings = FALSE, recursive = TRUE)

# LaTeX-escape a plain string for safe \newcommand/text use.
tex_escape <- function(s) {
  s <- gsub("\\\\", "\\\\textbackslash{}", s)
  s <- gsub("([%&_#{}$])", "\\\\\\1", s)
  s
}

write_tex_fragment <- function(lines) {
  writeLines(lines, OUT_SECTION)
  cat(sprintf("  Saved: %s\n", OUT_SECTION))
}

# ── input: first CLI arg is a CSV path, or a directory to scan; default
# is the extractor's own conventional output location ──────────────────
args <- commandArgs(trailingOnly = TRUE)
input_arg <- if (length(args) >= 1) args[1] else file.path(BASE_DIR, "runs", "multiuser-doe-trials.csv")

cat("══════════════════════════════════════════════════════════════════\n")
cat("  StarForth LithosAnanke — Multiuser/Multitasking DoE Analysis\n")
cat("══════════════════════════════════════════════════════════════════\n\n")

load_trials <- function(path) {
  if (dir.exists(path)) {
    files <- list.files(path, pattern = "\\.csv$", full.names = TRUE)
    if (length(files) == 0) return(NULL)
    dfs <- lapply(files, function(f) tryCatch(read.csv(f, stringsAsFactors = FALSE), error = function(e) NULL))
    dfs <- dfs[!sapply(dfs, is.null)]
    dfs <- dfs[sapply(dfs, function(d) all(c("cfg","nw","mode","fail_count") %in% names(d)))]
    if (length(dfs) == 0) return(NULL)
    bind_rows(dfs)
  } else if (file.exists(path)) {
    df <- read.csv(path, stringsAsFactors = FALSE)
    if (!all(c("cfg","nw","mode","fail_count") %in% names(df))) return(NULL)
    df
  } else {
    NULL
  }
}

df <- load_trials(input_arg)
n_trials <- if (is.null(df)) 0 else nrow(df)
cat(sprintf("  Input: %s\n  Trials found: %d\n\n", input_arg, n_trials))

Q48 <- 65536.0

write_placeholder <- function(reason) {
  cat(sprintf("  %s -- writing placeholder tables/charts so the report build\n", reason))
  cat("  still completes (this pipeline is meant to be re-run while the\n")
  cat("  campaign is still in progress).\n\n")

  placeholder_tbl <- data.frame(status = reason, trials_so_far = n_trials)
  write.csv(placeholder_tbl, file.path(OUT_TABLES, "multiuser_status.csv"), row.names = FALSE)

  p <- ggplot() +
    annotate("text", x = 0, y = 0, label = paste0(reason, "\n(", n_trials, " trials so far)"),
             size = 6, colour = "grey40") +
    theme_void()
  svglite(file.path(OUT_CHARTS, "multiuser_placeholder.svg"), width = 8, height = 4)
  print(p)
  dev.off()
  cat("  Saved: multiuser_placeholder.svg\n\n")
}

if (n_trials == 0) {
  write_placeholder("No trials persisted yet")
  write_tex_fragment(c(
    "% Auto-generated by analyse_multiuser_doe.R -- do not edit by hand.",
    "No trial data has been persisted to the DoE region yet. This section",
    "will populate automatically the next time this report is built after",
    "the campaign has produced at least one completed trial."
  ))
  quit(save = "no", status = 0)
}

# ── normalise / label factors ────────────────────────────────────────────
df$nw   <- factor(df$nw, levels = c(2, 4, 8))
df$mode <- factor(df$mode, levels = c(0, 1), labels = c("uniform", "heterogeneous"))
df$cfg  <- factor(df$cfg)
if (!"isa" %in% names(df) || all(df$isa == "" | is.na(df$isa))) df$isa <- "amd64"
df$isa  <- factor(df$isa, levels = intersect(c("amd64","aarch64","riscv64"), unique(df$isa)))
df$fleet_k <- df$fleet_k_q48 / Q48

# ── 1. Per-cfg summary ───────────────────────────────────────────────────
cat("Computing per-cfg summary (concurrency x mode)...\n")
cfg_tbl <- df %>%
  group_by(nw, mode) %>%
  summarise(
    n_trials        = n(),
    n_fail          = sum(fail_count > 0),
    fail_rate       = mean(fail_count > 0),
    mean_fail_count = mean(fail_count),
    mean_fleet_k    = mean(fleet_k, na.rm = TRUE),
    pct_conserved   = mean(fleet_conserved == 1, na.rm = TRUE),
    mean_switch_cnt = mean(switch_count_cumulative, na.rm = TRUE),
    .groups = "drop"
  )
write.csv(cfg_tbl, file.path(OUT_TABLES, "multiuser_cell_summary.csv"), row.names = FALSE)
cat("  Saved: multiuser_cell_summary.csv\n")
print(as.data.frame(cfg_tbl))
cat("\n")

# ── 2. Per-ISA summary (only meaningful once >1 ISA present) ────────────
if (nlevels(df$isa) > 1) {
  cat("Computing per-ISA summary...\n")
  isa_tbl <- df %>%
    group_by(isa) %>%
    summarise(
      n_trials  = n(),
      fail_rate = mean(fail_count > 0),
      mean_fleet_k = mean(fleet_k, na.rm = TRUE),
      .groups = "drop"
    )
  write.csv(isa_tbl, file.path(OUT_TABLES, "multiuser_isa_summary.csv"), row.names = FALSE)
  cat("  Saved: multiuser_isa_summary.csv\n\n")
}

# ── colour theme (house style, matches analyse_multirep.R) ──────────────
theme_light_report <- function() {
  theme_minimal(base_size = 11) %+replace%
    theme(
      panel.grid.minor = element_blank(),
      panel.grid.major = element_line(colour = "grey90"),
      strip.text       = element_text(face = "bold"),
      plot.title       = element_text(face = "bold", size = 12),
      plot.subtitle    = element_text(colour = "grey40", size = 9),
      legend.position  = "bottom"
    )
}

save_svg <- function(plot, name, w = 10, h = 6) {
  path <- file.path(OUT_CHARTS, paste0(name, ".svg"))
  svglite(path, width = w, height = h)
  print(plot)
  dev.off()
  cat(sprintf("  Saved: %s.svg\n", name))
  invisible(path)
}

cat("Generating charts...\n\n")

# ── CHART MU-1: fail rate by concurrency x mode ──────────────────────────
cat("[MU-1] Fail rate by concurrency x mode...\n")
p_fail <- ggplot(cfg_tbl, aes(x = nw, y = fail_rate, fill = mode)) +
  geom_col(position = "dodge", alpha = 0.85) +
  geom_text(aes(label = sprintf("%d/%d", n_fail, n_trials)),
            position = position_dodge(width = 0.9), vjust = -0.4, size = 3) +
  scale_y_continuous(labels = percent_format(accuracy = 1), limits = c(0, NA)) +
  scale_fill_manual(values = c(uniform = "#4A90D9", heterogeneous = "#E07B39")) +
  labs(
    title    = "Trial Fail Rate by Concurrency Level x Workload Mode",
    subtitle = paste0(n_trials, " trials"),
    x        = "Concurrent workers",
    y        = "Fraction of trials with fail_count > 0"
  ) +
  theme_light_report()
save_svg(p_fail, "multiuser_fail_rate", w = 8, h = 6)

# ── CHART MU-2: switch_count_cumulative distribution by concurrency ─────
cat("[MU-2] Switch count by concurrency...\n")
p_switch <- ggplot(df, aes(x = nw, y = switch_count_cumulative, fill = nw)) +
  geom_boxplot(alpha = 0.8, outlier.alpha = 0.4) +
  scale_fill_manual(values = c("2" = "#50C878", "4" = "#4A90D9", "8" = "#E07B39"), guide = "none") +
  labs(
    title    = "Cumulative Preemptive Switch Count by Concurrency Level",
    subtitle = "Stage-3 switch signal count at trial end, by number of concurrent workers",
    x        = "Concurrent workers",
    y        = "switch_count_cumulative"
  ) +
  theme_light_report()
save_svg(p_switch, "multiuser_switch_count", w = 8, h = 6)

# ── CHART MU-3: fleet_k_q48 (conservation) across trials in run order ───
cat("[MU-3] Fleet K conservation across trial sequence...\n")
df_seq <- df %>% arrange(run_id)
p_fleetk <- ggplot(df_seq, aes(x = run_id, y = fleet_k, colour = mode)) +
  geom_point(alpha = 0.6, size = 1.5) +
  geom_hline(yintercept = 1.0, colour = "grey50", linetype = "dashed") +
  scale_colour_manual(values = c(uniform = "#4A90D9", heterogeneous = "#E07B39")) +
  labs(
    title    = "Fleet Heat Conservation (K) Across the Trial Sequence",
    subtitle = "Dashed line = Q48.16 conserved value (1.0); trial-end reads, shuffled run order",
    x        = "Run id (shuffled trial index)",
    y        = "fleet_k_q48 / 65536"
  ) +
  theme_light_report()
save_svg(p_fleetk, "multiuser_fleet_k_sequence", w = 10, h = 6)

# ── CHART MU-4: per-ISA panel (only when >1 ISA present) ─────────────────
if (nlevels(df$isa) > 1) {
  cat("[MU-4] Per-ISA fail rate panel...\n")
  p_isa <- ggplot(df %>% group_by(isa, nw, mode) %>%
                     summarise(fail_rate = mean(fail_count > 0), .groups = "drop"),
                   aes(x = nw, y = fail_rate, fill = mode)) +
    geom_col(position = "dodge", alpha = 0.85) +
    scale_y_continuous(labels = percent_format(accuracy = 1), limits = c(0, NA)) +
    scale_fill_manual(values = c(uniform = "#4A90D9", heterogeneous = "#E07B39")) +
    facet_wrap(~ isa, ncol = 3) +
    labs(
      title    = "Fail Rate by Concurrency x Mode, per ISA",
      x        = "Concurrent workers",
      y        = "Fraction of trials with fail_count > 0"
    ) +
    theme_light_report()
  save_svg(p_isa, "multiuser_isa_panel", w = 12, h = 5)
}

# ── summary ───────────────────────────────────────────────────────────────
cat(sprintf("\n══════════════════════════════════════════════════════════════════\n"))
cat(sprintf("  Multiuser DoE analysis complete. %d trials, %d cfg cells.\n",
            n_trials, nrow(cfg_tbl)))
cat(sprintf("══════════════════════════════════════════════════════════════════\n\n"))

overall_fail_rate <- mean(df$fail_count > 0)
overall_conserved <- mean(df$fleet_conserved == 1, na.rm = TRUE)
cat(sprintf("  Overall fail rate: %.2f%%\n", overall_fail_rate * 100))
cat(sprintf("  Fraction of trials conserved (fleet_conserved==1): %.2f%%\n", overall_conserved * 100))

# ── auto-generated LaTeX summary fragment (re-written every run) ────────
isa_list <- paste(levels(df$isa), collapse = ", ")
cell_lines <- apply(cfg_tbl, 1, function(r) {
  sprintf("  \\item %s workers, %s: %s trials, fail rate %.1f\\%%, mean fleet K = %.4f",
          r["nw"], tex_escape(as.character(r["mode"])), r["n_trials"],
          as.numeric(r["fail_rate"]) * 100, as.numeric(r["mean_fleet_k"]))
})
write_tex_fragment(c(
  "% Auto-generated by analyse_multiuser_doe.R -- do not edit by hand.",
  sprintf("This report reflects \\textbf{%d} completed trial(s) persisted via", n_trials),
  "\\texttt{DOE-PERSIST-TRIAL} as of the last extraction, spanning ISA(s)",
  sprintf("\\texttt{%s}. Overall fail rate across all trials so far is", tex_escape(isa_list)),
  sprintf("\\textbf{%.1f\\%%}; \\textbf{%.1f\\%%} of trials read", overall_fail_rate * 100, overall_conserved * 100),
  "\\texttt{fleet\\_conserved==1} at trial end.",
  "",
  "\\begin{itemize}",
  cell_lines,
  "\\end{itemize}"
))