Replace the composition chart with a probability chart that actually sums to 100%

scripts/charts/combined_fault_probability.r

@@ -4,7 +4,17 @@ library(readr)
 library(stringr)
 library(tidyr)
 
-# Usage: Rscript marker_composition.r exp_abspath1 ... resultsdata_file
+# Usage: Rscript combined_fault_probability.r exp_abspath1 ... resultsdata_file
+#
+# Divides by the faultspace area instead of by a marker total, which makes the
+# running modes comparable: raw counts scale with how long WAMR runs, so they
+# say more about execution length than about susceptibility.
+#
+# Each segment is P[outcome] for a uniformly random single-bit flip in the
+# traced fault space, so a bar's height is P[anything goes wrong].
+#
+# Replaces combined_fault_composition.r, which divided by the marker total and
+# was therefore this chart with every bar rescaled to 100%.
 
 args <- commandArgs(trailingOnly = TRUE)
 if (length(args) < 2) {
@@ -37,7 +47,6 @@ extract_info <- function(path) {
 
 # Load data
 all_data <- data.frame()
-weight_data <- data.frame()
 
 for (arg in exp_args) {
   info <- extract_info(arg)
@@ -56,37 +65,48 @@ for (arg in exp_args) {
   all_data <- bind_rows(all_data, df)
 }
 
+# TODO: Finally put all the bullshit before this in some shared space
+
 if (nrow(all_data) == 0) {
   stop("No data loaded")
 }
 
-# Skip OK_MARKERs, sum GROUP1 + TRAP.
+marker_order <- c(
-all_data <- all_data |>
+  "OK_MARKER",
-  filter(resulttype != "OK_MARKER") |>
+  "DETECTED_MARKER",
-  mutate(
+  "GROUP1_MARKER",
-    resulttype = ifelse(resulttype == "GROUP1_MARKER", "TRAP", resulttype)
+  "TRAP",
-  ) |>
+  "TIMEOUT",
-  group_by(base_name, variant, benchmark, resulttype) |>
+  "WRITE_TEXTSEGMENT",
-  summarise(faults = sum(faults), .groups = "drop")
+  "ACCESS_OUTERSPACE",
+  "FAIL_MARKER"
+)
 
-out_suffix <- gsub("^resultsdata|\\.csv$", "", csv_suffix)
+# Don't merge GROUP1_MARKER into TRAP for this chart
-
+# Also keep the OK_MARKERs, so the "sum to 100%" is accurate
-# Calculate percentages
+probability <- all_data |>
-composition <- all_data |>
   group_by(base_name, variant, benchmark) |>
-  mutate(frac = faults / sum(faults)) |>
+  mutate(frac = faults / sum(faults, na.rm = TRUE)) |>
   ungroup()
 
+# Don't print alphabetically
+probability$resulttype <- factor(probability$resulttype, levels = marker_order)
+probability$variant <- factor(
+  probability$variant,
+  levels = c("c", "aot", "interp")
+)
+
 plot <- ggplot(
-  composition |> filter(variant %in% c("aot", "interp")),
+  probability,
   aes(x = variant, y = frac, fill = resulttype)
 ) +
   geom_col() +
   facet_grid(benchmark ~ base_name) +
+  scale_y_continuous(labels = scales::percent) +
   labs(
-    title = "Marker Composition (AOT vs Interp)",
+    title = "Fault Probability per Fault Space",
     x = NULL,
-    y = "Percentage of Faults",
+    y = "Probability of Failure",
     fill = "Fault Type"
   ) +
   theme_minimal() +
@@ -95,9 +115,6 @@ plot <- ggplot(
     axis.text.x = element_text(angle = 45, hjust = 1)
   )
 
-filename <- paste0(
+out_suffix <- gsub("^resultsdata|\\.csv$", "", csv_suffix)
-  "injections/fault_composition",
+filename <- paste0("injections/fault_probability", out_suffix, ".svg")
-  out_suffix,
-  ".svg"
-)
 ggsave(filename, plot = plot, width = 13, height = 8)

scripts/menu.pl

@@ -653,15 +653,15 @@ my %handlers = (
 
         # Sucks to put those here but I can't write them inside the R scripts
         my %chart_descriptions = (
-            combined_fault_composition =>
-              'stacked fault type percentages/composition (resultsdata.csv).',
-
             combined_fault_count_comparison =>
 'faults per benchmark, c/aot/interp side by side (resultsdata.csv).',
 
             combined_fault_count_correlation =>
 'correlation of raw aot vs. interp fault counts (resultsdata.csv).',
 
+            combined_fault_probability =>
+'marker probability per fault space (resultsdata.csv [+ traceweight.csv]).',
+
             combined_fault_rates_per_instruction =>
 'faults normalised by instruction count (faults.csv + mnemonics.csv).',
 
@@ -698,7 +698,7 @@ my %handlers = (
           } @selected_charts;
         my @resultsdata_charts =
           grep {
-/_result|_fault_count_comparison|_ratio_comparison|_fault_count_correlation|_fault_composition/
+/_result|_fault_count_comparison|_ratio_comparison|_fault_count_correlation|_fault_probability/
           } @selected_charts;
 
         # Select if faults.csv or a filtered variant should be used