From bf94e837385f183765a3a22d73dc587b387ffb2b Mon Sep 17 00:00:00 2001 From: Christoph Urlacher Date: Mon, 17 Aug 2026 23:40:30 +0200 Subject: [PATCH] Replace the composition chart with a probability chart that actually sums to 100% --- ..._native_call_data+no_native_call_instr.svg | 3 - ..._native_call_data+no_native_call_instr.svg | 3 + ...osition.r => combined_fault_probability.r} | 63 ++++++++++++------- scripts/menu.pl | 8 +-- 4 files changed, 47 insertions(+), 30 deletions(-) delete mode 100644 injections/fault_composition_no_native_call_data+no_native_call_instr.svg create mode 100644 injections/fault_probability_no_native_call_data+no_native_call_instr.svg rename scripts/charts/{combined_fault_composition.r => combined_fault_probability.r} (54%) diff --git a/injections/fault_composition_no_native_call_data+no_native_call_instr.svg b/injections/fault_composition_no_native_call_data+no_native_call_instr.svg deleted file mode 100644 index d555fd5..0000000 --- a/injections/fault_composition_no_native_call_data+no_native_call_instr.svg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d1d5b9b6586c31292124c09029fa94d205a711fd00758654f46debb97c8be3eb -size 102366 diff --git a/injections/fault_probability_no_native_call_data+no_native_call_instr.svg b/injections/fault_probability_no_native_call_data+no_native_call_instr.svg new file mode 100644 index 0000000..c52c47f --- /dev/null +++ b/injections/fault_probability_no_native_call_data+no_native_call_instr.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07140de5178959f82a93a841c3bc06e332cebaac1ff027a3423f5ca9ac9b84f0 +size 164492 diff --git a/scripts/charts/combined_fault_composition.r b/scripts/charts/combined_fault_probability.r similarity index 54% rename from scripts/charts/combined_fault_composition.r rename to scripts/charts/combined_fault_probability.r index 08bd400..53005ad 100644 --- a/scripts/charts/combined_fault_composition.r +++ b/scripts/charts/combined_fault_probability.r @@ -4,7 +4,17 @@ library(readr) library(stringr) library(tidyr) -# Usage: Rscript marker_composition.r exp_abspath1 ... resultsdata_file +# Usage: Rscript combined_fault_probability.r exp_abspath1 ... resultsdata_file +# +# Divides by the faultspace area instead of by a marker total, which makes the +# running modes comparable: raw counts scale with how long WAMR runs, so they +# say more about execution length than about susceptibility. +# +# Each segment is P[outcome] for a uniformly random single-bit flip in the +# traced fault space, so a bar's height is P[anything goes wrong]. +# +# Replaces combined_fault_composition.r, which divided by the marker total and +# was therefore this chart with every bar rescaled to 100%. args <- commandArgs(trailingOnly = TRUE) if (length(args) < 2) { @@ -37,7 +47,6 @@ extract_info <- function(path) { # Load data all_data <- data.frame() -weight_data <- data.frame() for (arg in exp_args) { info <- extract_info(arg) @@ -56,37 +65,48 @@ for (arg in exp_args) { all_data <- bind_rows(all_data, df) } +# TODO: Finally put all the bullshit before this in some shared space + if (nrow(all_data) == 0) { stop("No data loaded") } -# Skip OK_MARKERs, sum GROUP1 + TRAP. -all_data <- all_data |> - filter(resulttype != "OK_MARKER") |> - mutate( - resulttype = ifelse(resulttype == "GROUP1_MARKER", "TRAP", resulttype) - ) |> - group_by(base_name, variant, benchmark, resulttype) |> - summarise(faults = sum(faults), .groups = "drop") +marker_order <- c( + "OK_MARKER", + "DETECTED_MARKER", + "GROUP1_MARKER", + "TRAP", + "TIMEOUT", + "WRITE_TEXTSEGMENT", + "ACCESS_OUTERSPACE", + "FAIL_MARKER" +) -out_suffix <- gsub("^resultsdata|\\.csv$", "", csv_suffix) - -# Calculate percentages -composition <- all_data |> +# Don't merge GROUP1_MARKER into TRAP for this chart +# Also keep the OK_MARKERs, so the "sum to 100%" is accurate +probability <- all_data |> group_by(base_name, variant, benchmark) |> - mutate(frac = faults / sum(faults)) |> + mutate(frac = faults / sum(faults, na.rm = TRUE)) |> ungroup() +# Don't print alphabetically +probability$resulttype <- factor(probability$resulttype, levels = marker_order) +probability$variant <- factor( + probability$variant, + levels = c("c", "aot", "interp") +) + plot <- ggplot( - composition |> filter(variant %in% c("aot", "interp")), + probability, aes(x = variant, y = frac, fill = resulttype) ) + geom_col() + facet_grid(benchmark ~ base_name) + + scale_y_continuous(labels = scales::percent) + labs( - title = "Marker Composition (AOT vs Interp)", + title = "Fault Probability per Fault Space", x = NULL, - y = "Percentage of Faults", + y = "Probability of Failure", fill = "Fault Type" ) + theme_minimal() + @@ -95,9 +115,6 @@ plot <- ggplot( axis.text.x = element_text(angle = 45, hjust = 1) ) -filename <- paste0( - "injections/fault_composition", - out_suffix, - ".svg" -) +out_suffix <- gsub("^resultsdata|\\.csv$", "", csv_suffix) +filename <- paste0("injections/fault_probability", out_suffix, ".svg") ggsave(filename, plot = plot, width = 13, height = 8) diff --git a/scripts/menu.pl b/scripts/menu.pl index d576614..ced09b6 100644 --- a/scripts/menu.pl +++ b/scripts/menu.pl @@ -653,15 +653,15 @@ my %handlers = ( # Sucks to put those here but I can't write them inside the R scripts my %chart_descriptions = ( - combined_fault_composition => - 'stacked fault type percentages/composition (resultsdata.csv).', - combined_fault_count_comparison => 'faults per benchmark, c/aot/interp side by side (resultsdata.csv).', combined_fault_count_correlation => 'correlation of raw aot vs. interp fault counts (resultsdata.csv).', + combined_fault_probability => +'marker probability per fault space (resultsdata.csv [+ traceweight.csv]).', + combined_fault_rates_per_instruction => 'faults normalised by instruction count (faults.csv + mnemonics.csv).', @@ -698,7 +698,7 @@ my %handlers = ( } @selected_charts; my @resultsdata_charts = grep { -/_result|_fault_count_comparison|_ratio_comparison|_fault_count_correlation|_fault_composition/ +/_result|_fault_count_comparison|_ratio_comparison|_fault_count_correlation|_fault_probability/ } @selected_charts; # Select if faults.csv or a filtered variant should be used