Add ratio comparison plots

scripts/build.pl

@@ -202,6 +202,7 @@ local $ENV{WAMR_USE_LINEAR_POOL_IN_TEXT} =
 # ========================================================================================= #
 
 # NOTE: The runner will prefix "-Wf," to each flag
+# TODO: Exclude --wamr-exceptions from C builds automatically
 my %catch_flag_map = (
     "--catch-outer"     => "--catch-outerspace",
     "--catch-text"      => "--catch-write-textsegment",

scripts/charts/ratio_comparison.r

@@ -0,0 +1,94 @@
+library(ggplot2)
+library(dplyr)
+library(readr)
+library(stringr)
+
+# Usage: Rscript ratio_comparison.r exp_abspath1 exp_abspath2 ... [resultsdata_file]
+# Plots every benchmark separately
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2) {
+  stop("Need at least 2 experiments")
+}
+
+csv_suffix <- if (grepl("\\.csv$", args[length(args)])) {
+  args[length(args)]
+} else {
+  "resultsdata.csv"
+}
+exp_args <- if (grepl("\\.csv$", args[length(args)])) {
+  args[-length(args)]
+} else {
+  args
+}
+
+extract_info <- function(path) {
+  dir_name <- basename(path)
+  match <- str_match(
+    dir_name,
+    "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_"
+  )
+  if (is.na(match[1, 1])) {
+    warning(paste("Could not parse:", dir_name))
+    return(NULL)
+  }
+  list(base_name = match[1, 2], variant = match[1, 3], path = path)
+}
+
+all_data <- data.frame()
+
+for (arg in exp_args) {
+  info <- extract_info(arg)
+  if (is.null(info)) {
+    next
+  }
+
+  csv_file <- file.path(info$path, csv_suffix)
+  if (!file.exists(csv_file)) {
+    warning(paste("Missing:", csv_file))
+    next
+  }
+
+  df <- read_csv(csv_file, col_types = cols())
+  df$base_name <- info$base_name
+  df$variant <- info$variant
+  all_data <- bind_rows(all_data, df)
+}
+
+if (nrow(all_data) == 0) {
+  stop("No data loaded")
+}
+
+baseline <- all_data |> filter(variant == "c")
+comparisons <- all_data |> filter(variant != "c")
+
+ratios <- comparisons |>
+  left_join(
+    baseline |> select(base_name, benchmark, resulttype, faults),
+    by = c("base_name", "benchmark", "resulttype"),
+    suffix = c("", "_baseline")
+  ) |>
+  filter(!is.na(faults_baseline), faults_baseline > 0) |>
+  mutate(ratio = faults / faults_baseline)
+
+if (nrow(ratios) == 0) {
+  stop("No ratios computed (missing baseline or zero values)")
+}
+
+ratios$label <- paste0(ratios$benchmark, ".", ratios$resulttype)
+
+plot <- ggplot(
+  ratios,
+  aes(x = label, y = ratio, color = variant, group = variant)
+) +
+  geom_point(size = 2) +
+  geom_line() +
+  facet_wrap(~base_name, scales = "free_x") +
+  scale_y_log10(name = "Ratio (to C)") +
+  scale_x_discrete(name = "Marker") +
+  labs(color = "Variant") +
+  theme_minimal() +
+  theme(axis.text.x = element_text(angle = 45, hjust = 1))
+
+ggsave("injections/ratio_comparison.svg", plot = plot, width = 12, height = 8)
+print("Saved ratio_comparison.svg")

scripts/charts/ratio_comparison_merged.r

@@ -0,0 +1,102 @@
+library(ggplot2)
+library(dplyr)
+library(readr)
+library(stringr)
+
+# Usage: Rscript ratio_comparison_merged.r exp_abspath1 exp_abspath2 ... [resultsdata_file]
+# Sums all benchmarks
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2) {
+  stop("Need at least 2 experiments")
+}
+
+csv_suffix <- if (grepl("\\.csv$", args[length(args)])) {
+  args[length(args)]
+} else {
+  "resultsdata.csv"
+}
+exp_args <- if (grepl("\\.csv$", args[length(args)])) {
+  args[-length(args)]
+} else {
+  args
+}
+
+extract_info <- function(path) {
+  dir_name <- basename(path)
+  match <- str_match(
+    dir_name,
+    "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_"
+  )
+  if (is.na(match[1, 1])) {
+    warning(paste("Could not parse:", dir_name))
+    return(NULL)
+  }
+  list(base_name = match[1, 2], variant = match[1, 3], path = path)
+}
+
+all_data <- data.frame()
+
+for (arg in exp_args) {
+  info <- extract_info(arg)
+  if (is.null(info)) {
+    next
+  }
+
+  csv_file <- file.path(info$path, csv_suffix)
+  if (!file.exists(csv_file)) {
+    warning(paste("Missing:", csv_file))
+    next
+  }
+
+  df <- read_csv(csv_file, col_types = cols())
+  df$base_name <- info$base_name
+  df$variant <- info$variant
+  all_data <- bind_rows(all_data, df)
+}
+
+if (nrow(all_data) == 0) {
+  stop("No data loaded")
+}
+
+# Add all benchs together (per marker type)
+merged_data <- all_data |>
+  group_by(base_name, variant, resulttype) |>
+  summarise(faults = sum(faults), .groups = "drop")
+
+baseline <- merged_data |> filter(variant == "c")
+comparisons <- merged_data |> filter(variant != "c")
+
+ratios <- comparisons |>
+  left_join(
+    baseline |> select(base_name, resulttype, faults),
+    by = c("base_name", "resulttype"),
+    suffix = c("", "_baseline")
+  ) |>
+  filter(!is.na(faults_baseline), faults_baseline > 0) |>
+  mutate(ratio = faults / faults_baseline)
+
+if (nrow(ratios) == 0) {
+  stop("No ratios computed (missing baseline or zero values)")
+}
+
+plot <- ggplot(
+  ratios,
+  aes(x = resulttype, y = ratio, color = variant, group = variant)
+) +
+  geom_point(size = 2) +
+  geom_line() +
+  facet_wrap(~base_name) +
+  scale_y_log10(name = "Ratio (to C)") +
+  scale_x_discrete(name = "Marker") +
+  labs(color = "Variant") +
+  theme_minimal() +
+  theme(axis.text.x = element_text(angle = 45, hjust = 1))
+
+ggsave(
+  "injections/ratio_comparison_merged.svg",
+  plot = plot,
+  width = 12,
+  height = 8
+)
+print("Saved ratio_comparison_merged.svg")

scripts/charts/ratio_comparison_merged_trap.r

@@ -0,0 +1,100 @@
+library(ggplot2)
+library(dplyr)
+library(readr)
+library(stringr)
+
+# Usage: Rscript ratio_comparison_merged_trap.r exp_abspath1 exp_abspath2 ... [resultsdata_file]
+# Sums all benchmarks, merges GROUP1_MARKER into TRAP
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2) {
+  stop("Need at least 2 experiments")
+}
+
+csv_suffix <- if (grepl("\\.csv$", args[length(args)])) {
+  args[length(args)]
+} else {
+  "resultsdata.csv"
+}
+exp_args <- if (grepl("\\.csv$", args[length(args)])) args[-length(args)] else args
+
+extract_info <- function(path) {
+  dir_name <- basename(path)
+  match <- str_match(
+    dir_name,
+    "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_"
+  )
+  if (is.na(match[1, 1])) {
+    warning(paste("Could not parse:", dir_name))
+    return(NULL)
+  }
+  list(base_name = match[1, 2], variant = match[1, 3], path = path)
+}
+
+all_data <- data.frame()
+
+for (arg in exp_args) {
+  info <- extract_info(arg)
+  if (is.null(info)) {
+    next
+  }
+
+  csv_file <- file.path(info$path, csv_suffix)
+  if (!file.exists(csv_file)) {
+    warning(paste("Missing:", csv_file))
+    next
+  }
+
+  df <- read_csv(csv_file, col_types = cols())
+  df$base_name <- info$base_name
+  df$variant <- info$variant
+  all_data <- bind_rows(all_data, df)
+}
+
+if (nrow(all_data) == 0) {
+  stop("No data loaded")
+}
+
+all_data <- all_data |>
+  mutate(resulttype = ifelse(resulttype == "GROUP1_MARKER", "TRAP", resulttype))
+
+merged_data <- all_data |>
+  group_by(base_name, variant, resulttype) |>
+  summarise(faults = sum(faults), .groups = "drop")
+
+baseline <- merged_data |> filter(variant == "c")
+comparisons <- merged_data |> filter(variant != "c")
+
+ratios <- comparisons |>
+  left_join(
+    baseline |> select(base_name, resulttype, faults),
+    by = c("base_name", "resulttype"),
+    suffix = c("", "_baseline")
+  ) |>
+  filter(!is.na(faults_baseline), faults_baseline > 0) |>
+  mutate(ratio = faults / faults_baseline)
+
+if (nrow(ratios) == 0) {
+  stop("No ratios computed (missing baseline or zero values)")
+}
+
+plot <- ggplot(
+  ratios,
+  aes(x = resulttype, y = ratio, color = variant, group = variant)
+) +
+  geom_point(size = 2) +
+  geom_line() +
+  facet_wrap(~base_name) +
+  scale_y_log10(name = "Ratio (to C)") +
+  scale_x_discrete(name = "Marker") +
+  labs(color = "Variant") +
+  theme_minimal() +
+  theme(axis.text.x = element_text(angle = 45, hjust = 1))
+
+ggsave(
+  "injections/ratio_comparison_merged_trap.svg",
+  plot = plot,
+  width = 12,
+  height = 8
+)
+print("Saved ratio_comparison_merged_trap.svg")

scripts/charts/ratio_correlation.r

@@ -0,0 +1,122 @@
+library(ggplot2)
+library(dplyr)
+library(readr)
+library(stringr)
+library(tidyr)
+
+# Usage: Rscript ratio_correlation.r exp_abspath1 exp_abspath2 ... [resultsdata_file]
+# Plots correlation between aot/c and interp/c ratios
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2) {
+  stop("Need at least 2 experiments")
+}
+
+csv_suffix <- if (grepl("\\.csv$", args[length(args)])) {
+  args[length(args)]
+} else {
+  "resultsdata.csv"
+}
+exp_args <- if (grepl("\\.csv$", args[length(args)])) {
+  args[-length(args)]
+} else {
+  args
+}
+
+extract_info <- function(path) {
+  dir_name <- basename(path)
+  match <- str_match(
+    dir_name,
+    "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_"
+  )
+  if (is.na(match[1, 1])) {
+    warning(paste("Could not parse:", dir_name))
+    return(NULL)
+  }
+  list(base_name = match[1, 2], variant = match[1, 3], path = path)
+}
+
+all_data <- data.frame()
+
+for (arg in exp_args) {
+  info <- extract_info(arg)
+  if (is.null(info)) {
+    next
+  }
+
+  csv_file <- file.path(info$path, csv_suffix)
+  if (!file.exists(csv_file)) {
+    warning(paste("Missing:", csv_file))
+    next
+  }
+
+  df <- read_csv(csv_file, col_types = cols())
+  df$base_name <- info$base_name
+  df$variant <- info$variant
+  all_data <- bind_rows(all_data, df)
+}
+
+if (nrow(all_data) == 0) {
+  stop("No data loaded")
+}
+
+# Ignore OK_MARKER (only plot failures) and sum GROUP1_MARKER with TRAP
+all_data <- all_data |>
+  filter(resulttype != "OK_MARKER") |>
+  mutate(resulttype = ifelse(resulttype == "GROUP1_MARKER", "TRAP", resulttype))
+
+all_data <- all_data |>
+  group_by(base_name, variant, benchmark, resulttype) |>
+  summarise(faults = sum(faults), .groups = "drop")
+
+baseline <- all_data |> filter(variant == "c")
+comparisons <- all_data |> filter(variant != "c")
+
+ratios <- comparisons |>
+  left_join(
+    baseline |> select(base_name, benchmark, resulttype, faults),
+    by = c("base_name", "benchmark", "resulttype"),
+    suffix = c("", "_baseline")
+  ) |>
+  filter(!is.na(faults_baseline), faults_baseline > 0) |>
+  mutate(ratio = faults / faults_baseline)
+
+if (nrow(ratios) == 0) {
+  stop("No ratios computed (missing baseline or zero values)")
+}
+
+# Pivot to get aot and interp ratios side by side
+ratio_wide <- ratios |>
+  select(base_name, benchmark, resulttype, variant, ratio) |>
+  pivot_wider(names_from = variant, values_from = ratio) |>
+  filter(!is.na(aot), !is.na(interp))
+
+if (nrow(ratio_wide) == 0) {
+  stop("No paired aot/interp ratios found")
+}
+
+# Compute correlation
+cor_result <- cor(ratio_wide$aot, ratio_wide$interp, method = "pearson")
+cat(sprintf("Pearson correlation: %.4f\n", cor_result))
+
+# Create plot
+plot <- ggplot(
+  ratio_wide,
+  aes(x = aot, y = interp, color = base_name, shape = resulttype)
+) +
+  geom_point(size = 3, alpha = 0.7) +
+  scale_x_log10(name = "AOT / C Ratio") +
+  scale_y_log10(name = "Interpreter / C Ratio") +
+  labs(
+    title = sprintf("Ratio Correlation (r = %.4f)", cor_result),
+    color = "Target",
+    shape = "Marker"
+  ) +
+  theme_minimal() +
+  theme(
+    legend.position = "right",
+    plot.title = element_text(size = 14, face = "bold")
+  )
+
+ggsave("injections/ratio_correlation.svg", plot = plot, width = 10, height = 8)
+print("Saved ratio_correlation.svg")

scripts/menu.pl

@@ -308,6 +308,94 @@ my %handlers = (
             0, @entries );
     },
 
+    '11b. Compare to Baseline' => sub {
+
+        my $baseline = Util::select_experiment(0);
+
+        my @selected_experiments = Util::select_experiment(1);
+
+       # TODO: Fails silently if not every selected experiment has this datafile
+        my $resultsdata_csv =
+          Util::pick_data_file( "$local_archive_dir/$baseline", "resultsdata" );
+
+        my %all_results;
+        foreach my $experiment ( $baseline, @selected_experiments ) {
+
+            my $data = Text::CSV_XS::csv(
+                in      => "$local_archive_dir/$experiment/$resultsdata_csv",
+                headers => 'auto'
+            );
+
+            foreach my $row (@$data) {
+                $all_results{$experiment}{ $row->{benchmark} }
+                  { $row->{resulttype} } = $row->{faults};
+            }
+        }
+
+        my @benchs  = ( 'ip', 'mem', 'regs' );
+        my @markers = (
+            'OK_MARKER',         'FAIL_MARKER',
+            'DETECTED_MARKER',   'TIMEOUT',
+            'TRAP',              'WRITE_TEXTSEGMENT',
+            'ACCESS_OUTERSPACE', 'GROUP1_MARKER'
+        );
+
+        my $heading = sprintf( "%5s %20s %50s  ", "BENCH", "TYPE", $baseline );
+        my $subheading = sprintf( "%5s %20s %50s  ",
+            "", "", Util::read_experiment_info($baseline) );
+        foreach my $experiment (@selected_experiments) {
+            $heading .= sprintf( "%50s  ", $experiment );
+            $subheading .=
+              sprintf( "%50s  ", Util::read_experiment_info($experiment) );
+        }
+
+        my @entries = ( $heading, $subheading, "" );
+        foreach my $benchmark (@benchs) {
+            foreach my $marker (@markers) {
+                my $entry = sprintf( "%5s %20s ", $benchmark, $marker );
+
+                if ( exists $all_results{$baseline}{$benchmark}{$marker} ) {
+                    $entry .= sprintf(
+                        "%50s  ",
+                        Util::format_number_sep(
+                            $all_results{$baseline}{$benchmark}{$marker}
+                        )
+                    );
+                }
+                else {
+                    $entry .= sprintf( "%50s  ", "" );
+                }
+
+                foreach my $experiment (@selected_experiments) {
+                    if (    exists $all_results{$baseline}{$benchmark}{$marker}
+                        and
+                        exists $all_results{$experiment}{$benchmark}{$marker}
+                        and $all_results{$baseline}{$benchmark}{$marker} != 0 )
+                    {
+                        my $factor =
+                          $all_results{$experiment}{$benchmark}{$marker} /
+                          $all_results{$baseline}{$benchmark}{$marker};
+                        $entry .=
+                          sprintf( "%50s  ", sprintf( "%.2fx", $factor ) );
+                    }
+                    else {
+                        $entry .= sprintf( "%50s  ", "" );
+                    }
+                }
+
+                push @entries, $entry;
+            }
+            push @entries, "";
+        }
+
+        TUI::select_from_list(
+            "Baseline: $baseline — Comparing "
+              . scalar(@selected_experiments)
+              . " Experiments",
+            0, @entries
+        );
+    },
+
     '12. Open Experiment in BinaryNinja' => sub {
         my @selected_experiments = Util::select_experiment(1);
         my @paths =
@@ -487,7 +575,8 @@ my %handlers = (
         # Need to know which chart uses which datafile
         my @faults_charts = grep { /heatmap|scatter|sankey/ } @selected_charts;
         my @resultsdata_charts =
-          grep { /result$|combined_comparison/ } @selected_charts;
+          grep { /result$|combined_comparison|ratio_comparison|ratio_correlation/ }
+          @selected_charts;
 
         my $faults_csv;
         my $resultsdata_csv;
@@ -519,7 +608,8 @@ my %handlers = (
             }
         }
 
-        my @combined_charts   = grep { /combined/ } @selected_charts;
+        my @combined_charts =
+          grep { /combined|ratio_comparison|ratio_correlation/ } @selected_charts;
         my $print_experiments = join " ", @selected_experiments;
         my @path_experiments =
           map { "$local_archive_dir/$_" } @selected_experiments;
@@ -531,7 +621,7 @@ my %handlers = (
               if defined $faults_csv && $chart =~ /heatmap|scatter|sankey/;
             push @r_args, $resultsdata_csv
               if defined $resultsdata_csv
-              && $chart =~ /result$|combined_comparison/;
+              && $chart =~ /result$|combined_comparison|ratio_comparison|ratio_correlation/;
             system(@r_args);
         }
     },

targets/wasm-module/tacle-kernel-countnegative.cpp

@@ -0,0 +1,25 @@
+#include "../lib.h"
+
+extern "C" {
+void __pragma_loopbound(unsigned, unsigned) {}
+
+void countnegative_init(void);
+void countnegative_main(void);
+int countnegative_return(void);
+
+EXPORT("wasm_module") int wasm_module(void) {
+  countnegative_init();
+
+  fail_start_trace();
+  countnegative_main();
+  fail_stop_trace();
+
+  int ret = countnegative_return();
+  if (ret == 0) {
+    fail_marker_positive();
+  } else {
+    fail_marker_negative();
+  }
+  return ret;
+}
+}

targets/wasm-module/tacle-kernel-fft.cpp

@@ -0,0 +1,25 @@
+#include "../lib.h"
+
+extern "C" {
+void __pragma_loopbound(unsigned, unsigned) {}
+
+void fft_init(void);
+void fft_main(void);
+int fft_return(void);
+
+EXPORT("wasm_module") int wasm_module(void) {
+  fft_init();
+
+  fail_start_trace();
+  fft_main();
+  fail_stop_trace();
+
+  int ret = fft_return();
+  if (ret == 0) {
+    fail_marker_positive();
+  } else {
+    fail_marker_negative();
+  }
+  return ret;
+}
+}

targets/wasm-module/tacle-kernel-md5.cpp

@@ -0,0 +1,25 @@
+#include "../lib.h"
+
+extern "C" {
+void __pragma_loopbound(unsigned, unsigned) {}
+
+void md5_init(void);
+void md5_main(void);
+int md5_return(void);
+
+EXPORT("wasm_module") int wasm_module(void) {
+  md5_init();
+
+  fail_start_trace();
+  md5_main();
+  fail_stop_trace();
+
+  int ret = md5_return();
+  if (ret == 0) {
+    fail_marker_positive();
+  } else {
+    fail_marker_negative();
+  }
+  return ret;
+}
+}

targets/wasm-module/tacle-kernel-quicksort.cpp

@@ -0,0 +1,25 @@
+#include "../lib.h"
+
+extern "C" {
+void __pragma_loopbound(unsigned, unsigned) {}
+
+void quicksort_init(void);
+void quicksort_main(void);
+int quicksort_return(void);
+
+EXPORT("wasm_module") int wasm_module(void) {
+  quicksort_init();
+
+  fail_start_trace();
+  quicksort_main();
+  fail_stop_trace();
+
+  int ret = (quicksort_return() - 1527923179 != 0);
+  if (ret == 0) {
+    fail_marker_positive();
+  } else {
+    fail_marker_negative();
+  }
+  return ret;
+}
+}

targets/wasm-module/tacle-kernel-recursion.cpp

@@ -0,0 +1,25 @@
+#include "../lib.h"
+
+extern "C" {
+void __pragma_loopbound(unsigned, unsigned) {}
+
+void recursion_init(void);
+void recursion_main(void);
+int recursion_return(void);
+
+EXPORT("wasm_module") int wasm_module(void) {
+  recursion_init();
+
+  fail_start_trace();
+  recursion_main();
+  fail_stop_trace();
+
+  int ret = recursion_return();
+  if (ret == 0) {
+    fail_marker_positive();
+  } else {
+    fail_marker_negative();
+  }
+  return ret;
+}
+}

targets/wasm-module/tacle-kernel-sha.cpp

@@ -0,0 +1,25 @@
+#include "../lib.h"
+
+extern "C" {
+void __pragma_loopbound(unsigned, unsigned) {}
+
+void sha_init(void);
+void sha_main(void);
+int sha_return(void);
+
+EXPORT("wasm_module") int wasm_module(void) {
+  sha_init();
+
+  fail_start_trace();
+  sha_main();
+  fail_stop_trace();
+
+  int ret = sha_return();
+  if (ret == 0) {
+    fail_marker_positive();
+  } else {
+    fail_marker_negative();
+  }
+  return ret;
+}
+}