From 4797f398d408a4a6cc3f81d18389b2a6ad09b18f Mon Sep 17 00:00:00 2001 From: Christoph Urlacher Date: Wed, 22 Jul 2026 00:16:38 +0200 Subject: [PATCH] Add ratio comparison plots --- scripts/charts/ratio_comparison.r | 94 ++++++++++++++ scripts/charts/ratio_comparison_merged.r | 102 +++++++++++++++ scripts/charts/ratio_comparison_merged_trap.r | 100 ++++++++++++++ scripts/charts/ratio_correlation.r | 122 ++++++++++++++++++ 4 files changed, 418 insertions(+) create mode 100644 scripts/charts/ratio_comparison.r create mode 100644 scripts/charts/ratio_comparison_merged.r create mode 100644 scripts/charts/ratio_comparison_merged_trap.r create mode 100644 scripts/charts/ratio_correlation.r diff --git a/scripts/charts/ratio_comparison.r b/scripts/charts/ratio_comparison.r new file mode 100644 index 0000000..20c68e9 --- /dev/null +++ b/scripts/charts/ratio_comparison.r @@ -0,0 +1,94 @@ +library(ggplot2) +library(dplyr) +library(readr) +library(stringr) + +# Usage: Rscript ratio_comparison.r exp_abspath1 exp_abspath2 ... [resultsdata_file] +# Plots every benchmark separately + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2) { + stop("Need at least 2 experiments") +} + +csv_suffix <- if (grepl("\\.csv$", args[length(args)])) { + args[length(args)] +} else { + "resultsdata.csv" +} +exp_args <- if (grepl("\\.csv$", args[length(args)])) { + args[-length(args)] +} else { + args +} + +extract_info <- function(path) { + dir_name <- basename(path) + match <- str_match( + dir_name, + "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_" + ) + if (is.na(match[1, 1])) { + warning(paste("Could not parse:", dir_name)) + return(NULL) + } + list(base_name = match[1, 2], variant = match[1, 3], path = path) +} + +all_data <- data.frame() + +for (arg in exp_args) { + info <- extract_info(arg) + if (is.null(info)) { + next + } + + csv_file <- file.path(info$path, csv_suffix) + if (!file.exists(csv_file)) { + warning(paste("Missing:", csv_file)) + next + } + + df <- read_csv(csv_file, col_types = cols()) + df$base_name <- info$base_name + df$variant <- info$variant + all_data <- bind_rows(all_data, df) +} + +if (nrow(all_data) == 0) { + stop("No data loaded") +} + +baseline <- all_data |> filter(variant == "c") +comparisons <- all_data |> filter(variant != "c") + +ratios <- comparisons |> + left_join( + baseline |> select(base_name, benchmark, resulttype, faults), + by = c("base_name", "benchmark", "resulttype"), + suffix = c("", "_baseline") + ) |> + filter(!is.na(faults_baseline), faults_baseline > 0) |> + mutate(ratio = faults / faults_baseline) + +if (nrow(ratios) == 0) { + stop("No ratios computed (missing baseline or zero values)") +} + +ratios$label <- paste0(ratios$benchmark, ".", ratios$resulttype) + +plot <- ggplot( + ratios, + aes(x = label, y = ratio, color = variant, group = variant) +) + + geom_point(size = 2) + + geom_line() + + facet_wrap(~base_name, scales = "free_x") + + scale_y_log10(name = "Ratio (to C)") + + scale_x_discrete(name = "Marker") + + labs(color = "Variant") + + theme_minimal() + + theme(axis.text.x = element_text(angle = 45, hjust = 1)) + +ggsave("injections/ratio_comparison.svg", plot = plot, width = 12, height = 8) +print("Saved ratio_comparison.svg") diff --git a/scripts/charts/ratio_comparison_merged.r b/scripts/charts/ratio_comparison_merged.r new file mode 100644 index 0000000..8799816 --- /dev/null +++ b/scripts/charts/ratio_comparison_merged.r @@ -0,0 +1,102 @@ +library(ggplot2) +library(dplyr) +library(readr) +library(stringr) + +# Usage: Rscript ratio_comparison_merged.r exp_abspath1 exp_abspath2 ... [resultsdata_file] +# Sums all benchmarks + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2) { + stop("Need at least 2 experiments") +} + +csv_suffix <- if (grepl("\\.csv$", args[length(args)])) { + args[length(args)] +} else { + "resultsdata.csv" +} +exp_args <- if (grepl("\\.csv$", args[length(args)])) { + args[-length(args)] +} else { + args +} + +extract_info <- function(path) { + dir_name <- basename(path) + match <- str_match( + dir_name, + "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_" + ) + if (is.na(match[1, 1])) { + warning(paste("Could not parse:", dir_name)) + return(NULL) + } + list(base_name = match[1, 2], variant = match[1, 3], path = path) +} + +all_data <- data.frame() + +for (arg in exp_args) { + info <- extract_info(arg) + if (is.null(info)) { + next + } + + csv_file <- file.path(info$path, csv_suffix) + if (!file.exists(csv_file)) { + warning(paste("Missing:", csv_file)) + next + } + + df <- read_csv(csv_file, col_types = cols()) + df$base_name <- info$base_name + df$variant <- info$variant + all_data <- bind_rows(all_data, df) +} + +if (nrow(all_data) == 0) { + stop("No data loaded") +} + +# Add all benchs together (per marker type) +merged_data <- all_data |> + group_by(base_name, variant, resulttype) |> + summarise(faults = sum(faults), .groups = "drop") + +baseline <- merged_data |> filter(variant == "c") +comparisons <- merged_data |> filter(variant != "c") + +ratios <- comparisons |> + left_join( + baseline |> select(base_name, resulttype, faults), + by = c("base_name", "resulttype"), + suffix = c("", "_baseline") + ) |> + filter(!is.na(faults_baseline), faults_baseline > 0) |> + mutate(ratio = faults / faults_baseline) + +if (nrow(ratios) == 0) { + stop("No ratios computed (missing baseline or zero values)") +} + +plot <- ggplot( + ratios, + aes(x = resulttype, y = ratio, color = variant, group = variant) +) + + geom_point(size = 2) + + geom_line() + + facet_wrap(~base_name) + + scale_y_log10(name = "Ratio (to C)") + + scale_x_discrete(name = "Marker") + + labs(color = "Variant") + + theme_minimal() + + theme(axis.text.x = element_text(angle = 45, hjust = 1)) + +ggsave( + "injections/ratio_comparison_merged.svg", + plot = plot, + width = 12, + height = 8 +) +print("Saved ratio_comparison_merged.svg") diff --git a/scripts/charts/ratio_comparison_merged_trap.r b/scripts/charts/ratio_comparison_merged_trap.r new file mode 100644 index 0000000..fee8ac8 --- /dev/null +++ b/scripts/charts/ratio_comparison_merged_trap.r @@ -0,0 +1,100 @@ +library(ggplot2) +library(dplyr) +library(readr) +library(stringr) + +# Usage: Rscript ratio_comparison_merged_trap.r exp_abspath1 exp_abspath2 ... [resultsdata_file] +# Sums all benchmarks, merges GROUP1_MARKER into TRAP + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2) { + stop("Need at least 2 experiments") +} + +csv_suffix <- if (grepl("\\.csv$", args[length(args)])) { + args[length(args)] +} else { + "resultsdata.csv" +} +exp_args <- if (grepl("\\.csv$", args[length(args)])) args[-length(args)] else args + +extract_info <- function(path) { + dir_name <- basename(path) + match <- str_match( + dir_name, + "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_" + ) + if (is.na(match[1, 1])) { + warning(paste("Could not parse:", dir_name)) + return(NULL) + } + list(base_name = match[1, 2], variant = match[1, 3], path = path) +} + +all_data <- data.frame() + +for (arg in exp_args) { + info <- extract_info(arg) + if (is.null(info)) { + next + } + + csv_file <- file.path(info$path, csv_suffix) + if (!file.exists(csv_file)) { + warning(paste("Missing:", csv_file)) + next + } + + df <- read_csv(csv_file, col_types = cols()) + df$base_name <- info$base_name + df$variant <- info$variant + all_data <- bind_rows(all_data, df) +} + +if (nrow(all_data) == 0) { + stop("No data loaded") +} + +all_data <- all_data |> + mutate(resulttype = ifelse(resulttype == "GROUP1_MARKER", "TRAP", resulttype)) + +merged_data <- all_data |> + group_by(base_name, variant, resulttype) |> + summarise(faults = sum(faults), .groups = "drop") + +baseline <- merged_data |> filter(variant == "c") +comparisons <- merged_data |> filter(variant != "c") + +ratios <- comparisons |> + left_join( + baseline |> select(base_name, resulttype, faults), + by = c("base_name", "resulttype"), + suffix = c("", "_baseline") + ) |> + filter(!is.na(faults_baseline), faults_baseline > 0) |> + mutate(ratio = faults / faults_baseline) + +if (nrow(ratios) == 0) { + stop("No ratios computed (missing baseline or zero values)") +} + +plot <- ggplot( + ratios, + aes(x = resulttype, y = ratio, color = variant, group = variant) +) + + geom_point(size = 2) + + geom_line() + + facet_wrap(~base_name) + + scale_y_log10(name = "Ratio (to C)") + + scale_x_discrete(name = "Marker") + + labs(color = "Variant") + + theme_minimal() + + theme(axis.text.x = element_text(angle = 45, hjust = 1)) + +ggsave( + "injections/ratio_comparison_merged_trap.svg", + plot = plot, + width = 12, + height = 8 +) +print("Saved ratio_comparison_merged_trap.svg") diff --git a/scripts/charts/ratio_correlation.r b/scripts/charts/ratio_correlation.r new file mode 100644 index 0000000..1eaea72 --- /dev/null +++ b/scripts/charts/ratio_correlation.r @@ -0,0 +1,122 @@ +library(ggplot2) +library(dplyr) +library(readr) +library(stringr) +library(tidyr) + +# Usage: Rscript ratio_correlation.r exp_abspath1 exp_abspath2 ... [resultsdata_file] +# Plots correlation between aot/c and interp/c ratios + +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 2) { + stop("Need at least 2 experiments") +} + +csv_suffix <- if (grepl("\\.csv$", args[length(args)])) { + args[length(args)] +} else { + "resultsdata.csv" +} +exp_args <- if (grepl("\\.csv$", args[length(args)])) { + args[-length(args)] +} else { + args +} + +extract_info <- function(path) { + dir_name <- basename(path) + match <- str_match( + dir_name, + "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_" + ) + if (is.na(match[1, 1])) { + warning(paste("Could not parse:", dir_name)) + return(NULL) + } + list(base_name = match[1, 2], variant = match[1, 3], path = path) +} + +all_data <- data.frame() + +for (arg in exp_args) { + info <- extract_info(arg) + if (is.null(info)) { + next + } + + csv_file <- file.path(info$path, csv_suffix) + if (!file.exists(csv_file)) { + warning(paste("Missing:", csv_file)) + next + } + + df <- read_csv(csv_file, col_types = cols()) + df$base_name <- info$base_name + df$variant <- info$variant + all_data <- bind_rows(all_data, df) +} + +if (nrow(all_data) == 0) { + stop("No data loaded") +} + +# Ignore OK_MARKER (only plot failures) and sum GROUP1_MARKER with TRAP +all_data <- all_data |> + filter(resulttype != "OK_MARKER") |> + mutate(resulttype = ifelse(resulttype == "GROUP1_MARKER", "TRAP", resulttype)) + +all_data <- all_data |> + group_by(base_name, variant, benchmark, resulttype) |> + summarise(faults = sum(faults), .groups = "drop") + +baseline <- all_data |> filter(variant == "c") +comparisons <- all_data |> filter(variant != "c") + +ratios <- comparisons |> + left_join( + baseline |> select(base_name, benchmark, resulttype, faults), + by = c("base_name", "benchmark", "resulttype"), + suffix = c("", "_baseline") + ) |> + filter(!is.na(faults_baseline), faults_baseline > 0) |> + mutate(ratio = faults / faults_baseline) + +if (nrow(ratios) == 0) { + stop("No ratios computed (missing baseline or zero values)") +} + +# Pivot to get aot and interp ratios side by side +ratio_wide <- ratios |> + select(base_name, benchmark, resulttype, variant, ratio) |> + pivot_wider(names_from = variant, values_from = ratio) |> + filter(!is.na(aot), !is.na(interp)) + +if (nrow(ratio_wide) == 0) { + stop("No paired aot/interp ratios found") +} + +# Compute correlation +cor_result <- cor(ratio_wide$aot, ratio_wide$interp, method = "pearson") +cat(sprintf("Pearson correlation: %.4f\n", cor_result)) + +# Create plot +plot <- ggplot( + ratio_wide, + aes(x = aot, y = interp, color = base_name, shape = resulttype) +) + + geom_point(size = 3, alpha = 0.7) + + scale_x_log10(name = "AOT / C Ratio") + + scale_y_log10(name = "Interpreter / C Ratio") + + labs( + title = sprintf("Ratio Correlation (r = %.4f)", cor_result), + color = "Target", + shape = "Marker" + ) + + theme_minimal() + + theme( + legend.position = "right", + plot.title = element_text(size = 14, face = "bold") + ) + +ggsave("injections/ratio_correlation.svg", plot = plot, width = 10, height = 8) +print("Saved ratio_correlation.svg")