Add merged fault_probability chart

scripts/Modules/Mars.pm

@@ -166,6 +166,31 @@ sub db_do {
     $db->do(@cmd) or die "Database command failed (@cmd): " . $db->errstr;
 }
 
+# db_do only reports affected rows, this one also reads the value back
+sub db_selectrow {
+    my (@cmd) = @_;
+
+    my $db = db_connect();
+
+    my @row = $db->selectrow_array(@cmd);
+    die "Database query failed (@cmd): " . $db->errstr if $db->err;
+
+    return @row;
+}
+
+sub db_variant_name {
+    my ($db_name) = @_;
+
+    my ($count) =
+      db_selectrow("SELECT COUNT(DISTINCT variant) FROM `$db_name`.variant");
+    return undef unless defined $count && $count == 1;
+
+    my ($variant) =
+      db_selectrow("SELECT DISTINCT variant FROM `$db_name`.variant");
+
+    return $variant;
+}
+
 sub db_create {
     my ($db_name) = @_;
     say " - Creating database $db_name...";

scripts/Modules/TUI.pm

@@ -4,7 +4,9 @@ use strict;
 use warnings;
 use diagnostics;
 
+use Curses;
 use Curses::UI;
+use Curses::UI::Common;
 
 # Singleton
 my $_cui;
@@ -125,4 +127,60 @@ sub select_from_list {
     return @selection;
 }
 
+# Returns the entered string or undef when cancelled
+sub read_string {
+    my ( $title, $default ) = @_;
+
+    my $value;
+
+    my $cui = init_cui();
+    my $win = $cui->add( 'root', 'Window', );
+
+    my $entry;
+    $entry = $win->add(
+        'string_entry',
+        'TextEntry',
+        -title  => $title,
+        -border => 1,
+        -text   => ( defined $default ? $default : '' ),
+    );
+
+    $win->add(
+        'info', 'Label',
+        -y    => -1,
+        -text => "Enter = confirm, Esc = cancel",
+    );
+
+    # Set cursor to the end
+    $entry->pos( length $entry->get() );
+
+    # Enter would otherwise just leave the entry
+    $entry->clear_binding('loose-focus');
+
+    $entry->set_binding(
+        sub {
+            $value = $entry->get();
+            $value =~ s/^\s+|\s+$//g;
+            $cui->mainloopExit();
+        },
+        KEY_ENTER(),
+    );
+
+    $entry->set_binding(
+        sub {
+            $value = undef;
+            $cui->mainloopExit();
+        },
+        CUI_ESCAPE(),
+    );
+
+    $entry->focus();
+    $cui->mainloop();
+
+    $cui->leave_curses();
+    $cui->delete('root');
+
+    return $value;
+}
+
 1;

scripts/Queries/ResultsDataPruned.pm

@@ -0,0 +1,58 @@
+package ResultsDataPruned;
+
+use strict;
+use warnings;
+use diagnostics;
+
+use FindBin;
+use lib "$FindBin::Bin/../Modules";
+use Filters;
+
+use feature 'say';
+
+sub query {
+    my ( $experiment, $experiment_dir, @filter_config_names ) = @_;
+
+    my $extra =
+      Filters::build_filter_clause( $experiment_dir, @filter_config_names );
+
+    # This query basically asks: How many equivalence classes
+    # ended up in each possible outcome?
+    # In contrast to ResultsData.pm it doesn't do any "de-pruning", e.g.
+    # no expansion of the "data liveness interval length"
+    # SUM(t.time2 - t.time1 + 1).
+    #
+    # We run with --inject-single-bit, so the bits of one class can
+    # end up in different outcomes.
+    # This would be counted multiple times,
+    # so we'll have more results than pilots.
+    #
+    # The column is named "faults" so the charts still work with this.
+    my $querystring = "SELECT
+benchmark, resulttype, COUNT(DISTINCT g.pilot_id) AS faults
+FROM variant v
+JOIN trace t ON v.id = t.variant_id
+JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_physical_address = t.data_physical_address
+JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
+JOIN fsppilot p ON r.pilot_id = p.id
+WHERE v.variant = '$experiment'$extra
+GROUP BY v.id, resulttype
+ORDER BY variant, benchmark, resulttype;";
+
+    say $querystring;
+
+    return $querystring;
+}
+
+sub args { return "--batch --raw"; }
+
+sub filename {
+    my @filter_config_names = grep { defined && length } @_;
+    my $suffix =
+      @filter_config_names ? "_" . join( "+", sort @filter_config_names ) : "";
+    return "resultsdata_pruned${suffix}.csv";
+}
+
+sub postprocess { $_[0] =~ s/\t/,/g; }
+
+1;

scripts/charts/combined_fault_probability_merged.r

@@ -0,0 +1,117 @@
+library(ggplot2)
+library(dplyr)
+library(readr)
+library(stringr)
+library(tidyr)
+
+# Usage: Rscript combined_fault_probability_merged.r exp_abspath1 ... resultsdata_file
+
+args <- commandArgs(trailingOnly = TRUE)
+if (length(args) < 2) {
+  stop("Need at least 2 experiments")
+}
+
+csv_suffix <- if (grepl("\\.csv$", args[length(args)])) {
+  args[length(args)]
+} else {
+  "resultsdata.csv"
+}
+exp_args <- if (grepl("\\.csv$", args[length(args)])) {
+  args[-length(args)]
+} else {
+  args
+}
+
+extract_info <- function(path) {
+  dir_name <- basename(path)
+  match <- str_match(
+    dir_name,
+    "^\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_(.+?)_(c|aot|interp)_"
+  )
+  if (is.na(match[1, 1])) {
+    warning(paste("Could not parse:", dir_name))
+    return(NULL)
+  }
+
+  # "tacle-kernel-bsort" -> "bsort", otherwise it doesn't fit
+  base_name <- sub("^tacle-[^-]+-", "", match[1, 2])
+
+  list(base_name = base_name, variant = match[1, 3], path = path)
+}
+
+# Load data
+all_data <- data.frame()
+
+for (arg in exp_args) {
+  info <- extract_info(arg)
+  if (is.null(info)) {
+    next
+  }
+
+  csv_file <- file.path(info$path, csv_suffix)
+  if (!file.exists(csv_file)) {
+    warning(paste("Missing:", csv_file))
+    next
+  }
+  df <- read_csv(csv_file, col_types = cols())
+  df$base_name <- info$base_name
+  df$variant <- info$variant
+  all_data <- bind_rows(all_data, df)
+}
+
+# TODO: Finally put all the bullshit before this in some shared space
+
+if (nrow(all_data) == 0) {
+  stop("No data loaded")
+}
+
+marker_order <- c(
+  "OK_MARKER",
+  "DETECTED_MARKER",
+  "GROUP1_MARKER",
+  "TRAP",
+  "TIMEOUT",
+  "WRITE_TEXTSEGMENT",
+  "ACCESS_OUTERSPACE",
+  "FAIL_MARKER"
+)
+
+# Merge ip/mem/regs, then divide by the merged faultspace area
+# Don't merge GROUP1_MARKER into TRAP for this chart
+# Also keep the OK_MARKERs, so the "sum to 100%" is accurate
+probability <- all_data |>
+  group_by(base_name, variant, resulttype) |>
+  summarise(faults = sum(faults, na.rm = TRUE), .groups = "drop") |>
+  group_by(base_name, variant) |>
+  mutate(frac = faults / sum(faults, na.rm = TRUE)) |>
+  ungroup()
+
+# Don't print alphabetically
+probability$resulttype <- factor(probability$resulttype, levels = marker_order)
+probability$variant <- factor(
+  probability$variant,
+  levels = c("c", "aot", "interp")
+)
+
+plot <- ggplot(
+  probability,
+  aes(x = variant, y = frac, fill = resulttype)
+) +
+  geom_col() +
+  facet_wrap(~base_name) +
+  scale_y_continuous(labels = scales::percent) +
+  labs(
+    title = "Fault Probability (merged ip + mem + regs)",
+    x = NULL,
+    y = "Probability of Failure",
+    fill = "Fault Type"
+  ) +
+  theme_minimal() +
+  theme(
+    plot.title = element_text(size = 13, face = "bold"),
+    axis.text.x = element_text(angle = 45, hjust = 1)
+  )
+
+out_suffix <- gsub("^resultsdata|\\.csv$", "", csv_suffix)
+filename <- paste0("injections/fault_probability_merged", out_suffix, ".svg")
+ggsave(filename, plot = plot, width = 13, height = 8)

scripts/menu.pl

@@ -11,6 +11,7 @@ use Util;
 use Mars;
 use TUI;
 use Filters;
+use File::Temp;
 use Text::CSV_XS;
 
 use feature 'say';
@@ -29,6 +30,7 @@ my $local_archive_dir     = "$local_root/injections";
 my $local_charts_dir      = "$local_root/scripts/charts";
 my $local_ghidra_projects = "$local_root/ghidra";
 my $local_ghidra_scripts  = "$local_root/scripts/ghidra";
+my $local_dump_dir        = "$local_root/dumps";
 my $local_db_conf         = "$local_root/db.conf";
 
 my $resultbrowser_port = '5000';
@@ -95,6 +97,9 @@ my %handlers = (
             ResultsData =>
               'Faults summary per benchmark with resulttype (resultsdata.csv)',
 
+            ResultsDataPruned =>
+'Faults summary per benchmark with resulttype, using the pruned data without expansion (resultsdata_pruned.csv)',
+
             TargetClass =>
 'Faults by code type with data type (stack/heap/bss/...) and resulttype -> targetclass.csv',
 
@@ -124,6 +129,19 @@ my %handlers = (
 
         # Run queries on databases
         foreach my $db (@selected_dbs) {
+            my $experiment = $db =~ s/smchurla_//r;
+
+            my $variant_name = Mars::db_variant_name($db);
+            if ( !defined $variant_name ) {
+                say "Skipping $db: contains multiple variants";
+                next;
+            }
+            if ( $variant_name ne $experiment ) {
+                say
+"Skipping $db: the variant is '$variant_name' but the queries use '$experiment'.";
+                next;
+            }
+
             foreach my $query (@selected_queries) {
                 Util::rewrite_file( $local_db_conf, "database=",
                     "database=$db\n" );
@@ -133,7 +151,7 @@ my %handlers = (
                   ? " (" . join( "+", @filter_configs ) . ")"
                   : "";
                 say "Running $query$config_label on $db...";
-                Util::execute_query( $db =~ s/smchurla_//r,
+                Util::execute_query( $experiment,
                     $query, $local_db_conf, $local_archive_dir, 0,
                     @filter_configs );
             }
@@ -662,6 +680,9 @@ my %handlers = (
             combined_fault_probability =>
 'marker probability per fault space (resultsdata.csv [+ traceweight.csv]).',
 
+            combined_fault_probability_merged =>
+'marker probability per fault space, ip/mem/regs merged (resultsdata.csv).',
+
             combined_fault_rates_per_instruction =>
 'faults normalised by instruction count (faults.csv + mnemonics.csv).',
 
@@ -767,6 +788,209 @@ my %handlers = (
         }
     },
 
+    '31. Dump Database (Mars)' => sub {
+
+        # Dump databases from mars into $local_dump_dir
+        my @dbs = Mars::db_list();
+        my @dbs_with_notes;
+        foreach my $db (@dbs) {
+            my $info =
+              Util::read_experiment_info( $db =~ s/smchurla_//r =~ s/:/-/gr );
+
+            push @dbs_with_notes,
+              ( defined $info && length($info) > 0 )
+              ? sprintf( "%-60s (%s)", $db, $info )
+              : $db;
+        }
+
+        my @selected_dbs =
+          TUI::select_from_list( "Select Databases to Dump from Mars",
+            1, @dbs_with_notes );
+        die "No database selected" unless @selected_dbs;
+
+        @selected_dbs =
+          map { s/(.*?)\s+\(.+\)$/$1/r } @selected_dbs;
+
+        system( 'mkdir', '-p', $local_dump_dir );
+
+        # Mariadb complains about the database= line, so strip it out
+        my ( $dump_conf_handle, $dump_conf ) =
+          File::Temp::tempfile( 'db-dump-XXXXXX', TMPDIR => 1, UNLINK => 1 );
+        print $dump_conf_handle
+          grep { !/^\s*database\s*=/ } split /^/,
+          Util::read_file($local_db_conf);
+        close($dump_conf_handle) or die "failed to close $dump_conf: $!";
+
+        foreach my $db (@selected_dbs) {
+            my $dump_file = "$local_dump_dir/" . $db =~ s/:/-/gr . ".sql";
+
+            say "Dumping $db to $dump_file...";
+            Util::run( 'mariadb-dump', "--defaults-file=$dump_conf",
+                "--result-file=$dump_file", $db );
+        }
+    },
+
+    '32. Import Database (Mars)' => sub {
+
+        # Import databse dump on mars
+        die "No dumps in $local_dump_dir" unless -d $local_dump_dir;
+        my @dumps = grep { /\.sql$/ } Util::find_files($local_dump_dir);
+        my @selected_dumps =
+          TUI::select_from_list( "Select Dump to Import into Mars", 0, @dumps );
+        die "No dump selected" unless @selected_dumps;
+        my $dump_file = "$local_dump_dir/$selected_dumps[0]";
+
+        # Determine database name
+        my $db = TUI::read_string(
+            "Database Name to Import Into",
+            $selected_dumps[0] =~ s/\.sql$//r
+        );
+        die "No database name given"     unless defined $db && length $db;
+        die "Invalid database name: $db" unless $db =~ /^[\w.:-]+$/;
+        die "Database $db already exists on mars"
+          if grep { $_ eq $db } Mars::db_list();
+
+        # mariadb-dump doesn't create the database, so create it here
+        Mars::db_create($db);
+
+        # point db.conf at the database
+        Util::rewrite_file( $local_db_conf, "database=", "database=$db\n" );
+
+        say "Importing $dump_file into $db...";
+        Util::run( join ' ', 'mariadb', "--defaults-file=$local_db_conf",
+            '<', Util::shell_quote($dump_file) );
+    },
+
+    '33. Repair fspgroup Write Groups (Mars)' => sub {
+
+        # Retroactively fixes (hopefully) the missing equivalence class
+        # mappings (EC <-> Pilot) the BasicPruner misses (because it maps
+        # multiple classes to a single pilot, but uses the pilot as the
+        # primary key)
+        my @dbs = Mars::db_list();
+        my @dbs_with_notes;
+        foreach my $db (@dbs) {
+            my $info =
+              Util::read_experiment_info( $db =~ s/smchurla_//r =~ s/:/-/gr );
+
+            push @dbs_with_notes,
+              ( defined $info && length($info) > 0 )
+              ? sprintf( "%-60s (%s)", $db, $info )
+              : $db;
+        }
+
+        my @selected_dbs = TUI::select_from_list( "Select Databases to Repair",
+            1, @dbs_with_notes );
+        die "No database selected" unless @selected_dbs;
+
+        @selected_dbs =
+          map { s/(.*?)\s+\(.+\)$/$1/r } @selected_dbs;
+
+        # The PRIMARY KEY gets dropped by the repair and replaced with a KEY.
+        # To not run this shit on already repaired DBs, check for the PRIMARY
+        # index as a determinant
+        my $index_exists = sub {
+            my ($index) = @_;
+            my ($count) = Mars::db_selectrow(
+                "SELECT COUNT(*) FROM information_schema.STATISTICS
+                 WHERE TABLE_SCHEMA = DATABASE()
+                   AND TABLE_NAME = 'fspgroup'
+                   AND INDEX_NAME = '$index'"
+            );
+            return $count;
+        };
+
+        foreach my $db (@selected_dbs) {
+            say "Repairing $db...";
+
+            # Select the DB so index_exists works
+            Mars::db_do("use `$db`");
+
+            # Update the variant names, so the queries still work
+            my $expected_variant = $db =~ s/^smchurla_//r;
+            my ($variant_count) =
+              Mars::db_selectrow("SELECT COUNT(DISTINCT variant) FROM variant");
+            my ($current_variant) =
+              Mars::db_selectrow("SELECT DISTINCT variant FROM variant");
+
+            if ( !defined $variant_count || $variant_count != 1 ) {
+                say " - WARNING: $variant_count distinct variants!";
+            }
+            elsif ( $current_variant ne $expected_variant ) {
+                say " - Variant is '$current_variant' but queries look up"
+                  . " '$expected_variant' (from the database name), renaming...";
+                Mars::db_do("UPDATE variant SET variant = '$expected_variant'");
+            }
+
+            # Drop the primary key
+            if ( $index_exists->('PRIMARY') ) {
+                say " - Dropping PRIMARY KEY (pilot_id)...";
+                Mars::db_do("ALTER TABLE fspgroup DROP PRIMARY KEY");
+            }
+
+            # Add an index for the columns we actually join on (mostly)
+            unless ( $index_exists->('eq_class') ) {
+                say " - Adding eq_class index...";
+                Mars::db_do(
+                    "ALTER TABLE fspgroup
+ADD KEY eq_class (variant_id, instr2, data_physical_address)"
+                );
+            }
+
+            # I'm currently only using BasicPruner, but don't touch other
+            # pruning methods for now, for safety
+            my ($fspmethod_id) =
+              Mars::db_selectrow(
+                "SELECT id FROM fspmethod WHERE method = 'basic'");
+            die "No 'basic' fspmethod in $db" unless defined $fspmethod_id;
+
+            # The first row succeeded before the duplicate key error, remove it
+            Mars::db_do(
+                "DELETE g
+FROM fspgroup g
+JOIN fsppilot p ON p.id = g.pilot_id
+WHERE p.known_outcome = 1 AND p.fspmethod_id = $fspmethod_id"
+            );
+
+            my ($expected) =
+              Mars::db_selectrow(
+                "SELECT COUNT(*) FROM trace WHERE accesstype = 'W'");
+
+            # Query from BasicPruner.cc, now ran against the updated DB
+            say " - Inserting $expected write groups...";
+            Mars::db_do(
+                "INSERT INTO
+fspgroup (variant_id, instr2, data_physical_address, bit_pos, fspmethod_id, pilot_id)
+SELECT STRAIGHT_JOIN t.variant_id, t.instr2, t.data_physical_address, p.bit_pos, p.fspmethod_id, p.id
+FROM fsppilot p
+JOIN trace t
+ON t.variant_id = p.variant_id AND p.fspmethod_id = $fspmethod_id AND p.known_outcome = 1
+WHERE t.accesstype = 'W'"
+            );
+
+            # We expect one row per write group/EC.
+            # Otherwise no idea what's going on :O
+            my ($actual) = Mars::db_selectrow(
+                "SELECT COUNT(*)
+FROM fspgroup g
+JOIN fsppilot p ON p.id = g.pilot_id
+WHERE p.known_outcome = 1 AND p.fspmethod_id = $fspmethod_id"
+            );
+
+            if ( $actual == $expected ) {
+                say " - OK: $actual write groups match the write ECs in trace";
+            }
+            else {
+                say " - WARNING: inserted $actual, expected $expected."
+                  . " Check for duplicate known_outcome pilots:"
+                  . " SELECT variant_id, COUNT(*) FROM fsppilot"
+                  . " WHERE known_outcome = 1 GROUP BY variant_id;";
+            }
+        }
+
+        say "Queries have to be re-run.";
+    },
+
     '95. Delete Builds' => sub {
 
         # Delete old build files