separate client/server logs + move logs to experiment dir

charts/combined_sankey.r

@@ -0,0 +1,80 @@
+library(ggplot2)
+library(ggalluvial)
+
+# Usage: Rscript combined_comparion.r exp_abspath1 exp_abspath2 ...
+
+args <- commandArgs(trailingOnly = TRUE)
+argc <- length(args)
+
+if (argc != 2) {
+  print("Expecting two input files")
+  stop()
+}
+
+for (experiment in args) {
+  datafile <- paste(experiment, "/faults.csv", sep = "")
+  if (!file.exists(datafile)) {
+    print(paste("Input file", datafile, "is missing"))
+    stop()
+  }
+}
+
+resulttype_labels <- c(
+  OK_MARKER = "OK",
+  FAIL_MARKER = "FAIL",
+  DETECTED_MARKER = "DETECTED",
+  TIMEOUT = "TIMEOUT",
+  TRAP = "TRAP",
+  ACCESS_OUTERSPACE = "OUTERSPC",
+  WRITE_TEXTSEGMENT = "WRITETXT"
+)
+
+# Read data
+datafile1 <- paste(args[1], "/faults.csv", sep = "")
+data1 <- readr::read_csv(datafile1)
+data1$fault_address <- strtoi(data1$fault_address)
+data1$resulttype <- resulttype_labels[data1$resulttype]
+# tibble::glimpse(data1)
+
+datafile2 <- paste(args[2], "/faults.csv", sep = "")
+data2 <- readr::read_csv(datafile2)
+data2$fault_address <- strtoi(data2$fault_address)
+data2$resulttype <- resulttype_labels[data2$resulttype]
+# tibble::glimpse(data2)
+
+# https://corybrunson.github.io/ggalluvial/
+joined <- merge(
+  data1[, c("fault_address", "resulttype", "faults")],
+  data2[, c("fault_address", "resulttype", "faults")],
+  by = "fault_address",
+  suffixes = c("_bench1", "_bench2")
+)
+
+streams <- aggregate(
+  faults_bench2 ~ resulttype_bench1 + resulttype_bench2,
+  data = joined,
+  sum
+)
+names(streams) <- c("bench1", "bench2", "faults")
+
+plot <- ggplot(
+  data = streams,
+  aes(axis1 = bench1, axis2 = bench2, y = faults)
+) +
+  scale_x_discrete(
+    # TODO: Name the benchmarks
+    # limits = c(args[1], args[2])
+    limits = c("Bench A", "Bench B")
+  ) +
+  labs(x = "Benchmark", y = "Faults") +
+  geom_alluvium(aes(fill = bench1)) +
+  geom_stratum() +
+  geom_text(stat = "stratum", aes(label = after_stat(stratum))) +
+  theme_minimal() +
+  theme(legend.position = "none")
+
+# TODO: Name the file according to the benchmarks
+ggsave(
+  paste(args[2], "/../sankey.svg", sep = ""),
+  plot = plot,
+)

flake.nix

@@ -321,10 +321,12 @@ rec {
           src = pkgs.fetchFromGitea {
             domain = "gitea.local.chriphost.de";
             owner = "christoph";
+            # domain = "git.cs.tu-dortmund.de";
+            # owner = "christoph.urlacher";
             repo = "wamr";
 
             # With mmap_space in .text.wamr_aot
-            rev = "cda2008deb85511089b04b0ac736ad4da2d07e58";
+            rev = "cda2009deb85511089b04b0ac736ad4da2d07e58";
             hash = "sha256-CN6xTiwzF4Jbrpf21TF5c/C03Xb3urwkibRuIXjoU/w=";
 
             # Without mmap_space in .text.wamr_aot
@@ -451,17 +453,28 @@ rec {
         buildInputs = with pkgs; [
           # FAIL runtime dependencies
           python # bochs-experiment-runner.py, resultbrowser.py
-          alsa-lib # libasound.so.2
-          boost_pkgs.boost174 # libboost_coroutine.so.1.74.0, libboost_regex.so.1.74.0, libboost_thread.so.1.74.0
-          capstone_4 # libcapstone.so.4
+
+          # For old VSS FAIL
+          # alsa-lib # libasound.so.2
+          # boost_pkgs.boost174 # libboost_coroutine.so.1.74.0, libboost_regex.so.1.74.0, libboost_thread.so.1.74.0
+          # capstone_4 # libcapstone.so.4
+          # libdwarf_pkgs.libdwarf # libdwarf.so.1
+          # elfutils # libelf.so.1
+          # mariadb # libmariadb.so.3
+          # libpcl # libpcl.so.1
+          # protobuf_21 # libprotobuf.so.32
+          # SDL # libSDL-1.2.so.0
+          # libx11 # libX11.so.6
+          # libxrandr # libXrandr.so.2
+          # libz # libz.so.1
+
+          # For current FAIL
+          boost183
+          capstone # libcapstone.so.5
           libdwarf_pkgs.libdwarf # libdwarf.so.1
           elfutils # libelf.so.1
           mariadb # libmariadb.so.3
-          libpcl # libpcl.so.1
           protobuf_21 # libprotobuf.so.32
-          SDL # libSDL-1.2.so.0
-          libx11 # libX11.so.6
-          libxrandr # libXrandr.so.2
           libz # libz.so.1
         ];
 

scripts/Queries/Faults.pm

@@ -12,7 +12,7 @@ benchmark, resulttype, SUM(t.time2 - t.time1 + 1) AS faults,
 CONCAT('0x', HEX(p.injection_instr_absolute)) AS fault_address
 FROM variant v
 JOIN trace t ON v.id = t.variant_id
-JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address
+JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_physical_address = t.data_physical_address
 JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
 JOIN fsppilot p ON r.pilot_id = p.id
 WHERE v.variant = '$experiment'

scripts/Queries/Results.pm

@@ -11,7 +11,7 @@ sub query {
 benchmark, resulttype, sum(t.time2 - t.time1 + 1) AS faults
 FROM variant v
 JOIN trace t ON v.id = t.variant_id
-JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address
+JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_physical_address = t.data_physical_address
 JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
 JOIN fsppilot p ON r.pilot_id = p.id
 WHERE v.variant = '$experiment'

scripts/Queries/ResultsData.pm

@@ -11,7 +11,7 @@ sub query {
 benchmark, resulttype, sum(t.time2 - t.time1 + 1) AS faults
 FROM variant v
 JOIN trace t ON v.id = t.variant_id
-JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address
+JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_physical_address = t.data_physical_address
 JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
 JOIN fsppilot p ON r.pilot_id = p.id
 WHERE v.variant = '$experiment'

scripts/Util.pm

@@ -252,7 +252,7 @@ sub select_experiment {
 
         push @exp_with_notes,
           ( defined $info && length($info) > 0 )
-          ? sprintf( "%-50s (Note: %s)", $exp, $info )
+          ? sprintf( "%-50s (%s)", $exp, $info )
           : $exp;
     }
 
@@ -260,7 +260,7 @@ sub select_experiment {
       TUI::select_from_list( "Select Experiment", $multi, @exp_with_notes );
     die "No experiment selected" unless @selected_experiments;
 
-    map { s/(.*?)\s+\(Note:.+\)$/$1/ } @selected_experiments;
+    map { s/(.*?)\s+\(.+\)$/$1/ } @selected_experiments;
 
     return $multi == 1 ? @selected_experiments : $selected_experiments[0];
 }

scripts/menu.pl

@@ -201,59 +201,7 @@ my %handlers = (
     '10. Open Experiment In Explorer' =>
       sub { do "$local_scripts_dir/explore.pl" },
 
-    '11. Open Experiment in Ghidra' => sub {
-
-        my @projects =
-          map { s/\.gpr//r } Util::find_files($local_ghidra_projects);
-        my @selected_projects =
-          TUI::select_from_list( "Select Project to Open in Ghidra",
-            0, @projects );
-        die "No project selected" unless @selected_projects;
-        my $project = $selected_projects[0];
-        system(
-            join " ",
-            (
-                "_JAVA_AWT_WM_NONREPARENTING=1", "ghidra",
-                "$local_ghidra_projects/$project.gpr",
-            )
-        );
-    },
-
-    '12. Open Experiment in Binsider' => sub {
-        my $selected_experiment = Util::select_experiment(0);
-
-        system( 'binsider',
-            "$local_archive_dir/$selected_experiment/system.elf" );
-    },
-
-    '13. Open Experiment in ResultBrowser' => sub {
-
-        my @db_names = Mars::db_list();
-        my @selected_dbs =
-          TUI::select_from_list( "Select Database for ResultBrowser",
-            0, @db_names );
-        die "No database selected" unless @selected_dbs;
-        my $selected_db = $selected_dbs[0];
-
-        Util::rewrite_file( $local_db_conf, "database=",
-            "database=$selected_db\n" );
-
-        system( $resultbrowser, '-c', $local_db_conf, '--host=0.0.0.0',
-            "--port=$resultbrowser_port" );
-    },
-
-    '14. Open Database in LazySQL' => sub {
-        my $experiment =
-          Util::select_experiment(0) =~ s/T(\d\d)-(\d\d)-(\d\d)/T$1:$2:$3/r;
-        my $ssh         = Mars::ssh_connect();
-        my $db_password = Mars::read_db_password_file();
-
-        system( 'lazysql', '-read-only',
-"mariadb://$db_user:$db_password\@$db_host:$db_port/${db_user}_$experiment"
-        );
-    },
-
-    '15. Compare Experiment Results' => sub {
+    '11. Compare Experiment Results' => sub {
 
         my @selected_experiments = Util::select_experiment(1);
 
@@ -319,48 +267,125 @@ my %handlers = (
             0, @entries );
     },
 
-    '16. Plot Results' => sub {
+    '12. Open Experiment in BinaryNinja' => sub {
+        my @selected_experiments = Util::select_experiment(1);
+        my @paths =
+          map { "$local_archive_dir/$_/system.elf" } @selected_experiments;
 
-        # Generate R ggplot2 charts
-        my @experiments = Util::find_subdirs($local_archive_dir);
-        my @selected_experiments =
-          TUI::select_from_list( "Select Experiments to Plot", 1,
-            @experiments );
-        die "No experiment selected" unless @selected_experiments;
-
-        my @charts = map { s/\.r//r } Util::find_files($local_charts_dir);
-        my @selected_charts =
-          TUI::select_from_list( "Select Plots to Generate", 1, @charts );
-        die "No plot selected" unless @selected_charts;
-
-        my @single_charts = grep { /single/ } @selected_charts;
-        foreach my $experiment (@selected_experiments) {
-            foreach my $chart (@single_charts) {
-                say " - Generating plot $chart for $experiment...";
-                system(
-                    'Rscript',
-                    "$local_charts_dir/$chart.r",
-                    "$local_archive_dir/$experiment"
-                );
-            }
-        }
-
-        my @combined_charts   = grep { /combined/ } @selected_charts;
-        my $print_experiments = join " ", @selected_experiments;
-        my @path_experiments =
-          map { "$local_archive_dir/$_" } @selected_experiments;
-        foreach my $chart (@combined_charts) {
-            say " - Generating plot $chart for ($print_experiments)...";
-            system( 'Rscript', "$local_charts_dir/$chart.r",
-                @path_experiments );
-        }
+        system( 'binaryninja', @paths );
     },
 
-    '17. Run Build in GDB' => sub {
+    '13. Open Experiment in Binsider' => sub {
+        my $selected_experiment = Util::select_experiment(0);
+
+        system( 'binsider',
+            "$local_archive_dir/$selected_experiment/system.elf" );
+    },
+
+    '14. Open Experiment in Ghidra' => sub {
+
+        my @projects =
+          map { s/\.gpr//r } Util::find_files($local_ghidra_projects);
+        my @selected_projects =
+          TUI::select_from_list( "Select Project to Open in Ghidra",
+            0, @projects );
+        die "No project selected" unless @selected_projects;
+        my $project = $selected_projects[0];
+        system(
+            join " ",
+            (
+                "_JAVA_AWT_WM_NONREPARENTING=1", "ghidra",
+                "$local_ghidra_projects/$project.gpr",
+            )
+        );
+    },
+
+    '15. Run Objdump on Experiment' => sub {
+        my $selected_experiment = Util::select_experiment(0);
+
+        system(
+"objdump $local_archive_dir/$selected_experiment/system.elf -D -M intel -S | bat --color never"
+        );
+    },
+
+    '16. Run Wasm-Objdump on Experiment' => sub {
+        my $selected_experiment = Util::select_experiment(0);
+
+        system(
+"wasm-objdump -d $local_archive_dir/$selected_experiment/wasm_module.wasm | bat --color never"
+        );
+    },
+
+    '17. Run Radare2 on Experiment' => sub {
+        my $selected_experiment = Util::select_experiment(0);
+
+        say "Radare help:";
+        say "s <address>      - Seek to address";
+        say "pd <n>           - Disassemble n instructions";
+        say "pdf              - Disassemble current function";
+        say "pdf @ <function> - Disassemble function";
+        say "pdr              - Disassemble recursively";
+        say "V                - Switch view";
+        say "p                - Switch print mode";
+        say "P                - Switch layout";
+        system(
+            'radare2', '-AA',
+            '-c',      '"-s dbg.os_main"',
+            '-e',      'scr.color=3',
+            '-e',      'scr.scrollbar=0',
+            '-e',      'scr.responsive=true',
+            '-e',      'scr.interactive=true',
+            '-e',      'scr.utf8=true',
+            '-e',      'scr.utf8.curvy=true',
+            '-e',      'asm.syntax=intel',
+            '-e',      'asm.lines=false',
+            '-e',      'asm.xrefs=true',
+            '-e',      'asm.flags=true',
+            '-e',      'asm.comments=true',
+            '-e',      'asm.functions=true',
+            '-e',      'asm.var=true',
+            '-e',      'asm.cmt.right=true',
+            '-e',      'asm.dwarf=true',
+            '-e',      'asm.pseudo=false',
+            '-e',      'asm.describe=false',
+            '-e',      'bin.relocs.apply=true',
+            "$local_archive_dir/$selected_experiment/system.elf",
+
+        );
+    },
+
+    '18. Open Database in ResultBrowser (Mars)' => sub {
+
+        my @db_names = Mars::db_list();
+        my @selected_dbs =
+          TUI::select_from_list( "Select Database for ResultBrowser",
+            0, @db_names );
+        die "No database selected" unless @selected_dbs;
+        my $selected_db = $selected_dbs[0];
+
+        Util::rewrite_file( $local_db_conf, "database=",
+            "database=$selected_db\n" );
+
+        system( $resultbrowser, '-c', $local_db_conf, '--host=0.0.0.0',
+            "--port=$resultbrowser_port" );
+    },
+
+    '19. Open Database in LazySQL (Mars)' => sub {
+        my $experiment =
+          Util::select_experiment(0) =~ s/T(\d\d)-(\d\d)-(\d\d)/T$1:$2:$3/r;
+        my $ssh         = Mars::ssh_connect();
+        my $db_password = Mars::read_db_password_file();
+
+        system( 'lazysql', '-read-only',
+"mariadb://$db_user:$db_password\@$db_host:$db_port/${db_user}_$experiment"
+        );
+    },
+
+    '20. Run Build in GDB' => sub {
         my @builds = grep { /linux/ } Util::find_subdirs($local_builds_dir);
         my @selected_builds =
           TUI::select_from_list( "Select Build to Run in GDB", 0, @builds );
-        die "No experiment selected" unless @selected_builds;
+        die "No build selected" unless @selected_builds;
         my $selected_build = $selected_builds[0];
 
         my $build_dir  = "$local_builds_dir/$selected_build";
@@ -393,6 +418,39 @@ my %handlers = (
         );
     },
 
+    '30. Plot Results' => sub {
+
+        # Generate R ggplot2 charts
+        my @selected_experiments = Util::select_experiment(1);
+
+        my @charts = map { s/\.r//r } Util::find_files($local_charts_dir);
+        my @selected_charts =
+          TUI::select_from_list( "Select Plots to Generate", 1, @charts );
+        die "No plot selected" unless @selected_charts;
+
+        my @single_charts = grep { /single/ } @selected_charts;
+        foreach my $experiment (@selected_experiments) {
+            foreach my $chart (@single_charts) {
+                say " - Generating plot $chart for $experiment...";
+                system(
+                    'Rscript',
+                    "$local_charts_dir/$chart.r",
+                    "$local_archive_dir/$experiment"
+                );
+            }
+        }
+
+        my @combined_charts   = grep { /combined/ } @selected_charts;
+        my $print_experiments = join " ", @selected_experiments;
+        my @path_experiments =
+          map { "$local_archive_dir/$_" } @selected_experiments;
+        foreach my $chart (@combined_charts) {
+            say " - Generating plot $chart for ($print_experiments)...";
+            system( 'Rscript', "$local_charts_dir/$chart.r",
+                @path_experiments );
+        }
+    },
+
     '95. Delete Builds' => sub {
 
         # Delete old build files

scripts/runner.pl

@@ -54,6 +54,9 @@ sub trace {
             "-Wf,--state-file=$remote_builds_dir/$experiment/state",
             "-Wf,--trace-file=$remote_builds_dir/$experiment/trace.pb",
             "-Wf,--elf-file=$remote_builds_dir/$experiment/system.elf"
+
+            # "-Wf,--full-trace",
+            # "-Wf,--check-bounds",
         )
     );
 }
@@ -133,7 +136,7 @@ sub import_trace {
             "-v $experiment",
             "-b ip",
             "--objdump objdump",
-            "--sources",
+			# "--sources",
         )
     );
     system(
@@ -147,7 +150,7 @@ sub import_trace {
             "-v $experiment",
             "-b mem",
             "--objdump objdump",
-            "--sources",
+			# "--sources",
         )
     );
     system(
@@ -161,7 +164,7 @@ sub import_trace {
             "-v $experiment",
             "-b regs",
             "--objdump objdump",
-            "--sources",
+			# "--sources",
         )
     );
 
@@ -183,6 +186,41 @@ sub inject {
 
     Util::notify("Injecting $experiment using $count cores...");
 
+    my $server_command = join " ", (
+        "$fail_server",
+        "--port $fail_server_port",
+        "--database-option-file $remote_db_conf",
+        "-v $experiment",
+        "-b %",
+        "--inject-single-bit",
+        "--inject-registers",
+        ">$remote_root/server.log"
+    );
+    say "Server command: $server_command";
+
+    my $client_command = join " ", (
+        "nice $bochs_runner",
+        "-V $fail_share/vgabios.bin",
+        "-b $fail_share/BIOS-bochs-latest",
+        "-f $fail_inject",
+        "-e $remote_builds_dir/$experiment/system.elf",
+        "-i $remote_builds_dir/$experiment/system.iso",
+        "-j $count",
+        "--",
+        "-Wf,--server-port=$fail_server_port",
+        "-Wf,--state-dir=$remote_builds_dir/$experiment/state",
+        "-Wf,--trap",
+        "-Wf,--catch-outerspace",
+        "-Wf,--catch-write-textsegment",
+        "-Wf,--timeout=500000",
+        "-Wf,--ok-marker=fail_marker_positive",
+        "-Wf,--fail-marker=fail_marker_negative",
+        "-Wf,--detected-marker=fail_marker_detected",
+        # ">/dev/null"
+        ">$remote_root/client.log"
+    );
+    say "Client command: $client_command";
+
     say "Forking...";
     my $pid = fork();
     die "fork failed: $!" unless defined $pid;
@@ -190,52 +228,16 @@ sub inject {
     if ( $pid == 0 ) {
 
         # child -> server
-
         say "Running server in child process...";
-        exec(
-            join " ",
-            (
-                "$fail_server",
-                "--port $fail_server_port",
-                "--database-option-file $remote_db_conf",
-                "-v $experiment",
-                "-b %",
-                "--inject-single-bit",
-                "--inject-registers"
-            )
-        ) == 0 or die "exec server failed: $!";
+        exec($server_command) == 0 or die "exec server failed: $!";
     }
 
     # parent -> client
-
     say "Waiting for server...";
     sleep(10);
 
     say "Running client with $count cores in parent process";
-
-    system(
-        join " ",
-        (
-            "nice $bochs_runner",
-            "-V $fail_share/vgabios.bin",
-            "-b $fail_share/BIOS-bochs-latest",
-            "-f $fail_inject",
-            "-e $remote_builds_dir/$experiment/system.elf",
-            "-i $remote_builds_dir/$experiment/system.iso",
-            "-j $count",
-            "--",
-            "-Wf,--server-port=$fail_server_port",
-            "-Wf,--state-dir=$remote_builds_dir/$experiment/state",
-            "-Wf,--trap",
-            "-Wf,--catch-outerspace",
-            "-Wf,--catch-write-textsegment",
-            "-Wf,--timeout=500000",
-            "-Wf,--ok-marker=fail_marker_positive",
-            "-Wf,--fail-marker=fail_marker_negative",
-            "-Wf,--detected-marker=fail_marker_detected",
-            ">/dev/null"
-        )
-    ) == 0 or die "client failed: $?";
+    system($client_command) == 0 or die "client failed: $?";
 
     say "Killing server with pid $pid...";
     kill 'TERM', $pid;
@@ -258,6 +260,10 @@ sub results {
     }
 }
 
+system("rm $remote_root/runner.log");
+system("rm $remote_root/server.log");
+system("rm $remote_root/client.log");
+
 # Run experiments
 my @experiments = Util::find_subdirs($remote_builds_dir);
 for my $experiment (@experiments) {
@@ -268,6 +274,10 @@ for my $experiment (@experiments) {
     import_trace($experiment);
     inject($experiment);
     results($experiment);
+
+    system("mv $remote_root/runner.log $remote_builds_dir/$experiment/runner.log");
+    system("mv $remote_root/server.log $remote_builds_dir/$experiment/server.log");
+    system("mv $remote_root/client.log $remote_builds_dir/$experiment/client.log");
 }
 
 Util::notify("Finished all experiments");

targets/wasm-module/matrix0_base.cpp

@@ -0,0 +1,63 @@
+#include "../lib.h"
+
+#define REPLICA_COUNT 1
+#define MAT_SIZE 3
+
+static uint32_t X[MAT_SIZE * MAT_SIZE] = {
+    0, 1, 2, //
+    3, 4, 5, //
+    6, 7, 8  //
+};
+static uint32_t Y[MAT_SIZE * MAT_SIZE] = {
+    8, 7, 6, //
+    5, 4, 3, //
+    2, 1, 0  //
+};
+static uint32_t Calculated[MAT_SIZE * MAT_SIZE] = {0};
+
+static INLINE uint32_t *idx(uint32_t *matrix, uint8_t x, uint8_t y) {
+  return &matrix[y * MAT_SIZE + x];
+}
+
+template <const unsigned int N> static INLINE void matrix(void) {
+  for (uint8_t y = 0; y < MAT_SIZE; ++y) {
+    for (uint8_t x = 0; x < MAT_SIZE; ++x) {
+      for (uint8_t k = 0; k < MAT_SIZE; ++k) {
+        *idx(Calculated, x, y) += (*idx(X, k, y)) * (*idx(Y, x, k));
+      }
+    }
+  }
+}
+
+static int cmp(uint32_t *A, uint32_t *B) {
+  for (uint8_t i = 0; i < MAT_SIZE * MAT_SIZE; ++i) {
+    if (A[i] != B[i]) {
+      return 0;
+    }
+  }
+  return 1;
+}
+
+extern "C" EXPORT("wasm_module") int wasm_module(void) {
+  fail_start_trace();
+
+  matrix<0>();
+
+  fail_stop_trace();
+
+  uint32_t Expected[MAT_SIZE * MAT_SIZE] = {
+      9,  6,  3,  //
+      54, 42, 30, //
+      99, 78, 57  //
+  };
+
+  if (cmp(Calculated, Expected)) {
+    HOST_PRINT("result correct.\n");
+    fail_marker_positive();
+    return 0;
+  } else {
+    HOST_PRINT("result incorrect.\n");
+    fail_marker_negative();
+    return 1;
+  }
+}