diff --git a/.gitignore b/.gitignore index 895ac15..02ec4b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /build-* +/builds /ghidra/projects/**/*.lock* +/mars-db.conf diff --git a/flake.nix b/flake.nix index 92ea817..3415d28 100644 --- a/flake.nix +++ b/flake.nix @@ -477,6 +477,7 @@ rec { # Dynamic libraries from buildinputs: # LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs; + JUST_WORKING_DIRECTORY = "./"; JUST_JUSTFILE = "./scripts/nixos.just"; # Those are read by the justfile diff --git a/scripts/build.pl b/scripts/build.pl new file mode 100755 index 0000000..ca2bcb0 --- /dev/null +++ b/scripts/build.pl @@ -0,0 +1,72 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use diagnostics; + +use feature 'say'; + +my $local_root = '/home/christoph/Notes/TU/MastersThesis/FailNix'; +my $local_builds_dir = "$local_root/builds"; +my $local_experiments_dir = "$local_root/targets/wasm-module"; +my $justbin = "$local_root/just-bin/just"; +my $justfile = "$local_root/scripts/nixos.just"; + +my @targets = ( "fail", "linux", "linux-baremetal" ); +my @modes = ( "c", "aot", "interp" ); + +sub just { + say "Running: just @_..."; + system("$justbin -d $local_root -f $justfile @_") + and die "Build failed"; +} + +die "Please archive or delete old experiments before building" + if ( -d $local_builds_dir ); + +# Find experiments +opendir( my $dhandle, $local_experiments_dir ) + or die "opendir$local_experiments_dir): $!"; +my @experiments = + map { s/\.cpp//r } grep { -f "$local_experiments_dir/$_" } readdir($dhandle); +closedir($dhandle); + +# Select experiments +say "Experiments:"; +foreach (@experiments) { say " - $_"; } +print "Enter single experiment name, comma-separated list or \"all\": "; +my $experiment_sel = ; +chomp $experiment_sel; +my @selected_experiments = + $experiment_sel eq "all" ? @experiments : split( ',', $experiment_sel ); + +# Select targets +say "Targets:"; +foreach (@targets) { say " - $_"; } +print "Enter single target, comma-separated list or \"all\": "; +my $target_sel = ; +chomp $target_sel; +my @selected_targets = + $target_sel eq "all" ? @targets : split( ',', $target_sel ); + +# Select modes +say "Modes:"; +foreach (@modes) { say " - $_"; } +print "Enter single mode, comma-separated list or \"all\": "; +my $mode_sel = ; +chomp $mode_sel; +my @selected_modes = $mode_sel eq "all" ? @modes : split( ',', $mode_sel ); + +# Build everything +system( "mkdir", "-p", "$local_builds_dir" ); +foreach my $experiment (@selected_experiments) { + foreach my $target (@selected_targets) { + foreach my $mode (@selected_modes) { + just( "build", $experiment, $target, $mode ); + system( +"mv $local_root/build-$experiment $local_builds_dir/$experiment-$target-$mode" + ); + system("rm -rf $local_root/build-$experiment"); + } + } +} diff --git a/scripts/deploy.pl b/scripts/deploy.pl new file mode 100755 index 0000000..83a8591 --- /dev/null +++ b/scripts/deploy.pl @@ -0,0 +1,123 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use diagnostics; + +use File::Basename qw(basename); +use Net::OpenSSH; +use DateTime; +use DBI; + +use feature 'say'; + +sub remote { + my ( $ssh, @cmd ) = @_; + $ssh->system(@cmd); + $ssh->error and die "Remote command failed (@cmd): " . $ssh->error; +} + +sub shell_quote { + my ($string) = @_; + $string =~ s/'/'"'"'/g; + return "'$string'"; +} + +my $date = DateTime->now->iso8601; +my $screen_name = 'smchurla_fail'; + +my $local_root = '/home/christoph/Notes/TU/MastersThesis/FailNix'; +my $local_builds_dir = "$local_root/builds"; + +my $remote_host = 'mars'; # smchurla@mars.cs.tu-dortmund.de +my $remote_root = '/home/lab/smchurla/Documents/failnix'; +my $remote_db_conf = "$remote_root/db.conf"; +my $remote_builds_dir = "$remote_root/builds"; +my $remote_runner = "$remote_root/scripts/runner.pl"; + +# The mars db is bound to local port 3306 over SSH. +# - This requires using the configured 'mars' +# remote_host instead of smchurla@mars +# or setting up another tunnel here +my $db_host = "127.0.0.1"; +my $db_port = "3306"; +my $db_user = "smchurla"; +my $db_prefix = "$db_user\_$date"; + +# Database password +open( my $fhandle, '<', "$local_root/mars-db.conf" ) + or die "Failed to read mars-db.conf: $!"; +chomp( my $db_password = <$fhandle> ); +close($fhandle); + +# Initialize SSH connection +# - This connection also sets up the database tunnel +my $ssh = Net::OpenSSH->new( + $remote_host, + timeout => 30, + master_opts => [ + -o => 'BatchMode=yes', + -o => 'StrictHostKeyChecking=accept-new', + ], +); +$ssh->error and die 'SSH connection failed: ' . $ssh->error; +say 'Connected to mars.cs.tu-dortmund.de'; + +# TODO: Abort if old experiments are still on the server => meaning they're not archived + +# Pull changes +# remote( $ssh, 'git', '-C', $remote_root, 'pull' ); # TODO: Requires auth + +# Find new experiments +opendir( my $dhandle, $local_builds_dir ) + or die "opendir($local_builds_dir): $!"; +my @experiments = grep { $_ ne '.' && $_ ne '..' && -d "$local_builds_dir/$_" } + readdir($dhandle); +closedir($dhandle); + +# Upload new experiments +remote( $ssh, 'mkdir', '-p', $remote_builds_dir ); +foreach (@experiments) { + say " - Uploading $_ to $remote_builds_dir/$date\_$_..."; + $ssh->scp_put( + { + recursive => 1, + copy_attrs => 1 + }, + "$local_builds_dir/$_", + "$remote_builds_dir/$date\_$_" + ) or die "Failed to upload $_: " . $ssh->error; +} + +# Initialize db connection +my $dbh = DBI->connect( "DBI:MariaDB:host=$db_host;port=$db_port", + $db_user, $db_password ) + or die 'Failed to connect to database: ' . $DBI::errstr; +say 'Connected to database'; + +say 'Existing databases:'; +my @db_names = + sort + map { s/DBI:MariaDB://r } + grep { !/information_schema|smchurla_ll/ } $dbh->data_sources(); +foreach (@db_names) { say " - $_"; } + +# Create dbs for new experiments +say 'Creating databases...'; +foreach (@experiments) { + say "Creating database $db_prefix\_$_..."; + $dbh->do("create database `$db_prefix\_$_`") + or die "Failed to create database: " . $dbh->errstr; +} + +# Kill old screen session +remote( $ssh, "screen", "-S", $screen_name, "-X", "quit" ); + +# Start new screen session +my $invoke_runner = "perl " . shell_quote($remote_runner); +remote( $ssh, "screen", "-dmS", $screen_name, "sh", "-lc", + "exec $invoke_runner" ); + +say "Started remote runner for ", scalar(@experiments), " experiments"; + +$dbh->disconnect or warn $dbh->errstr; diff --git a/scripts/dropdb.pl b/scripts/dropdb.pl new file mode 100755 index 0000000..578e1ef --- /dev/null +++ b/scripts/dropdb.pl @@ -0,0 +1,62 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use diagnostics; + +use Net::OpenSSH; +use DBI; + +use feature 'say'; + +my $local_root = '/home/christoph/Notes/TU/MastersThesis/FailNix'; + +my $remote_host = 'mars'; # smchurla@mars.cs.tu-dortmund.de + +# The mars db is bound to local port 3306 over SSH. +# - This requires using the configured 'mars' +# remote_host instead of smchurla@mars +# or setting up another tunnel here +my $db_host = "127.0.0.1"; +my $db_port = "3306"; +my $db_user = "smchurla"; + +# Database password +open( my $fhandle, '<', "$local_root/mars-db.conf" ) + or die "Failed to read mars-db.conf: $!"; +chomp( my $db_password = <$fhandle> ); +close($fhandle); + +# Initialize SSH connection +# - This connection also sets up the database tunnel +my $ssh = Net::OpenSSH->new( + $remote_host, + timeout => 30, + master_opts => [ + -o => 'BatchMode=yes', + -o => 'StrictHostKeyChecking=accept-new', + ], +); +$ssh->error and die 'SSH connection failed: ' . $ssh->error; +say 'Connected to mars.cs.tu-dortmund.de'; + +# Initialize db connection +my $dbh = DBI->connect( "DBI:MariaDB:host=$db_host;port=$db_port", + $db_user, $db_password ) + or die 'Failed to connect to database: ' . $DBI::errstr; +say 'Connected to database'; + +say 'Existing databases:'; +my @db_names = + sort + map { s/DBI:MariaDB://r } + grep { !/information_schema|smchurla_ll/ } $dbh->data_sources(); +foreach (@db_names) { say " - $_"; } + +print 'Enter name to delete: '; +my $db_sel = ; +chomp $db_sel; +$dbh->do("drop database `$db_sel`") + or die "Failed to drop database: " . $dbh->errstr; + +$dbh->disconnect or warn $dbh->errstr; diff --git a/scripts/runner.pl b/scripts/runner.pl new file mode 100644 index 0000000..61b48ca --- /dev/null +++ b/scripts/runner.pl @@ -0,0 +1,279 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use diagnostics; + +use feature 'say'; + +my $remote_root = '/home/lab/smchurla/Documents/failnix'; +my $remote_db_conf = "$remote_root/db.conf"; +my $remote_builds_dir = "$remote_root/builds"; +my $remote_runner = "$remote_root/scripts/runner.pl"; + +my $db_user = "smchurla"; +my $db_prefix = "$db_user"; + +my $fail_server_port = '22941'; +my $resultbrowser_port = '22941'; +my $fail_bin = "$remote_root/fail/bin"; +my $fail_share = "$remote_root/fail/share"; +my $bochs_runner = "$fail_bin/bochs-experiment-runner.py"; +my $fail_trace = "$fail_bin/fail-x86-tracing"; +my $fail_dump = "$fail_bin/dump-trace"; +my $fail_import = "$fail_bin/import-trace"; +my $fail_prune = "$fail_bin/prune-trace"; +my $fail_server = "$fail_bin/generic-experiment-server"; +my $fail_inject = "$fail_bin/generic-experiment-client"; +my $result_browser = "$fail_bin/resultbrowser.py"; + +my $ntfy_url = "https://ntfy.vps.chriphost.de"; +my $ntfy_token = "tk_rx8fd6hojuz4ekcb72j7juugkbmga"; # May be public +my $ntfy_topic = "fail-alerts"; + +sub notify { + my ($msg) = @_; + + system( "curl", "-H", "Authorization: Bearer $ntfy_token", + "-d", $msg, "$ntfy_url/$ntfy_topic" ); +} + +sub update_db_config { + my ($experiment) = @_; + + open( my $fhandle, '<', $remote_db_conf ) + or die "failed to open db.conf: $!"; + my @lines; + while ( my $line = <$fhandle> ) { + if ( $line =~ /^database=/ ) { + $line = "database=$db_prefix\_$experiment"; + } + push @lines, $line; + } + close($fhandle) or die "failed to close db.conf: $!"; + + open( my $fhandle, '>', $remote_db_conf ) + or die "failed to open db.conf: $!"; + print( $fhandle, @lines ); + close($fhandle) or die "failed to close db.conf: $!"; +} + +sub cpu_count { + open( my $handle, "/proc/cpuinfo" ) or die "Can't open cpuinfo: $!\n"; + my $count = scalar( map /^processor/, <$handle> ); + close $handle; + + return $count; +} + +# Find new experiments +opendir( my $dhandle, $remote_builds_dir ) + or die "opendir($remote_builds_dir): $!"; +my @experiments = grep { $_ ne '.' && $_ ne '..' && -d "$remote_builds_dir/$_" } + readdir($dhandle); +closedir($dhandle); + +sub trace { + my ($experiment) = @_; + + notify("Tracing $experiment..."); + + system( + $bochs_runner, + "-V $fail_share/vgabios.bin", + "-b $fail_share/BIOS-bochs-latest", + "-1", + "-f $fail_trace", + "-e $remote_builds_dir/$experiment/system.elf", + "-i $remote_builds_dir/$experiment/system.iso", + "--", + "-Wf,--start-symbol=fail_start_trace", + "-Wf,--save-symbol=fail_start_trace", + "-Wf,--end-symbol=fail_stop_trace", + "-Wf,--state-file=$remote_builds_dir/$experiment/state", + "-Wf,--trace-file=$remote_builds_dir/$experiment/trace.pb", + "-Wf,--elf-file=$remote_builds_dir/$experiment/system.elf" + ); + + notify("Tracing $experiment complete."); +} + +sub import_trace { + my ($experiment) = @_; + + notify("Importing $experiment trace..."); + + system( + "$fail_import", + "--database-option-file $remote_db_conf", + "-t $remote_builds_dir/$experiment/trace.pb", + "-i MemoryImporter", + "-e $remote_builds_dir/$experiment/system.elf", + "-v $experiment", + "-b mem" + ); + system( + "$fail_import", + "--database-option-file $remote_db_conf", + "-t $remote_builds_dir/$experiment/trace.pb", + "-i RegisterImporter", + "-e $remote_builds_dir/$experiment/system.elf", + "-v $experiment", + "-b regs --flags" + ); + system( + "$fail_import", + "--database-option-file $remote_db_conf", + "-t $remote_builds_dir/$experiment/trace.pb", + "-i RegisterImporter", + "-e $remote_builds_dir/$experiment/system.elf", + "-v $experiment", + "-b ip --no-gp --ip" + ); + system( + "$fail_import", + "--database-option-file $remote_db_conf", + "-t $remote_builds_dir/$experiment/trace.pb", + "-i ElfImporter", + "--objdump objdump -e $remote_builds_dir/$experiment/system.elf", + "-v $experiment", + "-b ip" + ); + system( + "$fail_import", + "--database-option-file $remote_db_conf", + "-t $remote_builds_dir/$experiment/trace.pb", + "-i ElfImporter", + "--objdump objdump -e $remote_builds_dir/$experiment/system.elf", + "-v $experiment", + "-b mem" + ); + system( + "$fail_import", + "--database-option-file $remote_db_conf", + "-t $remote_builds_dir/$experiment/trace.pb", + "-i ElfImporter", + "--objdump objdump -e $remote_builds_dir/$experiment/system.elf", + "-v $experiment", + "-b regs" + ); + system( + "$fail_prune", + "--database-option-file $remote_db_conf", + "-v $experiment", + "-b %%", "--overwrite" + ); + + notify("Importing $experiment trace complete."); +} + +sub inject { + my ($experiment) = @_; + + notify("Injecting $experiment..."); + + my $pid = fork(); + die "fork failed: $!" unless defined $pid; + + if ( $pid == 0 ) { + + # child -> server + exec( + $fail_server, + "--port $fail_server_port", + "--database-option-file $remote_db_conf", + "-v $experiment", + "-b %", + "--inject-single-bit", + "--inject-registers" + ) or die "exec server failed: $!"; + } + + # parent -> client + my $count = cpu_count(); + system( + "nice", + $bochs_runner, + "-V $fail_share/vgabios.bin", + "-b $fail_share/BIOS-bochs-latest", + "-f $fail_inject", + "-e $remote_builds_dir/$experiment/system.elf", + "-i $remote_builds_dir/$experiment/system.iso", + "-j $count", + "--", + "-Wf,--server-port=$fail_server_port", + "-Wf,--state-dir=$remote_builds_dir/$experiment/state", + "-Wf,--trap", + "-Wf,--catch-outerspace", + + # "-Wf,--catch-write-textsegment", + "-Wf,--timeout=500000", + "-Wf,--ok-marker=fail_marker_positive", + "-Wf,--fail-marker=fail_marker_negative", + "-Wf,--detected-marker=fail_marker_detected", + "> /dev/null" + ) or die "client failed: $?"; + + kill 'TERM', $pid; + waitpid( $pid, 0 ); + + notify("Injecting $experiment complete."); +} + +sub results { + my ($experiment) = @_; + + my $results_overview_query = "SELECT +variant, benchmark, resulttype, sum(t.time2 - t.time1 + 1) as faults +FROM variant v +JOIN trace t ON v.id = t.variant_id +JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address +JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id +JOIN fsppilot p ON r.pilot_id = p.id +WHERE v.variant = '$experiment' +GROUP BY v.id, resulttype +ORDER BY variant, benchmark, resulttype;"; + + my $results_overview = qx{ + mariadb --defaults-file=$remote_db_conf -t -e "$results_overview_query" + }; + die "Query failed: $?" if $? != 0; + + open( my $fhandle, '>', "$remote_builds_dir/$experiment/results.txt" ) + or die "failed to open file: $!"; + print( $fhandle, $results_overview ); + close($fhandle) or die "failed to close file: $!"; + + my $fail_markers_query = "SELECT +CONCAT('0x', HEX(p.injection_instr_absolute)) AS fault_address, +SUM(t.time2 - t.time1 + 1) AS total_fail_markers +FROM trace t +JOIN variant v ON v.id = t.variant_id +JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address +JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id +JOIN fsppilot p ON p.id = r.pilot_id +WHERE v.variant = '$experiment' AND r.resulttype = 'FAIL_MARKER' +GROUP BY p.injection_instr_absolute +ORDER BY SUM(t.time2 - t.time1 + 1) DESC;"; + + my $fail_markers = qx{ + mariadb --defaults-file=$remote_db_conf --batch --raw -e "$fail_markers_query" + }; + die "Query failed: $?" if $? != 0; + $fail_markers =~ s/\t/,/g; + + open( my $fhandle, '>', "$remote_builds_dir/$experiment/markers.csv" ) + or die "failed to open file: $!"; + print( $fhandle, $fail_markers ); + close($fhandle) or die "failed to close file: $!"; +} + +# Run experiments +for my $experiment (@experiments) { + update_db_config($experiment); + + trace($experiment); + import_trace($experiment); + inject($experiment); + results($experiment); +}