add wip buildscripts for automated experiment execution
This commit is contained in:
72
scripts/build.pl
Executable file
72
scripts/build.pl
Executable file
@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use diagnostics;
|
||||
|
||||
use feature 'say';
|
||||
|
||||
my $local_root = '/home/christoph/Notes/TU/MastersThesis/FailNix';
|
||||
my $local_builds_dir = "$local_root/builds";
|
||||
my $local_experiments_dir = "$local_root/targets/wasm-module";
|
||||
my $justbin = "$local_root/just-bin/just";
|
||||
my $justfile = "$local_root/scripts/nixos.just";
|
||||
|
||||
my @targets = ( "fail", "linux", "linux-baremetal" );
|
||||
my @modes = ( "c", "aot", "interp" );
|
||||
|
||||
sub just {
|
||||
say "Running: just @_...";
|
||||
system("$justbin -d $local_root -f $justfile @_")
|
||||
and die "Build failed";
|
||||
}
|
||||
|
||||
die "Please archive or delete old experiments before building"
|
||||
if ( -d $local_builds_dir );
|
||||
|
||||
# Find experiments
|
||||
opendir( my $dhandle, $local_experiments_dir )
|
||||
or die "opendir$local_experiments_dir): $!";
|
||||
my @experiments =
|
||||
map { s/\.cpp//r } grep { -f "$local_experiments_dir/$_" } readdir($dhandle);
|
||||
closedir($dhandle);
|
||||
|
||||
# Select experiments
|
||||
say "Experiments:";
|
||||
foreach (@experiments) { say " - $_"; }
|
||||
print "Enter single experiment name, comma-separated list or \"all\": ";
|
||||
my $experiment_sel = <STDIN>;
|
||||
chomp $experiment_sel;
|
||||
my @selected_experiments =
|
||||
$experiment_sel eq "all" ? @experiments : split( ',', $experiment_sel );
|
||||
|
||||
# Select targets
|
||||
say "Targets:";
|
||||
foreach (@targets) { say " - $_"; }
|
||||
print "Enter single target, comma-separated list or \"all\": ";
|
||||
my $target_sel = <STDIN>;
|
||||
chomp $target_sel;
|
||||
my @selected_targets =
|
||||
$target_sel eq "all" ? @targets : split( ',', $target_sel );
|
||||
|
||||
# Select modes
|
||||
say "Modes:";
|
||||
foreach (@modes) { say " - $_"; }
|
||||
print "Enter single mode, comma-separated list or \"all\": ";
|
||||
my $mode_sel = <STDIN>;
|
||||
chomp $mode_sel;
|
||||
my @selected_modes = $mode_sel eq "all" ? @modes : split( ',', $mode_sel );
|
||||
|
||||
# Build everything
|
||||
system( "mkdir", "-p", "$local_builds_dir" );
|
||||
foreach my $experiment (@selected_experiments) {
|
||||
foreach my $target (@selected_targets) {
|
||||
foreach my $mode (@selected_modes) {
|
||||
just( "build", $experiment, $target, $mode );
|
||||
system(
|
||||
"mv $local_root/build-$experiment $local_builds_dir/$experiment-$target-$mode"
|
||||
);
|
||||
system("rm -rf $local_root/build-$experiment");
|
||||
}
|
||||
}
|
||||
}
|
||||
123
scripts/deploy.pl
Executable file
123
scripts/deploy.pl
Executable file
@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use diagnostics;
|
||||
|
||||
use File::Basename qw(basename);
|
||||
use Net::OpenSSH;
|
||||
use DateTime;
|
||||
use DBI;
|
||||
|
||||
use feature 'say';
|
||||
|
||||
sub remote {
|
||||
my ( $ssh, @cmd ) = @_;
|
||||
$ssh->system(@cmd);
|
||||
$ssh->error and die "Remote command failed (@cmd): " . $ssh->error;
|
||||
}
|
||||
|
||||
sub shell_quote {
|
||||
my ($string) = @_;
|
||||
$string =~ s/'/'"'"'/g;
|
||||
return "'$string'";
|
||||
}
|
||||
|
||||
my $date = DateTime->now->iso8601;
|
||||
my $screen_name = 'smchurla_fail';
|
||||
|
||||
my $local_root = '/home/christoph/Notes/TU/MastersThesis/FailNix';
|
||||
my $local_builds_dir = "$local_root/builds";
|
||||
|
||||
my $remote_host = 'mars'; # smchurla@mars.cs.tu-dortmund.de
|
||||
my $remote_root = '/home/lab/smchurla/Documents/failnix';
|
||||
my $remote_db_conf = "$remote_root/db.conf";
|
||||
my $remote_builds_dir = "$remote_root/builds";
|
||||
my $remote_runner = "$remote_root/scripts/runner.pl";
|
||||
|
||||
# The mars db is bound to local port 3306 over SSH.
|
||||
# - This requires using the configured 'mars'
|
||||
# remote_host instead of smchurla@mars
|
||||
# or setting up another tunnel here
|
||||
my $db_host = "127.0.0.1";
|
||||
my $db_port = "3306";
|
||||
my $db_user = "smchurla";
|
||||
my $db_prefix = "$db_user\_$date";
|
||||
|
||||
# Database password
|
||||
open( my $fhandle, '<', "$local_root/mars-db.conf" )
|
||||
or die "Failed to read mars-db.conf: $!";
|
||||
chomp( my $db_password = <$fhandle> );
|
||||
close($fhandle);
|
||||
|
||||
# Initialize SSH connection
|
||||
# - This connection also sets up the database tunnel
|
||||
my $ssh = Net::OpenSSH->new(
|
||||
$remote_host,
|
||||
timeout => 30,
|
||||
master_opts => [
|
||||
-o => 'BatchMode=yes',
|
||||
-o => 'StrictHostKeyChecking=accept-new',
|
||||
],
|
||||
);
|
||||
$ssh->error and die 'SSH connection failed: ' . $ssh->error;
|
||||
say 'Connected to mars.cs.tu-dortmund.de';
|
||||
|
||||
# TODO: Abort if old experiments are still on the server => meaning they're not archived
|
||||
|
||||
# Pull changes
|
||||
# remote( $ssh, 'git', '-C', $remote_root, 'pull' ); # TODO: Requires auth
|
||||
|
||||
# Find new experiments
|
||||
opendir( my $dhandle, $local_builds_dir )
|
||||
or die "opendir($local_builds_dir): $!";
|
||||
my @experiments = grep { $_ ne '.' && $_ ne '..' && -d "$local_builds_dir/$_" }
|
||||
readdir($dhandle);
|
||||
closedir($dhandle);
|
||||
|
||||
# Upload new experiments
|
||||
remote( $ssh, 'mkdir', '-p', $remote_builds_dir );
|
||||
foreach (@experiments) {
|
||||
say " - Uploading $_ to $remote_builds_dir/$date\_$_...";
|
||||
$ssh->scp_put(
|
||||
{
|
||||
recursive => 1,
|
||||
copy_attrs => 1
|
||||
},
|
||||
"$local_builds_dir/$_",
|
||||
"$remote_builds_dir/$date\_$_"
|
||||
) or die "Failed to upload $_: " . $ssh->error;
|
||||
}
|
||||
|
||||
# Initialize db connection
|
||||
my $dbh = DBI->connect( "DBI:MariaDB:host=$db_host;port=$db_port",
|
||||
$db_user, $db_password )
|
||||
or die 'Failed to connect to database: ' . $DBI::errstr;
|
||||
say 'Connected to database';
|
||||
|
||||
say 'Existing databases:';
|
||||
my @db_names =
|
||||
sort
|
||||
map { s/DBI:MariaDB://r }
|
||||
grep { !/information_schema|smchurla_ll/ } $dbh->data_sources();
|
||||
foreach (@db_names) { say " - $_"; }
|
||||
|
||||
# Create dbs for new experiments
|
||||
say 'Creating databases...';
|
||||
foreach (@experiments) {
|
||||
say "Creating database $db_prefix\_$_...";
|
||||
$dbh->do("create database `$db_prefix\_$_`")
|
||||
or die "Failed to create database: " . $dbh->errstr;
|
||||
}
|
||||
|
||||
# Kill old screen session
|
||||
remote( $ssh, "screen", "-S", $screen_name, "-X", "quit" );
|
||||
|
||||
# Start new screen session
|
||||
my $invoke_runner = "perl " . shell_quote($remote_runner);
|
||||
remote( $ssh, "screen", "-dmS", $screen_name, "sh", "-lc",
|
||||
"exec $invoke_runner" );
|
||||
|
||||
say "Started remote runner for ", scalar(@experiments), " experiments";
|
||||
|
||||
$dbh->disconnect or warn $dbh->errstr;
|
||||
62
scripts/dropdb.pl
Executable file
62
scripts/dropdb.pl
Executable file
@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use diagnostics;
|
||||
|
||||
use Net::OpenSSH;
|
||||
use DBI;
|
||||
|
||||
use feature 'say';
|
||||
|
||||
my $local_root = '/home/christoph/Notes/TU/MastersThesis/FailNix';
|
||||
|
||||
my $remote_host = 'mars'; # smchurla@mars.cs.tu-dortmund.de
|
||||
|
||||
# The mars db is bound to local port 3306 over SSH.
|
||||
# - This requires using the configured 'mars'
|
||||
# remote_host instead of smchurla@mars
|
||||
# or setting up another tunnel here
|
||||
my $db_host = "127.0.0.1";
|
||||
my $db_port = "3306";
|
||||
my $db_user = "smchurla";
|
||||
|
||||
# Database password
|
||||
open( my $fhandle, '<', "$local_root/mars-db.conf" )
|
||||
or die "Failed to read mars-db.conf: $!";
|
||||
chomp( my $db_password = <$fhandle> );
|
||||
close($fhandle);
|
||||
|
||||
# Initialize SSH connection
|
||||
# - This connection also sets up the database tunnel
|
||||
my $ssh = Net::OpenSSH->new(
|
||||
$remote_host,
|
||||
timeout => 30,
|
||||
master_opts => [
|
||||
-o => 'BatchMode=yes',
|
||||
-o => 'StrictHostKeyChecking=accept-new',
|
||||
],
|
||||
);
|
||||
$ssh->error and die 'SSH connection failed: ' . $ssh->error;
|
||||
say 'Connected to mars.cs.tu-dortmund.de';
|
||||
|
||||
# Initialize db connection
|
||||
my $dbh = DBI->connect( "DBI:MariaDB:host=$db_host;port=$db_port",
|
||||
$db_user, $db_password )
|
||||
or die 'Failed to connect to database: ' . $DBI::errstr;
|
||||
say 'Connected to database';
|
||||
|
||||
say 'Existing databases:';
|
||||
my @db_names =
|
||||
sort
|
||||
map { s/DBI:MariaDB://r }
|
||||
grep { !/information_schema|smchurla_ll/ } $dbh->data_sources();
|
||||
foreach (@db_names) { say " - $_"; }
|
||||
|
||||
print 'Enter name to delete: ';
|
||||
my $db_sel = <STDIN>;
|
||||
chomp $db_sel;
|
||||
$dbh->do("drop database `$db_sel`")
|
||||
or die "Failed to drop database: " . $dbh->errstr;
|
||||
|
||||
$dbh->disconnect or warn $dbh->errstr;
|
||||
279
scripts/runner.pl
Normal file
279
scripts/runner.pl
Normal file
@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use diagnostics;
|
||||
|
||||
use feature 'say';
|
||||
|
||||
my $remote_root = '/home/lab/smchurla/Documents/failnix';
|
||||
my $remote_db_conf = "$remote_root/db.conf";
|
||||
my $remote_builds_dir = "$remote_root/builds";
|
||||
my $remote_runner = "$remote_root/scripts/runner.pl";
|
||||
|
||||
my $db_user = "smchurla";
|
||||
my $db_prefix = "$db_user";
|
||||
|
||||
my $fail_server_port = '22941';
|
||||
my $resultbrowser_port = '22941';
|
||||
my $fail_bin = "$remote_root/fail/bin";
|
||||
my $fail_share = "$remote_root/fail/share";
|
||||
my $bochs_runner = "$fail_bin/bochs-experiment-runner.py";
|
||||
my $fail_trace = "$fail_bin/fail-x86-tracing";
|
||||
my $fail_dump = "$fail_bin/dump-trace";
|
||||
my $fail_import = "$fail_bin/import-trace";
|
||||
my $fail_prune = "$fail_bin/prune-trace";
|
||||
my $fail_server = "$fail_bin/generic-experiment-server";
|
||||
my $fail_inject = "$fail_bin/generic-experiment-client";
|
||||
my $result_browser = "$fail_bin/resultbrowser.py";
|
||||
|
||||
my $ntfy_url = "https://ntfy.vps.chriphost.de";
|
||||
my $ntfy_token = "tk_rx8fd6hojuz4ekcb72j7juugkbmga"; # May be public
|
||||
my $ntfy_topic = "fail-alerts";
|
||||
|
||||
sub notify {
|
||||
my ($msg) = @_;
|
||||
|
||||
system( "curl", "-H", "Authorization: Bearer $ntfy_token",
|
||||
"-d", $msg, "$ntfy_url/$ntfy_topic" );
|
||||
}
|
||||
|
||||
sub update_db_config {
|
||||
my ($experiment) = @_;
|
||||
|
||||
open( my $fhandle, '<', $remote_db_conf )
|
||||
or die "failed to open db.conf: $!";
|
||||
my @lines;
|
||||
while ( my $line = <$fhandle> ) {
|
||||
if ( $line =~ /^database=/ ) {
|
||||
$line = "database=$db_prefix\_$experiment";
|
||||
}
|
||||
push @lines, $line;
|
||||
}
|
||||
close($fhandle) or die "failed to close db.conf: $!";
|
||||
|
||||
open( my $fhandle, '>', $remote_db_conf )
|
||||
or die "failed to open db.conf: $!";
|
||||
print( $fhandle, @lines );
|
||||
close($fhandle) or die "failed to close db.conf: $!";
|
||||
}
|
||||
|
||||
sub cpu_count {
|
||||
open( my $handle, "/proc/cpuinfo" ) or die "Can't open cpuinfo: $!\n";
|
||||
my $count = scalar( map /^processor/, <$handle> );
|
||||
close $handle;
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
# Find new experiments
|
||||
opendir( my $dhandle, $remote_builds_dir )
|
||||
or die "opendir($remote_builds_dir): $!";
|
||||
my @experiments = grep { $_ ne '.' && $_ ne '..' && -d "$remote_builds_dir/$_" }
|
||||
readdir($dhandle);
|
||||
closedir($dhandle);
|
||||
|
||||
sub trace {
|
||||
my ($experiment) = @_;
|
||||
|
||||
notify("Tracing $experiment...");
|
||||
|
||||
system(
|
||||
$bochs_runner,
|
||||
"-V $fail_share/vgabios.bin",
|
||||
"-b $fail_share/BIOS-bochs-latest",
|
||||
"-1",
|
||||
"-f $fail_trace",
|
||||
"-e $remote_builds_dir/$experiment/system.elf",
|
||||
"-i $remote_builds_dir/$experiment/system.iso",
|
||||
"--",
|
||||
"-Wf,--start-symbol=fail_start_trace",
|
||||
"-Wf,--save-symbol=fail_start_trace",
|
||||
"-Wf,--end-symbol=fail_stop_trace",
|
||||
"-Wf,--state-file=$remote_builds_dir/$experiment/state",
|
||||
"-Wf,--trace-file=$remote_builds_dir/$experiment/trace.pb",
|
||||
"-Wf,--elf-file=$remote_builds_dir/$experiment/system.elf"
|
||||
);
|
||||
|
||||
notify("Tracing $experiment complete.");
|
||||
}
|
||||
|
||||
sub import_trace {
|
||||
my ($experiment) = @_;
|
||||
|
||||
notify("Importing $experiment trace...");
|
||||
|
||||
system(
|
||||
"$fail_import",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-t $remote_builds_dir/$experiment/trace.pb",
|
||||
"-i MemoryImporter",
|
||||
"-e $remote_builds_dir/$experiment/system.elf",
|
||||
"-v $experiment",
|
||||
"-b mem"
|
||||
);
|
||||
system(
|
||||
"$fail_import",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-t $remote_builds_dir/$experiment/trace.pb",
|
||||
"-i RegisterImporter",
|
||||
"-e $remote_builds_dir/$experiment/system.elf",
|
||||
"-v $experiment",
|
||||
"-b regs --flags"
|
||||
);
|
||||
system(
|
||||
"$fail_import",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-t $remote_builds_dir/$experiment/trace.pb",
|
||||
"-i RegisterImporter",
|
||||
"-e $remote_builds_dir/$experiment/system.elf",
|
||||
"-v $experiment",
|
||||
"-b ip --no-gp --ip"
|
||||
);
|
||||
system(
|
||||
"$fail_import",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-t $remote_builds_dir/$experiment/trace.pb",
|
||||
"-i ElfImporter",
|
||||
"--objdump objdump -e $remote_builds_dir/$experiment/system.elf",
|
||||
"-v $experiment",
|
||||
"-b ip"
|
||||
);
|
||||
system(
|
||||
"$fail_import",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-t $remote_builds_dir/$experiment/trace.pb",
|
||||
"-i ElfImporter",
|
||||
"--objdump objdump -e $remote_builds_dir/$experiment/system.elf",
|
||||
"-v $experiment",
|
||||
"-b mem"
|
||||
);
|
||||
system(
|
||||
"$fail_import",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-t $remote_builds_dir/$experiment/trace.pb",
|
||||
"-i ElfImporter",
|
||||
"--objdump objdump -e $remote_builds_dir/$experiment/system.elf",
|
||||
"-v $experiment",
|
||||
"-b regs"
|
||||
);
|
||||
system(
|
||||
"$fail_prune",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-v $experiment",
|
||||
"-b %%", "--overwrite"
|
||||
);
|
||||
|
||||
notify("Importing $experiment trace complete.");
|
||||
}
|
||||
|
||||
sub inject {
|
||||
my ($experiment) = @_;
|
||||
|
||||
notify("Injecting $experiment...");
|
||||
|
||||
my $pid = fork();
|
||||
die "fork failed: $!" unless defined $pid;
|
||||
|
||||
if ( $pid == 0 ) {
|
||||
|
||||
# child -> server
|
||||
exec(
|
||||
$fail_server,
|
||||
"--port $fail_server_port",
|
||||
"--database-option-file $remote_db_conf",
|
||||
"-v $experiment",
|
||||
"-b %",
|
||||
"--inject-single-bit",
|
||||
"--inject-registers"
|
||||
) or die "exec server failed: $!";
|
||||
}
|
||||
|
||||
# parent -> client
|
||||
my $count = cpu_count();
|
||||
system(
|
||||
"nice",
|
||||
$bochs_runner,
|
||||
"-V $fail_share/vgabios.bin",
|
||||
"-b $fail_share/BIOS-bochs-latest",
|
||||
"-f $fail_inject",
|
||||
"-e $remote_builds_dir/$experiment/system.elf",
|
||||
"-i $remote_builds_dir/$experiment/system.iso",
|
||||
"-j $count",
|
||||
"--",
|
||||
"-Wf,--server-port=$fail_server_port",
|
||||
"-Wf,--state-dir=$remote_builds_dir/$experiment/state",
|
||||
"-Wf,--trap",
|
||||
"-Wf,--catch-outerspace",
|
||||
|
||||
# "-Wf,--catch-write-textsegment",
|
||||
"-Wf,--timeout=500000",
|
||||
"-Wf,--ok-marker=fail_marker_positive",
|
||||
"-Wf,--fail-marker=fail_marker_negative",
|
||||
"-Wf,--detected-marker=fail_marker_detected",
|
||||
"> /dev/null"
|
||||
) or die "client failed: $?";
|
||||
|
||||
kill 'TERM', $pid;
|
||||
waitpid( $pid, 0 );
|
||||
|
||||
notify("Injecting $experiment complete.");
|
||||
}
|
||||
|
||||
sub results {
|
||||
my ($experiment) = @_;
|
||||
|
||||
my $results_overview_query = "SELECT
|
||||
variant, benchmark, resulttype, sum(t.time2 - t.time1 + 1) as faults
|
||||
FROM variant v
|
||||
JOIN trace t ON v.id = t.variant_id
|
||||
JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address
|
||||
JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
|
||||
JOIN fsppilot p ON r.pilot_id = p.id
|
||||
WHERE v.variant = '$experiment'
|
||||
GROUP BY v.id, resulttype
|
||||
ORDER BY variant, benchmark, resulttype;";
|
||||
|
||||
my $results_overview = qx{
|
||||
mariadb --defaults-file=$remote_db_conf -t -e "$results_overview_query"
|
||||
};
|
||||
die "Query failed: $?" if $? != 0;
|
||||
|
||||
open( my $fhandle, '>', "$remote_builds_dir/$experiment/results.txt" )
|
||||
or die "failed to open file: $!";
|
||||
print( $fhandle, $results_overview );
|
||||
close($fhandle) or die "failed to close file: $!";
|
||||
|
||||
my $fail_markers_query = "SELECT
|
||||
CONCAT('0x', HEX(p.injection_instr_absolute)) AS fault_address,
|
||||
SUM(t.time2 - t.time1 + 1) AS total_fail_markers
|
||||
FROM trace t
|
||||
JOIN variant v ON v.id = t.variant_id
|
||||
JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_address = t.data_address
|
||||
JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
|
||||
JOIN fsppilot p ON p.id = r.pilot_id
|
||||
WHERE v.variant = '$experiment' AND r.resulttype = 'FAIL_MARKER'
|
||||
GROUP BY p.injection_instr_absolute
|
||||
ORDER BY SUM(t.time2 - t.time1 + 1) DESC;";
|
||||
|
||||
my $fail_markers = qx{
|
||||
mariadb --defaults-file=$remote_db_conf --batch --raw -e "$fail_markers_query"
|
||||
};
|
||||
die "Query failed: $?" if $? != 0;
|
||||
$fail_markers =~ s/\t/,/g;
|
||||
|
||||
open( my $fhandle, '>', "$remote_builds_dir/$experiment/markers.csv" )
|
||||
or die "failed to open file: $!";
|
||||
print( $fhandle, $fail_markers );
|
||||
close($fhandle) or die "failed to close file: $!";
|
||||
}
|
||||
|
||||
# Run experiments
|
||||
for my $experiment (@experiments) {
|
||||
update_db_config($experiment);
|
||||
|
||||
trace($experiment);
|
||||
import_trace($experiment);
|
||||
inject($experiment);
|
||||
results($experiment);
|
||||
}
|
||||
Reference in New Issue
Block a user