ecos: rewrite for DatabaseCampaign
+ modified resulttypes Change-Id: I463759e66e7497c80eeee9a065fc95e058ec3dc1
This commit is contained in:
Horst Schirmeier
@ -1,431 +1,9 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
|
||||
#include "campaign.hpp"
|
||||
#include "experimentInfo.hpp"
|
||||
#include "cpn/CampaignManager.hpp"
|
||||
#include "util/ProtoStream.hpp"
|
||||
#include "util/MemoryMap.hpp"
|
||||
#include "util/gzstream/gzstream.h"
|
||||
#include "util/CommandLine.hpp"
|
||||
|
||||
#include "../plugins/tracing/TracingPlugin.hpp"
|
||||
|
||||
using namespace std;
|
||||
using namespace fail;
|
||||
|
||||
#if BASELINE_ASSESSMENT
|
||||
const std::string EcosKernelTestCampaign::dir_prerequisites("prerequisites-baseline");
|
||||
const std::string EcosKernelTestCampaign::dir_images("images-baseline");
|
||||
#elif STACKPROTECTION
|
||||
const std::string EcosKernelTestCampaign::dir_prerequisites("prerequisites-stackprotection");
|
||||
const std::string EcosKernelTestCampaign::dir_images("images-stackprotection");
|
||||
#else
|
||||
const std::string EcosKernelTestCampaign::dir_prerequisites("prerequisites");
|
||||
const std::string EcosKernelTestCampaign::dir_images("images");
|
||||
#endif
|
||||
|
||||
bool EcosKernelTestCampaign::writeTraceInfo(unsigned instr_counter, unsigned timeout,
|
||||
unsigned mem1_low, unsigned mem1_high, // < 1M
|
||||
unsigned mem2_low, unsigned mem2_high, // >= 1M
|
||||
const std::string& variant, const std::string& benchmark) {
|
||||
ofstream ti(filename_traceinfo(variant, benchmark).c_str(), ios::out);
|
||||
if (!ti.is_open()) {
|
||||
cout << "failed to open " << filename_traceinfo(variant, benchmark) << endl;
|
||||
return false;
|
||||
}
|
||||
ti << instr_counter << endl << timeout << endl
|
||||
<< mem1_low << endl << mem1_high << endl
|
||||
<< mem2_low << endl << mem2_high << endl;
|
||||
ti.flush();
|
||||
ti.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EcosKernelTestCampaign::readTraceInfo(unsigned &instr_counter, unsigned &timeout,
|
||||
unsigned &mem1_low, unsigned &mem1_high, // < 1M
|
||||
unsigned &mem2_low, unsigned &mem2_high, // >= 1M
|
||||
const std::string& variant, const std::string& benchmark) {
|
||||
ifstream file(filename_traceinfo(variant, benchmark).c_str());
|
||||
if (!file.is_open()) {
|
||||
cout << "failed to open " << filename_traceinfo(variant, benchmark) << endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
string buf;
|
||||
unsigned count = 0;
|
||||
|
||||
while (getline(file, buf)) {
|
||||
stringstream ss(buf, ios::in);
|
||||
switch (count) {
|
||||
case 0:
|
||||
ss >> instr_counter;
|
||||
break;
|
||||
case 1:
|
||||
ss >> timeout;
|
||||
break;
|
||||
case 2:
|
||||
ss >> mem1_low;
|
||||
break;
|
||||
case 3:
|
||||
ss >> mem1_high;
|
||||
break;
|
||||
case 4:
|
||||
ss >> mem2_low;
|
||||
break;
|
||||
case 5:
|
||||
ss >> mem2_high;
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
file.close();
|
||||
assert(count == 6);
|
||||
return (count == 6);
|
||||
}
|
||||
|
||||
std::string EcosKernelTestCampaign::filename_memorymap(const std::string& variant, const std::string& benchmark)
|
||||
void EcosKernelTestCampaign::cb_send_pilot(DatabaseCampaignMessage pilot)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "memorymap.txt";
|
||||
}
|
||||
return "memorymap.txt";
|
||||
}
|
||||
|
||||
std::string EcosKernelTestCampaign::filename_state(unsigned instr_offset, const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
stringstream ss;
|
||||
ss << instr_offset;
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "state" + "-" + ss.str();
|
||||
}
|
||||
return "state-" + ss.str();
|
||||
}
|
||||
|
||||
std::string EcosKernelTestCampaign::filename_trace(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "trace.tc";
|
||||
}
|
||||
return "trace.tc";
|
||||
}
|
||||
|
||||
std::string EcosKernelTestCampaign::filename_traceinfo(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "traceinfo.txt";
|
||||
}
|
||||
return "traceinfo.txt";
|
||||
}
|
||||
|
||||
std::string EcosKernelTestCampaign::filename_elf(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_images + "/" + variant + "/" + benchmark + ".elf";
|
||||
}
|
||||
return "kernel.elf";
|
||||
}
|
||||
|
||||
// equivalence class type: addr, [i1, i2]
|
||||
// addr: byte to inject a bit-flip into
|
||||
// [i1, i2]: interval of instruction numbers, counted from experiment
|
||||
// begin
|
||||
typedef std::map<address_t, int> AddrLastaccessMap;
|
||||
|
||||
char const *variants[] = {
|
||||
#if !STACKPROTECTION
|
||||
"bitmap_vanilla",
|
||||
"bitmap_SUM+DMR",
|
||||
"bitmap_CRC",
|
||||
"bitmap_CRC+DMR",
|
||||
"bitmap_TMR",
|
||||
// "bitmap_Hamming",
|
||||
#elif STACKPROTECTION
|
||||
"bitmap_min_stacks_baseline",
|
||||
"bitmap_min_stacks_detection",
|
||||
"bitmap_min_stacks_protected",
|
||||
#endif
|
||||
0
|
||||
};
|
||||
|
||||
// big four (three): (mutex3,) bin_sem2, clocktruth, sync2
|
||||
// busy waiters, sloooow at ips=2666mhz: kill, mutex3, clocktruth
|
||||
// batch 1: line 1
|
||||
char const *benchmarks[] = {
|
||||
#if 1
|
||||
"bin_sem0", "bin_sem1", "bin_sem2", "bin_sem3", "clock1", "clockcnv",
|
||||
/*"clocktruth",*/ "cnt_sem1", "except1", "flag1", /*"kill",*/ "mqueue1", "mutex1",
|
||||
"mutex2", /*"mutex3",*/ "release", "sched1", "sync2", "sync3", "thread0",
|
||||
"thread1", "thread2",
|
||||
#elif 0 // clocktruth, mutex3, kill; sync2, bin_sem2; clockcnv
|
||||
"bin_sem0", "bin_sem1", /*"bin_sem2",*/ "bin_sem3", "clock1", "clockcnv",
|
||||
/**"clocktruth",*/ "cnt_sem1", "except1", "flag1", /**"kill",*/ "mqueue1", "mutex1",
|
||||
"mutex2", /**"mutex3",*/ "release", "sched1", /*"sync2",*/ "sync3", "thread0",
|
||||
"thread1", "thread2",
|
||||
#elif 0
|
||||
"thread1",
|
||||
#endif
|
||||
0
|
||||
};
|
||||
|
||||
bool EcosKernelTestCampaign::run()
|
||||
{
|
||||
CommandLine &cmd = CommandLine::Inst();
|
||||
|
||||
cmd.addOption("", "", Arg::None, "USAGE: fail-server [options...]");
|
||||
CommandLine::option_handle HELP =
|
||||
cmd.addOption("h", "help", Arg::None, "-h/--help \tPrint usage and exit");
|
||||
CommandLine::option_handle RESULTTABLE =
|
||||
cmd.addOption("r", "resulttable", Arg::Required, "-r/--resulttable \tTable to store results in (default: 'result')");
|
||||
Database::cmdline_setup();
|
||||
|
||||
if (!cmd.parse()) {
|
||||
m_log << "Error parsing arguments." << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
if (cmd[HELP].count() > 0) {
|
||||
cmd.printUsage();
|
||||
exit(0);
|
||||
}
|
||||
if (cmd[RESULTTABLE].count() > 0) {
|
||||
m_result_table = std::string(cmd[RESULTTABLE].first()->arg);
|
||||
} else {
|
||||
m_result_table = std::string("result");
|
||||
}
|
||||
m_log << "Storing results in table '" << m_result_table << "'\n";
|
||||
|
||||
db = Database::cmdline_connect();
|
||||
db_recv = Database::cmdline_connect();
|
||||
fspmethod_id = 1; // constant for now
|
||||
|
||||
std::stringstream ss;
|
||||
ss << "CREATE TABLE IF NOT EXISTS " << m_result_table << " ("
|
||||
"pilot_id int(11) NOT NULL,\n"
|
||||
"bitnr tinyint(4) NOT NULL,\n"
|
||||
"bit_width tinyint(4) NOT NULL,\n"
|
||||
"resulttype tinyint(4) NOT NULL,\n"
|
||||
"ecos_test_result tinyint(4) NOT NULL,\n"
|
||||
"latest_ip int(10) unsigned DEFAULT NULL,\n"
|
||||
"error_corrected tinyint(4) NOT NULL,\n"
|
||||
"details varchar(255) DEFAULT NULL,\n"
|
||||
"runtime float NOT NULL,\n"
|
||||
"PRIMARY KEY (pilot_id,bitnr))\n"
|
||||
"ENGINE = MyISAM";
|
||||
if (!db->query(ss.str().c_str())) return false;
|
||||
// collect results in parallel to avoid deadlock
|
||||
#ifndef __puma
|
||||
boost::thread collect_thread(&EcosKernelTestCampaign::collect_results, this);
|
||||
#endif
|
||||
|
||||
ss.str("");
|
||||
|
||||
/* Gather all unfinished jobs */
|
||||
m_log << "Looking for unfinished jobs in the database ..." << std::endl;
|
||||
ss << "(";
|
||||
for (int variant_nr = 0; variants[variant_nr]; ++variant_nr) {
|
||||
char const *variant = variants[variant_nr];
|
||||
ss << "v.variant = '" << variant << "' OR ";
|
||||
}
|
||||
ss << "0) AND ("; // dummy terminator
|
||||
for (int benchmark_nr = 0; benchmarks[benchmark_nr]; ++benchmark_nr) {
|
||||
char const *benchmark = benchmarks[benchmark_nr];
|
||||
ss << "v.benchmark = '" << benchmark << "' OR ";
|
||||
}
|
||||
ss << "0)"; // dummy terminator
|
||||
std::string sql_variants = ss.str();
|
||||
ss.str("");
|
||||
|
||||
#if 0
|
||||
ss << "SELECT STRAIGHT_JOIN p.id AS pilot_id, v.id AS variant_id, v.variant, v.benchmark, p.injection_instr, p.injection_instr_absolute, p.data_address, SUM(r.bit_width) AS existing_results "
|
||||
<< "FROM variant v "
|
||||
<< "JOIN fsppilot p ON p.variant_id = v.id "
|
||||
<< "LEFT JOIN result r ON r.pilot_id = p.id "
|
||||
<< "WHERE p.known_outcome = 0 "
|
||||
<< " AND p.fspmethod_id = " << fspmethod_id << " "
|
||||
<< " AND (" << sql_variants << ") "
|
||||
<< "GROUP BY p.id "
|
||||
<< "HAVING existing_results < 8 OR existing_results IS NULL "; // 8 results per pilot
|
||||
#elif 0
|
||||
std::string sql_select = "SELECT p.id AS pilot_id, v.id AS variant_id, v.variant, v.benchmark, p.injection_instr, p.injection_instr_absolute, p.data_address ";
|
||||
ss << "FROM variant v "
|
||||
<< "JOIN fsppilot p ON p.variant_id = v.id "
|
||||
<< "LEFT JOIN " << m_result_table << " r ON r.pilot_id = p.id "
|
||||
<< "WHERE p.known_outcome = 0 "
|
||||
<< " AND p.fspmethod_id = " << fspmethod_id << " "
|
||||
<< " AND (" << sql_variants << ") "
|
||||
<< " AND r.pilot_id IS NULL ";
|
||||
#elif 0
|
||||
std::string sql_select = "SELECT p.id AS pilot_id, v.id AS variant_id, v.variant, v.benchmark, p.injection_instr, p.injection_instr_absolute, p.data_address ";
|
||||
ss << "FROM variant v "
|
||||
<< "JOIN fsppilot p ON p.variant_id = v.id "
|
||||
// << "WHERE p.known_outcome = 0 "
|
||||
<< " AND p.fspmethod_id = " << fspmethod_id << " "
|
||||
<< " AND (" << sql_variants << ") ";
|
||||
#elif 1
|
||||
if (!db->query("CREATE TEMPORARY TABLE done_pilots (id INT UNSIGNED NOT NULL PRIMARY KEY) ENGINE=MyISAM")) return false;
|
||||
ss << "INSERT INTO done_pilots SELECT pilot_id FROM " << m_result_table << " GROUP BY pilot_id HAVING SUM(bit_width) = 8";
|
||||
if (!db->query(ss.str().c_str())) return false;
|
||||
unsigned finished_jobs = db->affected_rows();
|
||||
ss.str("");
|
||||
ss << "DELETE r FROM " << m_result_table << " r LEFT JOIN done_pilots ON r.pilot_id = done_pilots.id WHERE done_pilots.id IS NULL";
|
||||
if (!db->query(ss.str().c_str())) return false;
|
||||
unsigned deleted_rows = db->affected_rows();
|
||||
ss.str("");
|
||||
m_log << "Deleted " << deleted_rows << " rows from incomplete jobs" << std::endl;
|
||||
std::string sql_select = "SELECT p.id AS pilot_id, v.id AS variant_id, v.variant, v.benchmark, p.injection_instr, p.injection_instr_absolute, p.data_address ";
|
||||
ss << "FROM variant v "
|
||||
<< "JOIN fsppilot p ON p.variant_id = v.id "
|
||||
<< "LEFT JOIN done_pilots d ON d.id = p.id "
|
||||
<< "WHERE d.id IS NULL "
|
||||
<< " AND p.fspmethod_id = " << fspmethod_id << " "
|
||||
<< " AND (" << sql_variants << ") ";
|
||||
#endif
|
||||
std::string sql_body = ss.str();
|
||||
//std::string sql_order = "ORDER BY v.benchmark, v.variant";
|
||||
std::string sql_order = "ORDER BY v.id";
|
||||
//std::string sql_order = "";
|
||||
|
||||
/* Get the number of unfinished experiments */
|
||||
std::string sql_count = "SELECT COUNT(*) " + sql_body;
|
||||
m_log << sql_count << std::endl;
|
||||
MYSQL_RES *count = db->query(sql_count.c_str(), true);
|
||||
if (!count) {
|
||||
return false;
|
||||
}
|
||||
MYSQL_ROW row = mysql_fetch_row(count);
|
||||
unsigned unfinished_jobs;
|
||||
unfinished_jobs = strtoul(row[0], NULL, 10);
|
||||
|
||||
m_log << "Found " << unfinished_jobs << " unfinished jobs (" << finished_jobs << " already finished)." << std::endl;
|
||||
|
||||
std::string sql_pilots = sql_select + sql_body + sql_order;
|
||||
m_log << sql_pilots << std::endl;
|
||||
MYSQL_RES *pilots = db->query_stream(sql_pilots.c_str());
|
||||
if (!pilots) {
|
||||
return false;
|
||||
}
|
||||
|
||||
m_log << "Filling queue ..." << std::endl;
|
||||
|
||||
unsigned prev_variant_id = 0;
|
||||
while ((row = mysql_fetch_row(pilots))) {
|
||||
unsigned pilot_id = atoi(row[0]);
|
||||
unsigned variant_id = atoi(row[1]);
|
||||
unsigned injection_instr = atoi(row[4]);
|
||||
unsigned data_address = atoi(row[6]);
|
||||
|
||||
EcosKernelTestExperimentData *d = new EcosKernelTestExperimentData;
|
||||
d->msg.set_pilot_id(pilot_id);
|
||||
d->msg.set_variant(row[2]);
|
||||
d->msg.set_benchmark(row[3]);
|
||||
d->msg.set_instr2_offset(injection_instr);
|
||||
if (row[5]) {
|
||||
unsigned injection_instr_absolute = atoi(row[5]);
|
||||
d->msg.set_instr2_address(injection_instr_absolute);
|
||||
}
|
||||
d->msg.set_mem_addr(data_address);
|
||||
d->msg.set_faultmodel(ECOS_FAULTMODEL_BURST ? d->msg.BURST : d->msg.SINGLEBITFLIP);
|
||||
|
||||
if (prev_variant_id != variant_id) {
|
||||
m_log << "Enqueueing jobs for " << row[2] << "/" << row[3] << std::endl;
|
||||
}
|
||||
prev_variant_id = variant_id;
|
||||
|
||||
campaignmanager.addParam(d);
|
||||
}
|
||||
|
||||
if (mysql_errno(db->getHandle())) {
|
||||
std::cerr << "mysql_fetch_row failed: " << mysql_error(db->getHandle()) << std::endl;
|
||||
}
|
||||
|
||||
m_log << "finished, waiting for the clients to complete ..." << std::endl;
|
||||
campaignmanager.noMoreParameters();
|
||||
|
||||
#ifndef __puma
|
||||
collect_thread.join();
|
||||
#endif
|
||||
delete db_recv;
|
||||
m_log << "results complete, updating DB statistics ..." << std::endl;
|
||||
ss.str("");
|
||||
ss << "ANALYZE TABLE " << m_result_table;
|
||||
if (!db->query(ss.str().c_str())) return false;
|
||||
delete db;
|
||||
m_log << "terminating." << std::endl;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void EcosKernelTestCampaign::add_result(unsigned pilot_id,
|
||||
int instr2, address_t instr2_absolute, address_t ec_data_address,
|
||||
int bitnr, int bit_width, int resulttype, int ecos_test_result, address_t latest_ip,
|
||||
int error_corrected, const std::string& details, float runtime)
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "INSERT DELAYED INTO " << m_result_table << " "
|
||||
<< "(pilot_id, bitnr, bit_width, resulttype, ecos_test_result, latest_ip, error_corrected, details, runtime) VALUES "
|
||||
<< "(" << pilot_id << "," << bitnr << "," << bit_width << "," << resulttype << "," << ecos_test_result << ","
|
||||
<< latest_ip << "," << error_corrected << ",'" << details << "'," << runtime << ")";
|
||||
// Database::query is protected by a mutex
|
||||
db_recv->query(ss.str().c_str());
|
||||
}
|
||||
|
||||
void EcosKernelTestCampaign::collect_results()
|
||||
{
|
||||
EcosKernelTestExperimentData *res;
|
||||
while ((res = static_cast<EcosKernelTestExperimentData *>(campaignmanager.getDone()))) {
|
||||
// sanity check
|
||||
if ((!ECOS_FAULTMODEL_BURST && res->msg.result_size() != 8)
|
||||
|| (ECOS_FAULTMODEL_BURST && res->msg.result_size() != 1)) {
|
||||
m_log << "wtf, result_size = " << res->msg.result_size() << endl;
|
||||
continue;
|
||||
}
|
||||
|
||||
EcosKernelTestProtoMsg_Result const *prev_singleres = 0;
|
||||
int first_bit = 0, bit_width = 0;
|
||||
|
||||
#if !ECOS_FAULTMODEL_BURST
|
||||
// one job contains 8 experiments
|
||||
for (int idx = 0; idx < res->msg.result_size(); ++idx) {
|
||||
EcosKernelTestProtoMsg_Result const *cur_singleres = &res->msg.result(idx);
|
||||
if (!prev_singleres) {
|
||||
prev_singleres = cur_singleres;
|
||||
first_bit = cur_singleres->bit_offset();
|
||||
bit_width = 1;
|
||||
continue;
|
||||
}
|
||||
// compatible? merge.
|
||||
if (cur_singleres->bit_offset() == first_bit + bit_width // neighbor?
|
||||
&& prev_singleres->resulttype() == cur_singleres->resulttype()
|
||||
&& prev_singleres->latest_ip() == cur_singleres->latest_ip()
|
||||
&& prev_singleres->ecos_test_result() == cur_singleres->ecos_test_result()
|
||||
&& prev_singleres->error_corrected() == cur_singleres->error_corrected()
|
||||
&& prev_singleres->details() == cur_singleres->details()) {
|
||||
bit_width++;
|
||||
continue;
|
||||
}
|
||||
add_result(res->msg.pilot_id(),
|
||||
res->msg.instr2_offset(), res->msg.instr2_address(), res->msg.mem_addr(),
|
||||
first_bit, bit_width, prev_singleres->resulttype(), prev_singleres->ecos_test_result(),
|
||||
prev_singleres->latest_ip(), prev_singleres->error_corrected(), prev_singleres->details(),
|
||||
res->msg.runtime() * bit_width / 8.0);
|
||||
prev_singleres = cur_singleres;
|
||||
first_bit = cur_singleres->bit_offset();
|
||||
bit_width = 1;
|
||||
}
|
||||
#else
|
||||
// burst fault: bits 0-7, one experiment
|
||||
first_bit = 0;
|
||||
bit_width = 8;
|
||||
prev_singleres = &res->msg.result(0);
|
||||
#endif
|
||||
add_result(res->msg.pilot_id(),
|
||||
res->msg.instr2_offset(), res->msg.instr2_address(), res->msg.mem_addr(),
|
||||
first_bit, bit_width, prev_singleres->resulttype(), prev_singleres->ecos_test_result(),
|
||||
prev_singleres->latest_ip(), prev_singleres->error_corrected(), prev_singleres->details(),
|
||||
res->msg.runtime() * bit_width / 8.0);
|
||||
delete res;
|
||||
}
|
||||
EcosKernelTestExperimentData *data = new EcosKernelTestExperimentData;
|
||||
data->msg.mutable_fsppilot()->CopyFrom(pilot);
|
||||
fail::campaignmanager.addParam(data);
|
||||
}
|
||||
|
||||
@ -1,19 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <fstream>
|
||||
#include <mysql/mysql.h>
|
||||
|
||||
#ifndef __puma
|
||||
#include <boost/thread.hpp>
|
||||
#endif
|
||||
|
||||
#include "util/Database.hpp"
|
||||
#include "cpn/Campaign.hpp"
|
||||
#include "cpn/DatabaseCampaign.hpp"
|
||||
#include "comm/ExperimentData.hpp"
|
||||
#include "experimentInfo.hpp"
|
||||
#include "ecos_kernel_test.pb.h"
|
||||
#include "util/MemoryMap.hpp"
|
||||
#include "util/Logger.hpp"
|
||||
//#include <google/protobuf/descriptor.h>
|
||||
|
||||
class EcosKernelTestExperimentData : public fail::ExperimentData {
|
||||
public:
|
||||
@ -21,29 +12,20 @@ public:
|
||||
EcosKernelTestExperimentData() : fail::ExperimentData(&msg) {}
|
||||
};
|
||||
|
||||
class EcosKernelTestCampaign : public fail::Campaign {
|
||||
fail::Database *db;
|
||||
fail::Database *db_recv;
|
||||
int fspmethod_id;
|
||||
class EcosKernelTestCampaign : public fail::DatabaseCampaign {
|
||||
virtual const google::protobuf::Descriptor *cb_result_message()
|
||||
{
|
||||
return google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName("EcosKernelTestProtoMsg");
|
||||
}
|
||||
|
||||
static const std::string dir_images;
|
||||
static const std::string dir_prerequisites;
|
||||
std::string m_result_table;
|
||||
fail::Logger m_log;
|
||||
void add_result(unsigned pilot_id,
|
||||
int instr2, fail::address_t instr2_absolute, fail::address_t ec_data_address,
|
||||
int bitnr, int bit_width, int resulttype, int ecos_test_result, fail::address_t latest_ip,
|
||||
int error_corrected, const std::string& details, float runtime);
|
||||
void collect_results();
|
||||
public:
|
||||
EcosKernelTestCampaign() : m_log("EcosKernelTest Campaign") {}
|
||||
virtual bool run();
|
||||
static bool readMemoryMap(fail::MemoryMap &mm, char const * const filename);
|
||||
static bool writeTraceInfo(unsigned instr_counter, unsigned timeout, unsigned mem1_low, unsigned mem1_high, unsigned mem2_low, unsigned mem2_high, const std::string& variant = "", const std::string& benchmark = "");
|
||||
static bool readTraceInfo(unsigned &instr_counter, unsigned &timeout, unsigned &mem1_low, unsigned &mem1_high, unsigned &mem2_low, unsigned &mem2_high, const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_memorymap(const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_state(unsigned instr_offset, const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_trace(const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_traceinfo(const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_elf(const std::string& variant = "", const std::string& benchmark = "");
|
||||
virtual void cb_send_pilot(DatabaseCampaignMessage pilot);
|
||||
|
||||
virtual int expected_number_of_results(std::string variant, std::string benchmark)
|
||||
{
|
||||
#if ECOS_FAULTMODEL_BURST
|
||||
return 1;
|
||||
#else
|
||||
return 8;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@ -1,71 +1,48 @@
|
||||
import "DatabaseCampaignMessage.proto";
|
||||
|
||||
message EcosKernelTestProtoMsg {
|
||||
// Input: experiment parameters
|
||||
// (client executes 8 experiments, one for each bit at mem_addr)
|
||||
required DatabaseCampaignMessage fsppilot = 1;
|
||||
|
||||
// EDM/ERM variant
|
||||
required string variant = 1;
|
||||
// benchmark
|
||||
required string benchmark = 2;
|
||||
|
||||
// pilot ID (database)
|
||||
required uint32 pilot_id = 3;
|
||||
// FI at #instructions from experiment start
|
||||
required uint32 instr2_offset = 4;
|
||||
// the exact IP value at this point in time (from golden run)
|
||||
optional uint32 instr2_address = 5; // for sanity checks
|
||||
// address of the byte to inject bit-flips
|
||||
required uint32 mem_addr = 6;
|
||||
|
||||
// ----------------------------------------------------
|
||||
|
||||
// Output: experiment results
|
||||
|
||||
// IP where we did the injection: for debugging purposes, must be identical
|
||||
// to instr_address
|
||||
optional uint32 injection_ip = 7;
|
||||
|
||||
enum FaultModel {
|
||||
SINGLEBITFLIP = 1;
|
||||
BURST = 2;
|
||||
}
|
||||
optional FaultModel faultmodel = 10;
|
||||
|
||||
repeated group Result = 8 {
|
||||
repeated group Result = 2 {
|
||||
// single experiment bit offset
|
||||
required uint32 bit_offset = 1;
|
||||
required uint32 bit_offset = 1 [(sql_primary_key) = true];
|
||||
|
||||
// result type:
|
||||
// FINISHED = planned number of instructions were executed
|
||||
// TRAP = premature guest "crash"
|
||||
// OUTSIDE = IP left text segment
|
||||
enum ResultType {
|
||||
FINISHED = 1;
|
||||
TRAP = 2;
|
||||
OUTSIDE = 3;
|
||||
DETECTED = 4;
|
||||
TIMEOUT = 5;
|
||||
MEMORYACCESS = 6;
|
||||
UNKNOWN = 7;
|
||||
OK = 0;
|
||||
SDC = 1;
|
||||
DETECTED = 2;
|
||||
TRAP = 3;
|
||||
TIMEOUT = 4;
|
||||
UNKNOWN = 5;
|
||||
}
|
||||
required ResultType resulttype = 2;
|
||||
|
||||
// especially interesting for TRAP/UNKNOWN: latest IP
|
||||
required uint32 latest_ip = 3;
|
||||
|
||||
// test results
|
||||
enum EcosTestResultType {
|
||||
PASS = 1;
|
||||
FAIL = 2;
|
||||
// all subsequent fields are optional, resulting in NULLable DB
|
||||
// columns, saving space when unused; NULL should be interpreted as 0
|
||||
// for these cases
|
||||
enum MemAccessOutside {
|
||||
NONE = 0;
|
||||
READ = 1;
|
||||
WRITE = 2;
|
||||
}
|
||||
required EcosTestResultType ecos_test_result = 4;
|
||||
optional MemAccessOutside memaccess_outside = 3;
|
||||
|
||||
enum Flag {
|
||||
FALSE = 0;
|
||||
TRUE = 1;
|
||||
}
|
||||
optional Flag jump_outside = 4;
|
||||
|
||||
// did ECC correct the fault?
|
||||
optional int32 error_corrected = 6;
|
||||
optional Flag error_corrected = 5;
|
||||
|
||||
// especially interesting for TRAP/UNKNOWN: latest IP
|
||||
optional uint32 latest_ip = 6;
|
||||
|
||||
// optional textual description of what happened
|
||||
optional string details = 7;
|
||||
}
|
||||
|
||||
// aggregated runtime for this job (FIXME: this should be part of the basic campaign communication instead)
|
||||
optional float runtime = 9;
|
||||
// experiment runtime (FIXME: should be part of DatabaseCampaignMessage instead)
|
||||
optional float runtime = 8;
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,14 +6,12 @@
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
|
||||
#include "experiment.hpp"
|
||||
#include "experimentInfo.hpp"
|
||||
#include "campaign.hpp"
|
||||
#include "sal/SALConfig.hpp"
|
||||
#include "sal/SALInst.hpp"
|
||||
#include "sal/Memory.hpp"
|
||||
#include "sal/bochs/BochsListener.hpp"
|
||||
#include "sal/Listener.hpp"
|
||||
#include "util/ElfReader.hpp"
|
||||
#include "util/WallclockTimer.hpp"
|
||||
@ -24,6 +22,7 @@
|
||||
// You need to have the tracing plugin enabled for this
|
||||
#include "../plugins/tracing/TracingPlugin.hpp"
|
||||
|
||||
// for local experiment debugging: don't contact job server, use hard-coded parameters
|
||||
#define LOCAL 0
|
||||
|
||||
#ifndef PREREQUISITES
|
||||
@ -42,66 +41,116 @@
|
||||
using namespace std;
|
||||
using namespace fail;
|
||||
|
||||
#if PREREQUISITES
|
||||
bool EcosKernelTestExperiment::retrieveGuestAddresses(guest_address_t addr_finish, guest_address_t addr_data_start, guest_address_t addr_data_end) {
|
||||
#if BASELINE_ASSESSMENT || STACKPROTECTION
|
||||
log << "STEP 0: creating memory map spanning all of DATA and BSS" << endl;
|
||||
MemoryMap mm;
|
||||
mm.add(addr_data_start, addr_data_end - addr_data_start);
|
||||
mm.writeToFile(EcosKernelTestCampaign::filename_memorymap(m_variant, m_benchmark).c_str());
|
||||
#else
|
||||
log << "STEP 0: record memory map with addresses of 'interesting' objects" << endl;
|
||||
const std::string EcosKernelTestExperiment::dir_images(DIR_IMAGES);
|
||||
const std::string EcosKernelTestExperiment::dir_prerequisites(DIR_PREREQUISITES);
|
||||
|
||||
// run until func_finish is reached
|
||||
BPSingleListener bp;
|
||||
bp.setWatchInstructionPointer(addr_finish);
|
||||
bool EcosKernelTestExperiment::writeTraceInfo(unsigned instr_counter, unsigned timeout,
|
||||
unsigned mem1_low, unsigned mem1_high, // < 1M
|
||||
unsigned mem2_low, unsigned mem2_high, // >= 1M
|
||||
const std::string& variant, const std::string& benchmark) {
|
||||
ofstream ti(filename_traceinfo(variant, benchmark).c_str(), ios::out);
|
||||
if (!ti.is_open()) {
|
||||
cout << "failed to open " << filename_traceinfo(variant, benchmark) << endl;
|
||||
return false;
|
||||
}
|
||||
ti << instr_counter << endl << timeout << endl
|
||||
<< mem1_low << endl << mem1_high << endl
|
||||
<< mem2_low << endl << mem2_high << endl;
|
||||
ti.flush();
|
||||
ti.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
// memory map serialization
|
||||
// FIXME: use MemoryMap::writeToFile()
|
||||
ofstream mm(EcosKernelTestCampaign::filename_memorymap(m_variant, m_benchmark).c_str(), ios::out);
|
||||
if (!mm.is_open()) {
|
||||
log << "failed to open " << EcosKernelTestCampaign::filename_memorymap() << endl;
|
||||
bool EcosKernelTestExperiment::readTraceInfo(unsigned &instr_counter, unsigned &timeout,
|
||||
unsigned &mem1_low, unsigned &mem1_high, // < 1M
|
||||
unsigned &mem2_low, unsigned &mem2_high, // >= 1M
|
||||
const std::string& variant, const std::string& benchmark) {
|
||||
ifstream file(filename_traceinfo(variant, benchmark).c_str());
|
||||
if (!file.is_open()) {
|
||||
cout << "failed to open " << filename_traceinfo(variant, benchmark) << endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
GuestListener g;
|
||||
string *str = new string; // buffer for guest listeners' data
|
||||
unsigned number_of_guest_events = 0;
|
||||
string buf;
|
||||
unsigned count = 0;
|
||||
|
||||
while (simulator.addListenerAndResume(&g) == &g) {
|
||||
if (g.getData() == '\t') {
|
||||
// addr complete?
|
||||
//cout << "full: " << *str << "sub: " << str->substr(str->find_last_of('x') - 1) << endl;
|
||||
// interpret the string obtained by the guest listeners as address in hex
|
||||
unsigned guest_addr;
|
||||
stringstream converter(str->substr(str->find_last_of('x') + 1));
|
||||
converter >> hex >> guest_addr;
|
||||
mm << guest_addr << '\t';
|
||||
str->clear();
|
||||
} else if (g.getData() == '\n') {
|
||||
// len complete?
|
||||
// interpret the string obtained by the guest listeners as length in decimal
|
||||
unsigned guest_len;
|
||||
stringstream converter(*str);
|
||||
converter >> dec >> guest_len;
|
||||
mm << guest_len << '\n';
|
||||
str->clear();
|
||||
number_of_guest_events++;
|
||||
} else if (g.getData() == 'Q') {
|
||||
// when the guest system triggers the guest event 'Q',
|
||||
// we can assume that we are in protected mode
|
||||
simulator.addListener(&bp);
|
||||
} else {
|
||||
str->push_back(g.getData());
|
||||
while (getline(file, buf)) {
|
||||
stringstream ss(buf, ios::in);
|
||||
switch (count) {
|
||||
case 0:
|
||||
ss >> instr_counter;
|
||||
break;
|
||||
case 1:
|
||||
ss >> timeout;
|
||||
break;
|
||||
case 2:
|
||||
ss >> mem1_low;
|
||||
break;
|
||||
case 3:
|
||||
ss >> mem1_high;
|
||||
break;
|
||||
case 4:
|
||||
ss >> mem2_low;
|
||||
break;
|
||||
case 5:
|
||||
ss >> mem2_high;
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
assert(number_of_guest_events > 0);
|
||||
log << "Breakpoint at func_finish reached: created memory map (" << number_of_guest_events << " entries)" << endl;
|
||||
delete str;
|
||||
file.close();
|
||||
assert(count == 6);
|
||||
return (count == 6);
|
||||
}
|
||||
|
||||
// close serialized mm
|
||||
mm.close();
|
||||
#endif
|
||||
std::string EcosKernelTestExperiment::filename_memorymap(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "memorymap.txt";
|
||||
}
|
||||
return "memorymap.txt";
|
||||
}
|
||||
|
||||
std::string EcosKernelTestExperiment::filename_state(unsigned instr_offset, const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
stringstream ss;
|
||||
ss << instr_offset;
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "state" + "-" + ss.str();
|
||||
}
|
||||
return "state-" + ss.str();
|
||||
}
|
||||
|
||||
std::string EcosKernelTestExperiment::filename_trace(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "trace.tc";
|
||||
}
|
||||
return "trace.tc";
|
||||
}
|
||||
|
||||
std::string EcosKernelTestExperiment::filename_traceinfo(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_prerequisites + "/" + variant + "-" + benchmark + "-" + "traceinfo.txt";
|
||||
}
|
||||
return "traceinfo.txt";
|
||||
}
|
||||
|
||||
std::string EcosKernelTestExperiment::filename_elf(const std::string& variant, const std::string& benchmark)
|
||||
{
|
||||
if (variant.size() && benchmark.size()) {
|
||||
return dir_images + "/" + variant + "/" + benchmark + ".elf";
|
||||
}
|
||||
return "kernel.elf";
|
||||
}
|
||||
|
||||
#if PREREQUISITES
|
||||
bool EcosKernelTestExperiment::retrieveGuestAddresses(guest_address_t addr_finish, guest_address_t addr_data_start, guest_address_t addr_data_end) {
|
||||
log << "STEP 0: creating memory map spanning all of DATA and BSS" << endl;
|
||||
MemoryMap mm;
|
||||
mm.add(addr_data_start, addr_data_end - addr_data_start);
|
||||
mm.writeToFile(filename_memorymap(m_variant, m_benchmark).c_str());
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -137,9 +186,13 @@ bool EcosKernelTestExperiment::establishState(guest_address_t addr_entry, guest_
|
||||
|
||||
for (unsigned i = 0; ; ++i) {
|
||||
log << "saving state at offset " << dec << (i * MULTIPLE_SNAPSHOTS_DISTANCE) << endl;
|
||||
simulator.save(EcosKernelTestCampaign::filename_state(i * MULTIPLE_SNAPSHOTS_DISTANCE, m_variant, m_benchmark));
|
||||
if (!simulator.save(filename_state(i * MULTIPLE_SNAPSHOTS_DISTANCE, m_variant, m_benchmark))) {
|
||||
log << "state save failed!" << endl;
|
||||
simulator.terminate(1);
|
||||
}
|
||||
|
||||
#if MULTIPLE_SNAPSHOTS
|
||||
simulator.restore(EcosKernelTestCampaign::filename_state(i * MULTIPLE_SNAPSHOTS_DISTANCE, m_variant, m_benchmark));
|
||||
simulator.restore(filename_state(i * MULTIPLE_SNAPSHOTS_DISTANCE, m_variant, m_benchmark));
|
||||
|
||||
simulator.addListener(&step);
|
||||
simulator.addListener(&finish);
|
||||
@ -159,7 +212,7 @@ bool EcosKernelTestExperiment::performTrace(guest_address_t addr_entry, guest_ad
|
||||
log << "STEP 2: record trace for fault-space pruning" << endl;
|
||||
|
||||
log << "restoring state" << endl;
|
||||
simulator.restore(EcosKernelTestCampaign::filename_state(0, m_variant, m_benchmark));
|
||||
simulator.restore(filename_state(0, m_variant, m_benchmark));
|
||||
log << "EIP = " << hex << simulator.getCPU(0).getInstructionPointer() << endl;
|
||||
assert(simulator.getCPU(0).getInstructionPointer() == addr_entry);
|
||||
|
||||
@ -168,12 +221,15 @@ bool EcosKernelTestExperiment::performTrace(guest_address_t addr_entry, guest_ad
|
||||
|
||||
// restrict memory access logging to injection target
|
||||
MemoryMap mm;
|
||||
mm.readFromFile(EcosKernelTestCampaign::filename_memorymap(m_variant, m_benchmark).c_str());
|
||||
mm.readFromFile(filename_memorymap(m_variant, m_benchmark).c_str());
|
||||
|
||||
tp.restrictMemoryAddresses(&mm);
|
||||
#if RECORD_FULL_TRACE
|
||||
tp.setFullTrace(true);
|
||||
#endif
|
||||
|
||||
// record trace
|
||||
ogzstream of(EcosKernelTestCampaign::filename_trace(m_variant, m_benchmark).c_str());
|
||||
ogzstream of(filename_trace(m_variant, m_benchmark).c_str());
|
||||
tp.setTraceFile(&of);
|
||||
// this must be done *after* configuring the plugin:
|
||||
simulator.addFlow(&tp);
|
||||
@ -210,8 +266,7 @@ bool EcosKernelTestExperiment::performTrace(guest_address_t addr_entry, guest_ad
|
||||
return false;
|
||||
}
|
||||
simulator.addListener(&ev_count);
|
||||
}
|
||||
else if (ev == &ev_mem) {
|
||||
} else if (ev == &ev_mem) {
|
||||
unsigned lo = ev_mem.getTriggerAddress();
|
||||
unsigned hi = lo + ev_mem.getTriggerWidth() - 1;
|
||||
|
||||
@ -229,10 +284,11 @@ bool EcosKernelTestExperiment::performTrace(guest_address_t addr_entry, guest_ad
|
||||
ev = simulator.resume();
|
||||
}
|
||||
|
||||
unsigned long long estimated_timeout_overflow_check =
|
||||
simulator.getTimerTicks() - time_start + 55000; // 1s/18.2
|
||||
unsigned estimated_timeout =
|
||||
(unsigned) (estimated_timeout_overflow_check * 1000000 / simulator.getTimerTicksPerSecond());
|
||||
unsigned long long estimated_timeout_ticks =
|
||||
simulator.getTimerTicks() - time_start + simulator.getTimerTicksPerSecond() / 18.2; // 1s/18.2
|
||||
// convert to microseconds
|
||||
unsigned estimated_timeout = (unsigned)
|
||||
(estimated_timeout_ticks * 1000000 / simulator.getTimerTicksPerSecond());
|
||||
|
||||
log << dec << "tracing finished after " << instr_counter << " instructions" << endl;
|
||||
log << hex << "all memory accesses within [0x" << mem1_low << ", 0x" << mem1_high << "] u [0x" << mem2_low << ", 0x" << mem2_high << "] (ignoring VGA mem)" << endl;
|
||||
@ -247,18 +303,18 @@ bool EcosKernelTestExperiment::performTrace(guest_address_t addr_entry, guest_ad
|
||||
}
|
||||
|
||||
// save these values for experiment STEP 3
|
||||
EcosKernelTestCampaign::writeTraceInfo(instr_counter, estimated_timeout,
|
||||
writeTraceInfo(instr_counter, estimated_timeout,
|
||||
mem1_low, mem1_high, mem2_low, mem2_high, m_variant, m_benchmark);
|
||||
|
||||
simulator.removeFlow(&tp);
|
||||
|
||||
// serialize trace to file
|
||||
if (of.fail()) {
|
||||
log << "failed to write " << EcosKernelTestCampaign::filename_trace(m_variant, m_benchmark) << endl;
|
||||
log << "failed to write " << filename_trace(m_variant, m_benchmark) << endl;
|
||||
return false;
|
||||
}
|
||||
of.close();
|
||||
log << "trace written to " << EcosKernelTestCampaign::filename_trace(m_variant, m_benchmark) << endl;
|
||||
log << "trace written to " << filename_trace(m_variant, m_benchmark) << endl;
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -268,7 +324,11 @@ void EcosKernelTestExperiment::handle_func_test_output(bool &test_failed, bool&
|
||||
{
|
||||
// 1st argument of cyg_test_output shows what has happened (FAIL or PASS)
|
||||
address_t stack_ptr = simulator.getCPU(0).getStackPointer(); // esp
|
||||
int32_t cyg_test_output_argument = simulator.getMemoryManager().getByte(stack_ptr + 4); // 1st argument is at esp+4
|
||||
MemoryManager& mm = simulator.getMemoryManager();
|
||||
if (!mm.isMapped(stack_ptr + 4)) {
|
||||
return;
|
||||
}
|
||||
int32_t cyg_test_output_argument = mm.getByte(stack_ptr + 4); // 1st argument is at esp+4
|
||||
|
||||
log << "cyg_test_output_argument (#1): " << cyg_test_output_argument << endl;
|
||||
|
||||
@ -294,7 +354,7 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
log << "STEP 3: The actual experiment." << endl;
|
||||
|
||||
// trace info
|
||||
unsigned instr_counter, estimated_timeout, mem1_low, mem1_high, mem2_low, mem2_high;
|
||||
unsigned goldenrun_instr_counter, estimated_timeout, mem1_low, mem1_high, mem2_low, mem2_high;
|
||||
// ELF symbol addresses
|
||||
guest_address_t addr_entry, addr_finish, addr_test_output, addr_errors_corrected,
|
||||
addr_panic, addr_text_start, addr_text_end,
|
||||
@ -304,9 +364,11 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
|
||||
int experiments = 0;
|
||||
#if !LOCAL
|
||||
for (experiments = 0;
|
||||
experiments < 500 || (m_jc.getNumberOfUndoneJobs() != 0); ) { // stop after ~500 experiments to prevent swapping
|
||||
// TODO: measure these numbers again!
|
||||
// stop after ~500 experiments to prevent swapping
|
||||
// 50 exp ~ 0.5GB RAM usage per instance (linearly increasing)
|
||||
for (experiments = 0;
|
||||
experiments < 500 || (m_jc.getNumberOfUndoneJobs() != 0); ) {
|
||||
#endif
|
||||
|
||||
// get an experiment parameter set
|
||||
@ -320,23 +382,26 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
}
|
||||
#else
|
||||
// XXX debug
|
||||
param.msg.set_variant(m_variant);
|
||||
param.msg.set_benchmark(m_benchmark);
|
||||
param.msg.set_instr2_offset(7462);
|
||||
//param.msg.set_instr_address(12345);
|
||||
param.msg.set_mem_addr(44540);
|
||||
param.msg.mutable_fsppilot()->set_injection_instr(7462);
|
||||
//param.msg.mutable_fsppilot()->set_injection_instr_absolute(12345);
|
||||
param.msg.mutable_fsppilot()->set_data_address(44540);
|
||||
param.msg.mutable_fsppilot()->set_data_width(1);
|
||||
param.msg.mutable_fsppilot()->set_variant(m_variant);
|
||||
param.msg.mutable_fsppilot()->set_benchmark(m_benchmark);
|
||||
#endif
|
||||
|
||||
WallclockTimer timer;
|
||||
timer.startTimer();
|
||||
if (param.msg.fsppilot().data_width() != 1) {
|
||||
log << "cannot deal with data_width = " << param.msg.fsppilot().data_width() << endl;
|
||||
simulator.terminate(1);
|
||||
}
|
||||
|
||||
int id = param.getWorkloadID();
|
||||
m_variant = param.msg.variant();
|
||||
m_benchmark = param.msg.benchmark();
|
||||
int instr_offset = param.msg.instr2_offset();
|
||||
int mem_addr = param.msg.mem_addr();
|
||||
m_variant = param.msg.fsppilot().variant();
|
||||
m_benchmark = param.msg.fsppilot().benchmark();
|
||||
unsigned instr_offset = param.msg.fsppilot().injection_instr();
|
||||
unsigned mem_addr = param.msg.fsppilot().data_address();
|
||||
|
||||
EcosKernelTestCampaign::readTraceInfo(instr_counter, estimated_timeout,
|
||||
readTraceInfo(goldenrun_instr_counter, estimated_timeout,
|
||||
mem1_low, mem1_high, mem2_low, mem2_high, m_variant, m_benchmark);
|
||||
readELFSymbols(addr_entry, addr_finish, addr_test_output,
|
||||
addr_errors_corrected, addr_panic, addr_text_start, addr_text_end,
|
||||
@ -345,13 +410,13 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
int state_instr_offset = instr_offset - (instr_offset % MULTIPLE_SNAPSHOTS_DISTANCE);
|
||||
string statename;
|
||||
#if MULTIPLE_SNAPSHOTS
|
||||
if (access(EcosKernelTestCampaign::filename_state(state_instr_offset, m_variant, m_benchmark).c_str(), R_OK) == 0) {
|
||||
statename = EcosKernelTestCampaign::filename_state(state_instr_offset, m_variant, m_benchmark);
|
||||
if (access(filename_state(state_instr_offset, m_variant, m_benchmark).c_str(), R_OK) == 0) {
|
||||
statename = filename_state(state_instr_offset, m_variant, m_benchmark);
|
||||
log << "using state at offset " << state_instr_offset << endl;
|
||||
instr_offset -= state_instr_offset;
|
||||
} else { // fallback
|
||||
#endif
|
||||
statename = EcosKernelTestCampaign::filename_state(0, m_variant, m_benchmark);
|
||||
statename = filename_state(0, m_variant, m_benchmark);
|
||||
state_instr_offset = 0;
|
||||
log << "using state at offset 0 (fallback)" << endl;
|
||||
#if MULTIPLE_SNAPSHOTS
|
||||
@ -363,6 +428,11 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
for (int bit_offset = 0; bit_offset < 8; ++bit_offset) {
|
||||
++experiments;
|
||||
|
||||
// TODO timing measurement should be part of the
|
||||
// DatabaseCampaignMessage
|
||||
WallclockTimer timer;
|
||||
timer.startTimer();
|
||||
|
||||
// 8 results in one job
|
||||
EcosKernelTestProtoMsg_Result *result = param.msg.add_result();
|
||||
result->set_bit_offset(bit_offset);
|
||||
@ -373,15 +443,6 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
log << "restoring state" << endl;
|
||||
simulator.restore(statename);
|
||||
|
||||
// XXX debug
|
||||
/*
|
||||
stringstream fname;
|
||||
fname << "job." << ::getpid();
|
||||
ofstream job(fname.str().c_str());
|
||||
job << "job " << id << " instr " << instr_offset << " (" << param.msg.instr_address() << ") mem " << mem_addr << "+" << bit_offset << endl;
|
||||
job.close();
|
||||
*/
|
||||
|
||||
// the outcome of ecos' test case
|
||||
bool ecos_test_passed = false;
|
||||
bool ecos_test_failed = false;
|
||||
@ -400,7 +461,6 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
|
||||
// no need to wait if offset is 0
|
||||
if (instr_offset > 0) {
|
||||
// XXX could be improved with intermediate states (reducing runtime until injection)
|
||||
bp.setWatchInstructionPointer(ANY_ADDR);
|
||||
bp.setCounter(instr_offset);
|
||||
simulator.addListener(&bp);
|
||||
@ -425,38 +485,35 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
MemoryManager& mm = simulator.getMemoryManager();
|
||||
byte_t data = mm.getByte(mem_addr);
|
||||
byte_t newdata;
|
||||
if (param.msg.has_faultmodel() && param.msg.faultmodel() == param.msg.BURST) {
|
||||
newdata = data ^ 0xff;
|
||||
bit_offset = 8; // enforce loop termination
|
||||
} else if (!param.msg.has_faultmodel() || param.msg.faultmodel() == param.msg.SINGLEBITFLIP) {
|
||||
newdata = data ^ (1 << bit_offset);
|
||||
} else {
|
||||
// Won't happen with current campaign implementation. Keeps
|
||||
// compiler happy.
|
||||
newdata = data;
|
||||
}
|
||||
#if ECOS_FAULTMODEL_BURST
|
||||
newdata = data ^ 0xff;
|
||||
bit_offset = 8; // enforce loop termination
|
||||
#else
|
||||
newdata = data ^ (1 << bit_offset);
|
||||
#endif
|
||||
mm.setByte(mem_addr, newdata);
|
||||
// note at what IP we did it
|
||||
int32_t injection_ip = simulator.getCPU(0).getInstructionPointer();
|
||||
param.msg.set_injection_ip(injection_ip);
|
||||
uint32_t injection_ip = simulator.getCPU(0).getInstructionPointer();
|
||||
log << "fault injected @ ip " << injection_ip
|
||||
<< " 0x" << hex << ((int)data) << " -> 0x" << ((int)newdata) << endl;
|
||||
// sanity check
|
||||
if (param.msg.has_instr2_address() &&
|
||||
injection_ip != param.msg.instr2_address()) {
|
||||
if (param.msg.fsppilot().has_injection_instr_absolute() &&
|
||||
injection_ip != param.msg.fsppilot().injection_instr_absolute()) {
|
||||
stringstream ss;
|
||||
ss << "SANITY CHECK FAILED: " << injection_ip
|
||||
<< " != " << param.msg.instr2_address();
|
||||
<< " != " << param.msg.fsppilot().injection_instr_absolute();
|
||||
log << ss.str() << endl;
|
||||
result->set_resulttype(result->UNKNOWN);
|
||||
result->set_latest_ip(injection_ip);
|
||||
result->set_ecos_test_result(result->FAIL);
|
||||
result->set_details(ss.str());
|
||||
result->set_runtime(timer);
|
||||
|
||||
continue;
|
||||
}
|
||||
if (param.msg.has_instr2_address()) {
|
||||
if (param.msg.fsppilot().has_injection_instr_absolute()) {
|
||||
log << "Absolute IP sanity check OK" << endl;
|
||||
} else {
|
||||
log << "Absolute IP sanity check skipped (job parameters insufficient)" << endl;
|
||||
}
|
||||
|
||||
// --- aftermath ---
|
||||
@ -506,11 +563,12 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
TimerListener ev_timeout(estimated_timeout);
|
||||
simulator.addListener(&ev_timeout);
|
||||
|
||||
// remaining instructions until "normal" ending
|
||||
// number of instructions that are executed additionally for error corrections
|
||||
//BPSingleListener ev_end(ANY_ADDR);
|
||||
//ev_end.setCounter(instr_counter - instr_offset + ECOS_RECOVERYINSTR);
|
||||
//simulator.addListener(&ev_end);
|
||||
// grant generous (10x) more instructions before aborting to avoid false positives
|
||||
BPSingleListener ev_dyninstructions(ANY_ADDR);
|
||||
//ev_dyninstructions.setCounter((goldenrun_instr_counter - param.msg.fsppilot().injection_instr()) * 10);
|
||||
// FIXME overflow possible
|
||||
ev_dyninstructions.setCounter(goldenrun_instr_counter * 10);
|
||||
simulator.addListener(&ev_dyninstructions);
|
||||
|
||||
// function called by ecc aspects, when an uncorrectable error is detected
|
||||
BPSingleListener func_ecc_panic(addr_panic);
|
||||
@ -535,7 +593,24 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
// re-add this listener
|
||||
simulator.addListener(&func_test_output);
|
||||
handle_func_test_output(ecos_test_failed, ecos_test_passed);
|
||||
|
||||
} else if (ev == &ev_below_text || ev == &ev_beyond_text) {
|
||||
result->set_jump_outside(result->TRUE);
|
||||
// no need to re-add the affected listener
|
||||
} else if (ev == &ev_mem_outside1 || ev == &ev_mem_outside2
|
||||
|| ev == &ev_mem_outside3 || ev == &ev_mem_outside4) {
|
||||
MemAccessListener *mev = dynamic_cast<MemAccessListener *>(ev);
|
||||
if (mev->getTriggerAccessType() == MemAccessEvent::MEM_READ) {
|
||||
result->set_memaccess_outside(result->READ);
|
||||
// re-add this listener, may report a write later on
|
||||
simulator.addListener(mev);
|
||||
} else { // write
|
||||
result->set_memaccess_outside(result->WRITE);
|
||||
// remove all listeners to avoid downgrade to READ
|
||||
simulator.removeListener(&ev_mem_outside1);
|
||||
simulator.removeListener(&ev_mem_outside2);
|
||||
simulator.removeListener(&ev_mem_outside3);
|
||||
simulator.removeListener(&ev_mem_outside4);
|
||||
}
|
||||
// special case: except1 and clockcnv actively generate traps
|
||||
} else if (ev == &ev_trap
|
||||
&& ((m_benchmark == "except1" && ev_trap.getTriggerNumber() == 13)
|
||||
@ -549,39 +624,34 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
}
|
||||
|
||||
// record latest IP regardless of result
|
||||
// TODO: consider recording latest IP within text segment, too, which
|
||||
// would make this usable for the jump-outside case
|
||||
result->set_latest_ip(simulator.getCPU(0).getInstructionPointer());
|
||||
|
||||
// record error_corrected regardless of result
|
||||
if (addr_errors_corrected != ADDR_INV) {
|
||||
int32_t error_corrected = simulator.getMemoryManager().getByte(addr_errors_corrected);
|
||||
result->set_error_corrected(error_corrected);
|
||||
int32_t error_corrected = mm.getByte(addr_errors_corrected);
|
||||
result->set_error_corrected(error_corrected ? result->TRUE : result->FALSE);
|
||||
} else {
|
||||
result->set_error_corrected(0);
|
||||
// not setting this yields NULL in the DB
|
||||
//result->set_error_corrected(0);
|
||||
}
|
||||
|
||||
// record ecos_test_result
|
||||
if ( (ecos_test_passed == true) && (ecos_test_failed == false) ) {
|
||||
result->set_ecos_test_result(result->PASS);
|
||||
log << "Ecos Test PASS" << endl;
|
||||
} else {
|
||||
result->set_ecos_test_result(result->FAIL);
|
||||
log << "Ecos Test FAIL" << endl;
|
||||
}
|
||||
bool ecos_test_success = ecos_test_passed && !ecos_test_failed;
|
||||
log << "Ecos Test " << (ecos_test_success ? "PASS" : "FAIL") << endl;
|
||||
|
||||
if (ev == &func_finish) {
|
||||
if (ev == &func_finish && ecos_test_success) {
|
||||
// do we reach finish?
|
||||
log << "experiment finished ordinarily" << endl;
|
||||
result->set_resulttype(result->FINISHED);
|
||||
} else if (ev == &ev_timeout /*|| ev == &ev_end*/) {
|
||||
log << "Result TIMEOUT" << endl;
|
||||
result->set_resulttype(result->TIMEOUT);
|
||||
} else if (ev == &ev_below_text || ev == &ev_beyond_text) {
|
||||
log << "Result OUTSIDE" << endl;
|
||||
result->set_resulttype(result->OUTSIDE);
|
||||
} else if (ev == &ev_mem_outside1 || ev == &ev_mem_outside2
|
||||
|| ev == &ev_mem_outside3 || ev == &ev_mem_outside4) {
|
||||
log << "Result MEMORYACCESS" << endl;
|
||||
result->set_resulttype(result->MEMORYACCESS);
|
||||
result->set_resulttype(result->OK);
|
||||
} else if (ev == &func_finish && !ecos_test_success) {
|
||||
// do we reach finish?
|
||||
log << "experiment finished, but ecos test failed" << endl;
|
||||
result->set_resulttype(result->SDC);
|
||||
} else if (ev == &func_ecc_panic) {
|
||||
log << "ECC Panic: uncorrectable error" << endl;
|
||||
result->set_resulttype(result->DETECTED); // DETECTED <=> ECC_PANIC <=> reboot
|
||||
} else if (ev == &ev_trap) {
|
||||
log << dec << "Result TRAP #" << ev_trap.getTriggerNumber() << endl;
|
||||
result->set_resulttype(result->TRAP);
|
||||
@ -589,9 +659,12 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
stringstream ss;
|
||||
ss << ev_trap.getTriggerNumber();
|
||||
result->set_details(ss.str());
|
||||
} else if (ev == &func_ecc_panic) {
|
||||
log << "ECC Panic: uncorrectable error" << endl;
|
||||
result->set_resulttype(result->DETECTED); // DETECTED <=> ECC_PANIC <=> reboot
|
||||
} else if (ev == &ev_timeout || ev == &ev_dyninstructions) {
|
||||
log << "Result TIMEOUT" << endl;
|
||||
result->set_resulttype(result->TIMEOUT);
|
||||
if (ev == &ev_dyninstructions) {
|
||||
result->set_details("i");
|
||||
}
|
||||
} else {
|
||||
log << "Result WTF?" << endl;
|
||||
result->set_resulttype(result->UNKNOWN);
|
||||
@ -600,17 +673,13 @@ bool EcosKernelTestExperiment::faultInjection() {
|
||||
ss << "event addr " << ev << " EIP " << simulator.getCPU(0).getInstructionPointer();
|
||||
result->set_details(ss.str());
|
||||
}
|
||||
|
||||
result->set_runtime(timer);
|
||||
}
|
||||
// sanity check: do we have exactly 8 results?
|
||||
if ((!param.msg.has_faultmodel() || param.msg.faultmodel() == param.msg.SINGLEBITFLIP)
|
||||
&& param.msg.result_size() != 8) {
|
||||
log << "WTF? param.msg.result_size() != 8" << endl;
|
||||
} else {
|
||||
param.msg.set_runtime(timer);
|
||||
|
||||
#if !LOCAL
|
||||
m_jc.sendResult(param);
|
||||
m_jc.sendResult(param);
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !LOCAL
|
||||
}
|
||||
@ -630,7 +699,7 @@ bool EcosKernelTestExperiment::readELFSymbols(
|
||||
fail::guest_address_t& data_start,
|
||||
fail::guest_address_t& data_end)
|
||||
{
|
||||
ElfReader elfreader(EcosKernelTestCampaign::filename_elf(m_variant, m_benchmark).c_str());
|
||||
ElfReader elfreader(filename_elf(m_variant, m_benchmark).c_str());
|
||||
entry = elfreader.getSymbol("cyg_start").getAddress();
|
||||
finish = elfreader.getSymbol("cyg_test_exit").getAddress();
|
||||
test_output = elfreader.getSymbol("cyg_test_output").getAddress();
|
||||
@ -698,8 +767,7 @@ bool EcosKernelTestExperiment::run()
|
||||
|
||||
// step 0
|
||||
if (retrieveGuestAddresses(finish, data_start, data_end)) {
|
||||
log << "STEP 0 finished: rebooting ..." << endl;
|
||||
simulator.reboot();
|
||||
log << "STEP 0 finished: proceeding ..." << endl;
|
||||
} else { return false; }
|
||||
|
||||
// step 1
|
||||
|
||||
@ -11,6 +11,9 @@ class EcosKernelTestExperiment : public fail::ExperimentFlow {
|
||||
fail::JobClient m_jc;
|
||||
fail::Logger log;
|
||||
std::string m_variant, m_benchmark;
|
||||
|
||||
static const std::string dir_images;
|
||||
static const std::string dir_prerequisites;
|
||||
public:
|
||||
EcosKernelTestExperiment() : log("eCos Kernel Test", false) {}
|
||||
bool run();
|
||||
@ -33,4 +36,12 @@ public:
|
||||
fail::guest_address_t& data_end);
|
||||
|
||||
void handle_func_test_output(bool &test_failed, bool& test_passed);
|
||||
|
||||
static bool writeTraceInfo(unsigned instr_counter, unsigned timeout, unsigned mem1_low, unsigned mem1_high, unsigned mem2_low, unsigned mem2_high, const std::string& variant = "", const std::string& benchmark = "");
|
||||
static bool readTraceInfo(unsigned &instr_counter, unsigned &timeout, unsigned &mem1_low, unsigned &mem1_high, unsigned &mem2_low, unsigned &mem2_high, const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_memorymap(const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_state(unsigned instr_offset, const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_trace(const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_traceinfo(const std::string& variant = "", const std::string& benchmark = "");
|
||||
static std::string filename_elf(const std::string& variant = "", const std::string& benchmark = "");
|
||||
};
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#define BASELINE_ASSESSMENT 0
|
||||
#define STACKPROTECTION 1
|
||||
#define DIR_IMAGES "images"
|
||||
#define DIR_PREREQUISITES "prerequisites"
|
||||
#define PREREQUISITES 0
|
||||
#define ECOS_FAULTMODEL_BURST 1
|
||||
#define ECOS_FAULTMODEL_BURST 0
|
||||
#define RECORD_FULL_TRACE 0
|
||||
|
||||
Reference in New Issue
Block a user