christoph/fail
1
Fork 0
Code Actions Packages Releases Activity

weather-monitor: now is a DatabaseCampaign

Browse Source 
"removed" unneccessary memory-mapping ("Step 0")
cleaned out ExperimentData - now consists only of fsppilot and resultset
resultset now contains bitoffset which is part of result-table's primary key
adapted code to work with msg.fsppilot() instead of ExperimentData-values

Change-Id: I3b310e7a71d4b28479028250cd5722b3b2ce9f8c
This commit is contained in:
Michael Lenz
2013-12-11 14:38:01 +01:00
parent 839913592a
commit 0907dfb0ae
5 changed files with 54 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/experiments/weather-monitor/CMakeLists.txt
+1 -1
View File
@@ -31,5 +31,5 @@ target_link_libraries(fail-${EXPERIMENT_NAME} ${PROTOBUF_LIBRARY})
## This is the example's campaign server distributing experiment parameters
add_executable(${EXPERIMENT_NAME}-server main.cc)
target_link_libraries(${EXPERIMENT_NAME}-server fail-${EXPERIMENT_NAME} fail ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY})
target_link_libraries(${EXPERIMENT_NAME}-server fail-${EXPERIMENT_NAME} fail fail-cpn fail-util -lmysqlclient ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY})
install(TARGETS ${EXPERIMENT_NAME}-server RUNTIME DESTINATION bin)
src/experiments/weather-monitor/campaign.cc
+6 -419
View File
@@ -1,18 +1,11 @@
#include <iostream>
#include <fstream>
#include <vector>
#include <map>
#include <boost/timer.hpp>
#include "campaign.hpp"
#include "experimentInfo.hpp"
#include "comm/DatabaseCampaignMessage.pb.h"
#include "cpn/CampaignManager.hpp"
#include "util/Logger.hpp"
#include "util/MemoryMap.hpp"
#include "util/ProtoStream.hpp"
#include "vptr_map.hpp"
#include "../plugins/tracing/TracingPlugin.hpp"
@@ -21,417 +14,11 @@
using namespace std;
using namespace fail;
char const * const trace_filename = "trace.tc" WEATHER_SUFFIX;
char const * const results_filename = "weathermonitor" WEATHER_SUFFIX ".csv";
using namespace google::protobuf;
// equivalence class type: addr, [i1, i2]
// addr: byte to inject a bit-flip into
// [i1, i2]: interval of instruction numbers, counted from experiment
// begin
struct equivalence_class {
address_t data_address;
int instr1, instr2;
address_t instr2_absolute;
};
bool WeatherMonitorCampaign::run()
void WeatherMonitorCampaign::cb_send_pilot(DatabaseCampaignMessage pilot)
{
Logger log("Weathermonitor Campaign");
// non-destructive: due to the CSV header we can always manually recover
// from an accident (append mode)
ofstream results(results_filename, ios::out | ios::app);
if (!results.is_open()) {
log << "failed to open " << results_filename << endl;
return false;
}
log << "startup" << endl;
boost::timer t;
// load trace
ifstream tracef(trace_filename);
if (tracef.fail()) {
log << "couldn't open " << trace_filename << endl;
return false;
}
ProtoIStream ps(&tracef);
// a map of FI data addresses
MemoryMap mm;
mm.add(WEATHER_DATA_START, WEATHER_DATA_END - WEATHER_DATA_START);
// set of equivalence classes that need one (rather: eight, one for
// each bit in that byte) experiment to determine them all
vector<equivalence_class> ecs_need_experiment;
// set of equivalence classes that need no experiment, because we know
// they'd be identical to the golden run
vector<equivalence_class> ecs_no_effect;
#if 0
equivalence_class current_ec;
// map for efficient access when results come in
map<WeatherMonitorExperimentData *, unsigned> experiment_ecs;
// experiment count
int count = 0;
// XXX do it the other way around: iterate over trace, search addresses
// -> one "open" EC for every address
// for every injection address ...
for (MemoryMap::iterator it = mm.begin(); it != mm.end(); ++it) {
//cerr << ".";
address_t data_address = *it;
current_ec.instr1 = 0;
int instr = 0;
address_t instr_absolute = 0; // FIXME this one probably should also be recorded ...
Trace_Event ev;
ps.reset();
// for every section in the trace between subsequent memory
// accesses to that address ...
while (ps.getNext(&ev) && instr < WEATHER_NUMINSTR_TRACING) {
// instruction events just get counted
if (!ev.has_memaddr()) {
// new instruction
instr++;
instr_absolute = ev.ip();
continue;
// skip accesses to other data
// FIXME again, do it the other way around, and use mm.isMatching()!
} else if (ev.memaddr() + ev.width() <= data_address
|| ev.memaddr() > data_address) {
continue;
// skip zero-sized intervals: these can
// occur when an instruction accesses a
// memory location more than once
// (e.g., INC, CMPXCHG)
} else if (current_ec.instr1 > instr) {
continue;
}
// we now have an interval-terminating R/W
// event to the memaddr we're currently looking
// at:
// complete the equivalence interval
current_ec.instr2 = instr;
current_ec.instr2_absolute = instr_absolute;
current_ec.data_address = data_address;
if (ev.accesstype() == ev.READ) {
// a sequence ending with READ: we need
// to do one experiment to cover it
// completely
ecs_need_experiment.push_back(current_ec);
#ifdef PRUNING_DEBUG_OUTPUT
cerr << dec << "EX " << current_ec.instr1 << " " << current_ec.instr2 << " " << current_ec.data_address << "\n";
#endif
// instantly enqueue job: that way the job clients can already
// start working in parallel
WeatherMonitorExperimentData *d = new WeatherMonitorExperimentData;
// we pick the rightmost instruction in that interval
d->msg.set_instr_offset(current_ec.instr2);
d->msg.set_instr_address(current_ec.instr2_absolute);
d->msg.set_mem_addr(current_ec.data_address);
// store index into ecs_need_experiment
experiment_ecs[d] = ecs_need_experiment.size() - 1;
campaignmanager.addParam(d);
++count;
} else if (ev.accesstype() == ev.WRITE) {
// a sequence ending with WRITE: an
// injection anywhere here would have
// no effect.
ecs_no_effect.push_back(current_ec);
#ifdef PRUNING_DEBUG_OUTPUT
cerr << dec << "NE " << current_ec.instr1 << " " << current_ec.instr2 << " " << current_ec.data_address << "\n";
#endif
} else {
log << "WAT" << endl;
}
// next interval must start at next
// instruction; the aforementioned
// skipping mechanism wouldn't work
// otherwise
current_ec.instr1 = instr + 1;
}
// close the last interval:
// Why -1? In most cases it does not make sense to inject before the
// very last instruction, as we won't execute it anymore. This *only*
// makes sense if we also inject into parts of the result vector. This
// is not the case in this experiment, and with -1 we'll get a
// result comparable to the non-pruned campaign.
// XXX still true for weathermonitor?
current_ec.instr2 = instr - 1;
current_ec.instr2_absolute = 0; // unknown
current_ec.data_address = data_address;
// zero-sized? skip.
if (current_ec.instr1 > current_ec.instr2) {
continue;
}
// the run continues after the FI window, so do this experiment
// XXX this creates at least one experiment for *every* bit!
// fix: full trace, limited FI window
//ecs_no_effect.push_back(current_ec);
ecs_need_experiment.push_back(current_ec);
#ifdef PRUNING_DEBUG_OUTPUT
cerr << dec << "EX " << current_ec.instr1 << " " << current_ec.instr2 << " " << current_ec.data_address << "\n";
#endif
// FIXME copy/paste, encapsulate this:
// instantly enqueue job: that way the job clients can already
// start working in parallel
WeatherMonitorExperimentData *d = new WeatherMonitorExperimentData;
// we pick the rightmost instruction in that interval
d->msg.set_instr_offset(current_ec.instr2);
//d->msg.set_instr_address(current_ec.instr2_absolute); // unknown!
d->msg.set_mem_addr(current_ec.data_address);
// store index into ecs_need_experiment
experiment_ecs[d] = ecs_need_experiment.size() - 1;
campaignmanager.addParam(d);
++count;
}
#else
// map for efficient access when results come in
map<WeatherMonitorExperimentData *, unsigned> experiment_ecs;
// map for keeping one "open" EC for every address
map<address_t, equivalence_class> open_ecs;
// experiment count
int count = 0;
// instruction counter within trace
int instr = 0;
// fill open_ecs with one EC for every address
for (MemoryMap::iterator it = mm.begin(); it != mm.end(); ++it) {
open_ecs[*it].instr1 = instr;
}
// absolute address of current trace instruction
address_t instr_absolute = 0; // FIXME this one probably should also be recorded ...
Trace_Event ev;
// for every event in the trace ...
while (ps.getNext(&ev) && instr < WEATHER_NUMINSTR_TRACING) {
// instruction events just get counted
if (!ev.has_memaddr()) {
// new instruction
instr++;
instr_absolute = ev.ip();
continue;
}
// for each single byte in this memory access ...
for (address_t data_address = ev.memaddr(); data_address < ev.memaddr() + ev.width();
++data_address) {
// skip accesses to data outside our map of interesting addresses
map<address_t, equivalence_class>::iterator current_ec_it;
if ((current_ec_it = open_ecs.find(data_address)) == open_ecs.end()) {
continue;
}
equivalence_class& current_ec = current_ec_it->second;
// skip zero-sized intervals: these can occur when an instruction
// accesses a memory location more than once (e.g., INC, CMPXCHG)
if (current_ec.instr1 > instr) {
continue;
}
// we now have an interval-terminating R/W event to the memaddr
// we're currently looking at:
// complete the equivalence interval
current_ec.instr2 = instr;
current_ec.instr2_absolute = instr_absolute;
current_ec.data_address = data_address;
if (ev.accesstype() == ev.READ) {
// a sequence ending with READ: we need to do one experiment to
// cover it completely
ecs_need_experiment.push_back(current_ec);
#ifdef PRUNING_DEBUG_OUTPUT
cerr << dec << "EX " << current_ec.instr1 << " " << current_ec.instr2 << " " << current_ec.data_address << "\n";
#endif
// instantly enqueue job: that way the job clients can already
// start working in parallel
WeatherMonitorExperimentData *d = new WeatherMonitorExperimentData;
// we pick the rightmost instruction in that interval
d->msg.set_instr_offset(current_ec.instr2);
d->msg.set_instr_address(current_ec.instr2_absolute);
d->msg.set_mem_addr(current_ec.data_address);
// store index into ecs_need_experiment
experiment_ecs[d] = ecs_need_experiment.size() - 1;
campaignmanager.addParam(d);
++count;
} else if (ev.accesstype() == ev.WRITE) {
// a sequence ending with WRITE: an injection anywhere here
// would have no effect.
ecs_no_effect.push_back(current_ec);
#ifdef PRUNING_DEBUG_OUTPUT
cerr << dec << "NE " << current_ec.instr1 << " " << current_ec.instr2 << " " << current_ec.data_address << "\n";
#endif
} else {
log << "WAT" << endl;
}
// next interval must start at next instruction; the aforementioned
// skipping mechanism wouldn't work otherwise
current_ec.instr1 = instr + 1;
}
}
// close all open intervals (right end of the fault-space)
for (map<address_t, equivalence_class>::iterator current_ec_it = open_ecs.begin();
current_ec_it != open_ecs.end(); ++current_ec_it) {
address_t data_address = current_ec_it->first;
equivalence_class& current_ec = current_ec_it->second;
// Why -1? In most cases it does not make sense to inject before the
// very last instruction, as we won't execute it anymore. This *only*
// makes sense if we also inject into parts of the result vector. This
// is not the case in this experiment, and with -1 we'll get a result
// comparable to the non-pruned campaign.
// XXX still true for weathermonitor?
current_ec.instr2 = instr - 1;
current_ec.instr2_absolute = 0; // unknown
current_ec.data_address = data_address;
// zero-sized? skip.
if (current_ec.instr1 > current_ec.instr2) {
continue;
}
// the run continues after the FI window, so do this experiment
// XXX this creates at least one experiment for *every* bit!
// fix: full trace, limited FI window
//ecs_no_effect.push_back(current_ec);
ecs_need_experiment.push_back(current_ec);
#ifdef PRUNING_DEBUG_OUTPUT
cerr << dec << "EX " << current_ec.instr1 << " " << current_ec.instr2 << " " << current_ec.data_address << "\n";
#endif
// FIXME copy/paste, encapsulate this:
// instantly enqueue job: that way the job clients can already start
// working in parallel
WeatherMonitorExperimentData *d = new WeatherMonitorExperimentData;
// we pick the rightmost instruction in that interval
d->msg.set_instr_offset(current_ec.instr2);
//d->msg.set_instr_address(current_ec.instr2_absolute); // unknown!
d->msg.set_mem_addr(current_ec.data_address);
// store index into ecs_need_experiment
experiment_ecs[d] = ecs_need_experiment.size() - 1;
campaignmanager.addParam(d);
++count;
}
// conserve some memory
open_ecs.clear();
#endif
campaignmanager.noMoreParameters();
log << "done enqueueing parameter sets (" << count << ")." << endl;
log << "equivalence classes generated:"
<< " need_experiment = " << ecs_need_experiment.size()
<< " no_effect = " << ecs_no_effect.size() << endl;
// statistics
unsigned long num_dumb_experiments = 0;
for (vector<equivalence_class>::const_iterator it = ecs_need_experiment.begin();
it != ecs_need_experiment.end(); ++it) {
num_dumb_experiments += (*it).instr2 - (*it).instr1 + 1;
}
for (vector<equivalence_class>::const_iterator it = ecs_no_effect.begin();
it != ecs_no_effect.end(); ++it) {
num_dumb_experiments += (*it).instr2 - (*it).instr1 + 1;
}
log << "pruning: reduced " << num_dumb_experiments * 8 <<
" experiments to " << ecs_need_experiment.size() * 8 << endl;
// CSV header
results << "ec_instr1\tec_instr2\tec_instr2_absolute\tec_data_address\tbitnr\tbit_width\tresulttype\tlatest_ip\titer1\titer2\tdetails" << endl;
// store no-effect "experiment" results
for (vector<equivalence_class>::const_iterator it = ecs_no_effect.begin();
it != ecs_no_effect.end(); ++it) {
results
<< (*it).instr1 << "\t"
<< (*it).instr2 << "\t"
<< (*it).instr2_absolute << "\t" // incorrect in all but one case!
<< (*it).data_address << "\t"
<< "0\t" // this entry starts at bit 0 ...
<< "8\t" // ... and is 8 bits wide
<< "1\t"
<< "99\t" // dummy value: we didn't do any real experiments
<< "0\t"
<< (WEATHER_NUMITER_TRACING + WEATHER_NUMITER_AFTER) << "\t\n";
}
// collect results
WeatherMonitorExperimentData *res;
int rescount = 0;
while ((res = static_cast<WeatherMonitorExperimentData *>(campaignmanager.getDone()))) {
rescount++;
map<WeatherMonitorExperimentData *, unsigned>::iterator it =
experiment_ecs.find(res);
if (it == experiment_ecs.end()) {
results << "WTF, didn't find res!" << endl;
log << "WTF, didn't find res!" << endl;
continue;
}
equivalence_class &ec = ecs_need_experiment[it->second];
// sanity check
if (ec.instr2 != res->msg.instr_offset()) {
results << "ec.instr2 != instr_offset" << endl;
log << "ec.instr2 != instr_offset" << endl;
}
if (res->msg.result_size() != 8) {
results << "result_size " << res->msg.result_size()
<< " instr2 " << ec.instr2
<< " data_address " << ec.data_address << endl;
log << "result_size " << res->msg.result_size() << endl;
}
// one job contains 8 experiments
for (int idx = 0; idx < res->msg.result_size(); ++idx) {
//results << "ec_instr1\tec_instr2\tec_instr2_absolute\tec_data_address\tbitnr\tresulttype\tlatest_ip\titer1\titer2\tdetails" << endl;
results
// repeated for all single experiments:
<< ec.instr1 << "\t"
<< ec.instr2 << "\t"
<< ec.instr2_absolute << "\t"
<< ec.data_address << "\t"
// individual results:
<< res->msg.result(idx).bit_offset() << "\t"
<< "1\t" // 1 bit wide
<< res->msg.result(idx).resulttype() << "\t"
<< res->msg.result(idx).latest_ip() << "\t"
<< res->msg.result(idx).iter_before_fi() << "\t"
<< res->msg.result(idx).iter_after_fi() << "\t"
<< res->msg.result(idx).details() << "\n";
}
//delete res; // currently racy if jobs are reassigned
}
results.close();
log << "done. sent " << count << " received " << rescount << endl;
log << "elapsed: " << t.elapsed() << "s" << endl;
return true;
WeatherMonitorExperimentData *data = new WeatherMonitorExperimentData;
data->msg.mutable_fsppilot()->CopyFrom(pilot);
campaignmanager.addParam(data);
}
src/experiments/weather-monitor/campaign.hpp
+10 -5
View File
@@ -1,9 +1,10 @@
#ifndef __WEATHERMONITOR_CAMPAIGN_HPP__
#define __WEATHERMONITOR_CAMPAIGN_HPP__
#define __WEATHERMONITOR_CAMPAIGN_HPP__
#include "cpn/Campaign.hpp"
#include "cpn/DatabaseCampaign.hpp"
#include "comm/ExperimentData.hpp"
#include "weathermonitor.pb.h"
#include <google/protobuf/descriptor.h>
class WeatherMonitorExperimentData : public fail::ExperimentData {
public:
@@ -11,9 +12,13 @@ public:
WeatherMonitorExperimentData() : fail::ExperimentData(&msg) {}
};
class WeatherMonitorCampaign : public fail::Campaign {
public:
virtual bool run();
class WeatherMonitorCampaign : public fail::DatabaseCampaign {
virtual const google::protobuf::Descriptor * cb_result_message()
{
return google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName("WeathermonitorProtoMsg");
}
virtual void cb_send_pilot(DatabaseCampaignMessage pilot);
};
#endif // __WEATHERMONITOR_CAMPAIGN_HPP__
src/experiments/weather-monitor/experiment.cc
+31 -20
View File
@@ -40,14 +40,24 @@ bool WeatherMonitorExperiment::run()
log << "startup" << endl;
#if 1
/*
* this does not work as the guestsys doesn't output anything
* albeit that, it's no longer needed in this experiment
*/
#if 0
// STEP 0: record memory map with vptr addresses
ofstream mmap;
mmap.open ("memory.map");
GuestListener g;
while (true) {
simulator.addListenerAndResume(&g);
cout << g.getData() << flush;
mmap << g.getData() << flush;
}
#elif 0
mmap.close();
#endif
#if 1
// STEP 1: run until interesting function starts, and save state
bp.setWatchInstructionPointer(WEATHER_FUNC_MAIN);
simulator.addListenerAndResume(&bp);
@@ -154,22 +164,24 @@ bool WeatherMonitorExperiment::run()
}
#else
// XXX debug
param.msg.set_instr_offset(1000);
//param.msg.set_instr_address(12345);
param.msg.set_mem_addr(0x00103bdc);
param.msg.fsppilot().set_injection_instr(1000);
param.msg.fsppilot().set_data_address(0x00103bdc);
#endif
int id = param.getWorkloadID();
int instr_offset = param.msg.instr_offset();
int mem_addr = param.msg.mem_addr();
unsigned injection_instr = param.msg.fsppilot().injection_instr();
address_t data_address = param.msg.fsppilot().data_address();
//old data. now it resides in the DatabaseCampaignMessage
// for each job we're actually doing *8* experiments (one for each bit)
for (int bit_offset = 0; bit_offset < 8; ++bit_offset) {
// 8 results in one job
WeathermonitorProtoMsg_Result *result = param.msg.add_result();
result->set_bit_offset(bit_offset);
log << dec << "job " << id << " instr " << instr_offset
<< " mem " << mem_addr << "+" << bit_offset << endl;
result->set_bitoffset(bit_offset);
log << dec << "job " << id << " instr " << injection_instr
<< " mem " << data_address << "+" << bit_offset << endl;
log << "restoring state" << endl;
simulator.restore(statename);
@@ -179,7 +191,7 @@ bool WeatherMonitorExperiment::run()
stringstream fname;
fname << "job." << ::getpid();
ofstream job(fname.str().c_str());
job << "job " << id << " instr " << instr_offset << " (" << param.msg.instr_address() << ") mem " << mem_addr << "+" << bit_offset << endl;
job << "job " << id << " instr " << injection_instr << " (" << param.msg.fsppilot().injection_instr_absolute() << ") mem " << param.msg.fsppilot().data_address() << "+" << bit_offset << endl;
job.close();
*/
@@ -196,10 +208,10 @@ bool WeatherMonitorExperiment::run()
int count_loop_iter_before = 0;
// no need to wait if offset is 0
if (instr_offset > 0) {
if (injection_instr > 0) {
// XXX could be improved with intermediate states (reducing runtime until injection)
bp.setWatchInstructionPointer(ANY_ADDR);
bp.setCounter(instr_offset);
bp.setCounter(injection_instr);
simulator.addListener(&bp);
// count loop iterations until FI
@@ -211,21 +223,20 @@ bool WeatherMonitorExperiment::run()
// --- fault injection ---
MemoryManager& mm = simulator.getMemoryManager();
byte_t data = mm.getByte(mem_addr);
byte_t data = mm.getByte(data_address);
byte_t newdata = data ^ (1 << bit_offset);
mm.setByte(mem_addr, newdata);
mm.setByte(data_address, newdata);
// note at what IP we did it
int32_t injection_ip = simulator.getCPU(0).getInstructionPointer();
param.msg.set_injection_ip(injection_ip);
result->set_iter_before_fi(count_loop_iter_before);
log << "fault injected @ ip " << injection_ip
<< " 0x" << hex << ((int)data) << " -> 0x" << ((int)newdata) << endl;
// sanity check
if (param.msg.has_instr_address() &&
injection_ip != param.msg.instr_address()) {
if (param.msg.fsppilot().has_injection_instr_absolute() &&
injection_ip != param.msg.fsppilot().injection_instr_absolute()) {
stringstream ss;
ss << "SANITY CHECK FAILED: " << injection_ip
<< " != " << param.msg.instr_address();
<< " != " << param.msg.fsppilot().injection_instr_absolute();
log << ss.str() << endl;
result->set_resulttype(result->UNKNOWN);
result->set_latest_ip(injection_ip);
src/experiments/weather-monitor/weathermonitor.proto
+6 -16
View File
@@ -1,25 +1,15 @@
import "DatabaseCampaignMessage.proto";
message WeathermonitorProtoMsg {
required DatabaseCampaignMessage fsppilot = 1;
// Input: experiment parameters
// (client executes 8 experiments, one for each bit at mem_addr)
// FI at #instructions from experiment start
required int32 instr_offset = 1;
// the exact IP value at this point in time (from golden run)
optional int32 instr_address = 2; // for sanity checks
// address of the byte to inject bit-flips
required int32 mem_addr = 3;
// ----------------------------------------------------
// Output: experiment results
// IP where we did the injection: for debugging purposes, must be identical
// to instr_address
optional int32 injection_ip = 4;
repeated group Result = 5 {
repeated group Result = 6 {
// single experiment bit offset
required int32 bit_offset = 1;
required int32 bitoffset = 1 [(sql_primary_key) = true];
// result type:
// FINISHED = planned number of instructions were executed