From bfcb1b415b9029e2051ad6a57bd211e25455148c Mon Sep 17 00:00:00 2001 From: Christian Dietrich Date: Wed, 27 May 2015 10:42:45 +0200 Subject: [PATCH] experiments: experiment for tracing and testing ERIKA ERIKA Enterprise is a OSEK conforming embedded RTOS. The supplied tracer and experiment are similar to the cored-{tracing,tester} experiment, but checks the integrity of the RTOS application in a different manner. Stacks and stackpointers are located differently in ERIKA. This experiment was used in RTAS'15 Hoffmann et al. Change-Id: Idc8d874eb4d4ef15837f903270cfa521bc9514a2 --- src/experiments/erika-tester/CMakeLists.txt | 36 ++ src/experiments/erika-tester/campaign.cc | 20 + src/experiments/erika-tester/campaign.hpp | 30 + src/experiments/erika-tester/config.cmake | 6 + .../erika-tester/erika-tester.proto | 33 + src/experiments/erika-tester/experiment.cc | 572 ++++++++++++++++++ src/experiments/erika-tester/experiment.hpp | 33 + src/experiments/erika-tester/main.cc | 20 + src/experiments/erika-tracing/CMakeLists.txt | 18 + src/experiments/erika-tracing/config.cmake | 12 + src/experiments/erika-tracing/experiment.cc | 291 +++++++++ src/experiments/erika-tracing/experiment.hpp | 37 ++ .../erika-tracing/fail_experiment.py | 99 +++ 13 files changed, 1207 insertions(+) create mode 100644 src/experiments/erika-tester/CMakeLists.txt create mode 100644 src/experiments/erika-tester/campaign.cc create mode 100644 src/experiments/erika-tester/campaign.hpp create mode 100644 src/experiments/erika-tester/config.cmake create mode 100644 src/experiments/erika-tester/erika-tester.proto create mode 100644 src/experiments/erika-tester/experiment.cc create mode 100644 src/experiments/erika-tester/experiment.hpp create mode 100644 src/experiments/erika-tester/main.cc create mode 100644 src/experiments/erika-tracing/CMakeLists.txt create mode 100644 src/experiments/erika-tracing/config.cmake create mode 100644 src/experiments/erika-tracing/experiment.cc create mode 100644 src/experiments/erika-tracing/experiment.hpp create mode 100755 src/experiments/erika-tracing/fail_experiment.py diff --git a/src/experiments/erika-tester/CMakeLists.txt b/src/experiments/erika-tester/CMakeLists.txt new file mode 100644 index 00000000..0bbb3a0b --- /dev/null +++ b/src/experiments/erika-tester/CMakeLists.txt @@ -0,0 +1,36 @@ +set(EXPERIMENT_NAME erika-tester) +set(EXPERIMENT_TYPE ErikaTester) +configure_file(../instantiate-experiment.ah.in + ${CMAKE_CURRENT_BINARY_DIR}/instantiate-${EXPERIMENT_NAME}.ah @ONLY +) + +## Setup desired protobuf descriptions HERE ## +set(MY_PROTOS + erika-tester.proto +) + +set(MY_CAMPAIGN_SRCS + experiment.hpp + experiment.cc + campaign.hpp + campaign.cc +) + +#### PROTOBUFS #### +find_package(Protobuf REQUIRED) +include_directories(${PROTOBUF_INCLUDE_DIRS}) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS ${MY_PROTOS}) + +## Build library +add_library(fail-${EXPERIMENT_NAME} ${PROTO_SRCS} ${PROTO_HDRS} ${MY_CAMPAIGN_SRCS}) +add_dependencies(fail-${EXPERIMENT_NAME} fail-comm) +target_link_libraries(fail-${EXPERIMENT_NAME} fail-comm) +target_link_libraries(fail-${EXPERIMENT_NAME} ${PROTOBUF_LIBRARY} fail-llvmdisassembler) + +## This is the example's campaign server distributing experiment parameters +add_executable(${EXPERIMENT_NAME}-server main.cc) +target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} ${Boost_SYSTEM_LIBRARY} -lmysqlclient -Wl,--end-group) +install(TARGETS ${EXPERIMENT_NAME}-server RUNTIME DESTINATION bin) + diff --git a/src/experiments/erika-tester/campaign.cc b/src/experiments/erika-tester/campaign.cc new file mode 100644 index 00000000..cd41a4dc --- /dev/null +++ b/src/experiments/erika-tester/campaign.cc @@ -0,0 +1,20 @@ +#include +#include + +#include "campaign.hpp" +#include "cpn/CampaignManager.hpp" +#include "util/Logger.hpp" +#include "util/ElfReader.hpp" +#include "util/ProtoStream.hpp" +#include "sal/SALConfig.hpp" + +using namespace std; +using namespace fail; +using namespace google::protobuf; + +void ErikaTesterCampaign::cb_send_pilot(DatabaseCampaignMessage pilot) { + ErikaTesterExperimentData *data = new ErikaTesterExperimentData; + data->msg.mutable_fsppilot()->CopyFrom(pilot); + campaignmanager.addParam(data); +} + diff --git a/src/experiments/erika-tester/campaign.hpp b/src/experiments/erika-tester/campaign.hpp new file mode 100644 index 00000000..04ea3e6f --- /dev/null +++ b/src/experiments/erika-tester/campaign.hpp @@ -0,0 +1,30 @@ +#ifndef __DCIAOCAMPAIGN_HPP__ +#define __DCIAOCAMPAIGN_HPP__ + +#include "cpn/DatabaseCampaign.hpp" +#include "comm/ExperimentData.hpp" +#include "erika-tester.pb.h" +#include "util/ElfReader.hpp" +#include + +class ErikaTesterExperimentData : public fail::ExperimentData { +public: + ErikaTesterProtoMsg msg; + ErikaTesterExperimentData() : fail::ExperimentData(&msg) {} +}; + +class ErikaTesterCampaign : public fail::DatabaseCampaign { + virtual const google::protobuf::Descriptor * cb_result_message() + { return google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName("ErikaTesterProtoMsg"); } + + virtual void cb_send_pilot(DatabaseCampaignMessage pilot); + virtual int expected_number_of_results(std::string variant, std::string benchmark) { + if (benchmark.find("jump") != std::string::npos) + return 1; + else + return 8; + } + +}; + +#endif // __KESOREFCAMPAIGN_HPP__ diff --git a/src/experiments/erika-tester/config.cmake b/src/experiments/erika-tester/config.cmake new file mode 100644 index 00000000..2a46e883 --- /dev/null +++ b/src/experiments/erika-tester/config.cmake @@ -0,0 +1,6 @@ +SET(bochs_configure_params --enable-a20-pin;--enable-x86-64;--enable-cpu-level=6;--enable-ne2000;--enable-acpi;--enable-pci;--enable-usb;--enable-trace-cache;--enable-fast-function-calls;--enable-host-specific-asms;--enable-readline;--enable-clgd54xx;--enable-fpu;--enable-vmx=2;--enable-monitor-mwait;--enable-cdrom;--enable-sb16=linux;--enable-gdb-stub;--with-nogui CACHE STRING "") + +SET(PLUGINS_ACTIVATED "checkpoint;randomgenerator" CACHE STRING "") + +SET(BUILD_LLVM_DISASSEMBLER ON CACHE BOOL "" FORCE) + diff --git a/src/experiments/erika-tester/erika-tester.proto b/src/experiments/erika-tester/erika-tester.proto new file mode 100644 index 00000000..85644fcc --- /dev/null +++ b/src/experiments/erika-tester/erika-tester.proto @@ -0,0 +1,33 @@ +import "DatabaseCampaignMessage.proto"; + +message ErikaTesterProtoMsg { + required DatabaseCampaignMessage fsppilot = 1; + + repeated group Result = 2 { + enum ResultType { + /* Test did the right thing */ + OK = 1; + OK_WRONG_CONTROL_FLOW = 2; + OK_DETECTED_ERROR = 3; + + /* Test did the wrong thing */ + ERR_WRONG_RESULT = 4; + + ERR_TRAP = 5; + ERR_TIMEOUT = 6; + ERR_OUTSIDE_TEXT = 7; + ERR_OUTSIDE_DATA = 8; + + UNKNOWN = 9; + NOINJECTION = 10; + } + + required int32 bitoffset = 1 [(sql_primary_key) = true]; + required ResultType resulttype = 2; + required uint32 experiment_number = 3; + + optional uint32 original_value = 4; + + optional string details = 5; + } +} diff --git a/src/experiments/erika-tester/experiment.cc b/src/experiments/erika-tester/experiment.cc new file mode 100644 index 00000000..2f25da18 --- /dev/null +++ b/src/experiments/erika-tester/experiment.cc @@ -0,0 +1,572 @@ +#include +#include +#include + +#include +#include + +#include +#include "experiment.hpp" +#include "sal/SALConfig.hpp" +#include "sal/SALInst.hpp" +#include "sal/Memory.hpp" +#include "sal/Listener.hpp" + +#include "sal/bochs/BochsListener.hpp" +#include "sal/bochs/BochsMemory.hpp" +#include "sal/bochs/BochsCPU.hpp" +#include +#include +#include +#include +#include + +#include "util/llvmdisassembler/LLVMtoFailTranslator.hpp" +#include "util/llvmdisassembler/LLVMtoFailBochs.hpp" + +#include "campaign.hpp" +#include "erika-tester.pb.h" + +#include "../plugins/randomgenerator/RandomGenerator.hpp" +#include "../plugins/checkpoint/Checkpoint.hpp" + +using namespace std; +using namespace fail; + +#define SAFESTATE (1) + +// Check if configuration dependencies are satisfied: +#if !defined(CONFIG_EVENT_BREAKPOINTS) || !defined(CONFIG_SR_RESTORE) || \ + !defined(CONFIG_SR_SAVE) +#error This experiment needs: breakpoints, traps, save, and restore. Enable these in the configuration. +#endif +#if !defined(CONFIG_EVENT_MEMREAD) || !defined(CONFIG_EVENT_MEMWRITE) +#error This experiment needs: MemRead and MemWrite. Enable these in the configuration. +#endif + +void ErikaTester::redecodeCurrentInstruction() { + /* Flush Instruction Caches and Prefetch queue */ + BX_CPU_C *cpu_context = simulator.getCPUContext(); + cpu_context->invalidate_prefetch_q(); + cpu_context->iCache.flushICacheEntries(); + + + guest_address_t pc = simulator.getCPU(0).getInstructionPointer(); + bxInstruction_c *currInstr = simulator.getCurrentInstruction(); + //unsigned length_in_bytes = currInstr->ilen(); + + m_log << "REDECODE INSTRUCTION @ IP 0x" << std::hex << pc << endl; + + Bit32u eipBiased = pc + cpu_context->eipPageBias; + Bit8u instr_plain[15]; + + MemoryManager& mm = simulator.getMemoryManager(); + for(unsigned i=0; ieipPageWindowSize - eipBiased; + int ret; +#if BX_SUPPORT_X86_64 + if (cpu_context->cpu_mode == BX_MODE_LONG_64) + ret = cpu_context->fetchDecode64(instr_plain, currInstr, remainingInPage); + else +#endif + ret = cpu_context->fetchDecode32(instr_plain, currInstr, remainingInPage); + if (ret < 0) { + // handle instrumentation callback inside boundaryFetch + cpu_context->boundaryFetch(instr_plain, remainingInPage, currInstr); + } +} + + +unsigned ErikaTester::injectBitFlip(address_t data_address, unsigned data_width, unsigned bitpos) { + /* First 32 Registers, this might neeed adaption */ + if (data_address < (32 << 8)) { + LLVMtoFailTranslator::reginfo_t reginfo = LLVMtoFailTranslator::reginfo_t::fromDataAddress(data_address, data_width); + + unsigned int value, injectedval; + + value = m_ltof->getRegisterContent(simulator.getCPU(0), reginfo); + injectedval = value ^ (1 << bitpos); + m_ltof->setRegisterContent(simulator.getCPU(0), reginfo, injectedval); + + m_log << "INJECTING register (" << dec << reginfo.id + << " offset " << (int) reginfo.offset + << ") bitpos: " << bitpos + << " value: 0x" << hex << setw(2) << setfill('0') << value << " -> 0x" << setw(2) << setfill('0') << injectedval + << dec << endl; + if (reginfo.id == RID_PC) + redecodeCurrentInstruction(); + + return value; + } else { + unsigned int value, injectedval; + + MemoryManager& mm = simulator.getMemoryManager(); + if(!mm.isMapped(data_address)) { + m_log << "DATA_ADDRESS NOT MAPPED" << endl; + return 0; + } + + value = mm.getByte(data_address); + injectedval = value ^ (1 << bitpos); + mm.setByte(data_address, injectedval); + + m_log << "INJECTION at: 0x" << hex << setw(2) << setfill('0') << data_address + << " value: 0x" << setw(2) << setfill('0') << value << " -> 0x" << setw(2) << setfill('0') << injectedval << endl; + + /* If it is the current instruction redecode it */ + guest_address_t pc = simulator.getCPU(0).getInstructionPointer(); + m_log << "IP: 0x" << std::hex << pc << std::endl; + bxInstruction_c *currInstr = simulator.getCurrentInstruction(); + assert(currInstr != NULL && "FATAL ERROR: current instruction was NULL (not expected)!"); + unsigned length_in_bytes = currInstr->ilen(); + + if (currInstr == NULL || (pc <= data_address && data_address <= (pc + currInstr->ilen()))) { + redecodeCurrentInstruction(); + } + + return value; + } +} + + +void handleEvent(ErikaTesterProtoMsg_Result& result, ErikaTesterProtoMsg_Result_ResultType restype, const std::string &msg) { + cout << msg << endl; + result.set_resulttype(restype); + result.set_details(msg); +} + + +std::string handleMemoryAccessEvent(fail::MemAccessListener* l_mem) { + stringstream sstr; + + sstr << "mem access ("; + switch (l_mem->getTriggerAccessType()) { + case MemAccessEvent::MEM_READ: + sstr << "r"; + break; + case MemAccessEvent::MEM_WRITE: + sstr << "w"; + break; + default: + break; + } + sstr << ") @ 0x" << hex << l_mem->getTriggerAddress(); + sstr << "; ip @ 0x" << hex << l_mem->getTriggerInstructionPointer(); + + return sstr.str(); +} + + +const ElfSymbol& ErikaTester::getELFSymbol(const std::string name) { + const ElfSymbol &symbol = m_elf.getSymbol(name); + if (!symbol.isValid()) { + m_log << "Couldn't find symbol: " << name << std::endl; + simulator.terminate(1); + } + + return symbol; +} + + +bool ErikaTester::run() { + m_log << "STARTING EXPERIMENT" << endl; + + // seed random number generator + timeval start; + gettimeofday(&start, NULL); + srand(start.tv_usec); + + // get symbols + const ElfSymbol &s_trace_end_marker = getELFSymbol("test_finish"); + BPSingleListener l_trace_end_marker(s_trace_end_marker.getAddress()); + + const ElfSymbol &s_stext_app = getELFSymbol("_stext_application"); + const ElfSymbol &s_etext_app = getELFSymbol("_etext_application"); + + const ElfSymbol &s_panic_handler = m_elf.getSymbol("panic_handler"); + assert (s_panic_handler.isValid()); + + const ElfSymbol &s_fail_trace = getELFSymbol("fail_trace"); + MemAccessListener l_fail_trace(s_fail_trace.getAddress()); + + const ElfSymbol &s_random_source = m_elf.getSymbol("random_source"); + const ElfSymbol &s_tos = getELFSymbol("EE_x86_system_tos"); + + + // allowed regions + address_t text_tasks_start = s_stext_app.getAddress(); + address_t text_tasks_end = s_etext_app.getAddress(); + + m_log << "not injecting in application: " << std::hex << text_tasks_start << " -- " << text_tasks_end << std::endl; + + // listeners for traps, code region + //InterruptListener l_interrupt(2); // NMI interrupt + //TrapListener l_trap(2); // NMI trap? + BPSingleListener* l_panic; + if(s_panic_handler.isValid()) { + l_panic = new BPSingleListener(s_panic_handler.getAddress()); + m_log << "PANIC handler @ " << std::hex << s_panic_handler.getAddress() << std::endl; + } + + unsigned i_timeout = 2000 * 1000; + TimerListener l_timeout(i_timeout); // 1s in microseconds + TimerListener l_timeout_hard(2 * 1000 * 1000); // 2s in microseconds + + + // initialize LLVM disassembler + m_ltof = LLVMtoFailTranslator::createFromBinary(m_elf.getFilename()); + + // memory manager + MemoryManager& mm = simulator.getMemoryManager(); + + // job client with environment parameters + char* server = getenv("FAIL_SERVER_HOST"); + if(server == NULL) server = (char*) SERVER_COMM_HOSTNAME; + char* cport = getenv("FAIL_SERVER_PORT"); + int port = (cport != NULL) ? atoi(cport) : SERVER_COMM_TCP_PORT; + fail::JobClient jobclient(server, port); + + // execute jobs + unsigned executed_jobs = 0; + while (executed_jobs < 25 || jobclient.getNumberOfUndoneJobs() > 0) { + // get next parameters from jobserver + m_log << "asking jobserver for parameters" << endl; + ErikaTesterExperimentData param; + if(!jobclient.getParam(param)){ + m_log << "Dying." << endl; // We were told to die. + simulator.terminate(128); // special return code to signal campaign finish + } + + // extract parameters + unsigned injection_instr = param.msg.fsppilot().injection_instr(); + address_t data_address = param.msg.fsppilot().data_address(); + unsigned data_width = param.msg.fsppilot().data_width(); + + // detect wheter we should inject the PC + bool pc_injection = param.msg.fsppilot().benchmark().find("jump") != std::string::npos; + + // setup experiments + int experiments = pc_injection ? 1 : 8; + + // run experiments + for (int experiment_id = 0; experiment_id < experiments; ++experiment_id) { + ErikaTesterProtoMsg_Result *result = 0; + + // restore to the image + m_log << "restoring state" << endl; + simulator.clearListeners(this); + simulator.restore("state"); + m_log << "state restored" << endl; + executed_jobs++; + + // init plugins + Checkpoint::range_vector check_ranges; + + // The idle stack + std::map stackpointers; + const ElfSymbol &s_os_stack = m_elf.getSymbol("os_stack"); + assert (s_os_stack.isValid()); + m_log << "found task stack: " << "os_stack" << std::endl; + fail::address_t paddr; + paddr = s_tos.getAddress() + 0*4; + Checkpoint::indirectable_address_t start = std::make_pair(paddr, true); + Checkpoint::indirectable_address_t end = std::make_pair(s_os_stack.getEnd(), false); + check_ranges.push_back(std::make_pair(start, end)); + stackpointers[paddr+0] = paddr; + stackpointers[paddr+1] = paddr; + stackpointers[paddr+2] = paddr; + stackpointers[paddr+3] = paddr; + + + for(int i=1; i<255; i++) { + stringstream stackname; + stackname << "EE_x86_stack_"; + stackname << i; + const ElfSymbol &s_stack = m_elf.getSymbol(stackname.str()); + if(!s_stack.isValid()) { + m_log << "found " << std::dec << (i-1) << " task stacks" << std::endl; + break; + } + + paddr = s_tos.getAddress() + i*4; + Checkpoint::indirectable_address_t start = std::make_pair(paddr, true); + Checkpoint::indirectable_address_t end = std::make_pair(s_stack.getEnd(), false); + check_ranges.push_back(std::make_pair(start, end)); + + stackpointers[paddr+0] = paddr; + stackpointers[paddr+1] = paddr; + stackpointers[paddr+2] = paddr; + stackpointers[paddr+3] = paddr; + } + + + Checkpoint cpoint(check_ranges, "checkpoint.trace"); + + RandomGenerator* rgen; + if (s_random_source.isValid()) { + const unsigned seed = 12342; + rgen = new RandomGenerator(s_random_source, seed); + simulator.addFlow(rgen); + } + + // fast forward to injection address + m_log << "Trying to inject @ instr #" << dec << injection_instr << endl; + simulator.clearListeners(this); + simulator.addListener(&l_fail_trace); + if (injection_instr != 0) { + + BPSingleListener bp; + bp.setWatchInstructionPointer(ANY_ADDR); + // TODO: why does this need a +1? + bp.setCounter(injection_instr); + simulator.addListener(&bp); + + fail::BaseListener * listener = simulator.resume(); + bool ok = true; + + while ( ok && (listener == &l_fail_trace) ) { + m_log << "CP IP 0x" << std::hex << simulator.getCPU(0).getInstructionPointer() << std::endl; + ok = cpoint.check(s_fail_trace, l_fail_trace.getTriggerInstructionPointer()) == Checkpoint::IDENTICAL; + if(ok) listener = simulator.addListenerAndResume(&l_fail_trace); + } + if(!ok) { + result = param.msg.add_result(); + handleEvent(*result, result->NOINJECTION, "CHECKPOINT FAIL BEFORE INJECTION?!"); + break; + } + if (listener != &bp) { + result = param.msg.add_result(); + handleEvent(*result, result->NOINJECTION, "WTF"); + break; + } + } + + // program counter sanity check + if (param.msg.fsppilot().has_injection_instr_absolute()) { + address_t PC = param.msg.fsppilot().injection_instr_absolute(); + if (simulator.getCPU(0).getInstructionPointer() != PC) { + m_log << "Invalid injection address EIP=0x" + << std::hex << simulator.getCPU(0).getInstructionPointer() + << " != injection_instr_absolute=0x" << PC << std::endl; + simulator.terminate(1); + } + } + + unsigned experiment_number = cpoint.getCount(); + m_log << "Trace " << std::dec << experiment_number << std::endl; + result = param.msg.add_result(); + result->set_experiment_number((unsigned int) experiment_number); + result->set_bitoffset(experiment_id); + result->set_original_value(0); + + // abort if outside targeted region + address_t PC = simulator.getCPU(0).getInstructionPointer(); + //if(PC < min_code || PC > max_code) { + if(PC < text_tasks_end && PC >= text_tasks_start) { + std::stringstream ss; + ss << "0x" << hex << PC; + ss << " inside task text: "; + ss << "0x" << hex << text_tasks_start << " - 0x" << hex << text_tasks_end; + handleEvent(*result, result->NOINJECTION, ss.str()); + continue; // next experiment + } + + MemWriteListener *stackpointer_event = 0; + bool stackpointer_event_valid = false; + bool check_digest = true; + + // perform injection + if (pc_injection) { + // jump to "data" address + address_t current_PC = simulator.getCPU(0).getInstructionPointer(); + address_t new_PC = param.msg.fsppilot().data_address(); + m_log << "jump from 0x" << hex << current_PC << " to 0x" << new_PC << std::endl; + simulator.getCPU(0).setRegisterContent(simulator.getCPU(0).getRegister(RID_PC), new_PC); + + // set program counter + result->set_bitoffset(0); + result->set_original_value(current_PC); + redecodeCurrentInstruction(); + } else { + // inject random bitflip + // experiment_id == bitpos + // We cache the stackpointer value inside the + // checkpoint plugin, if we inject it. This prevents + // the Checkpoint plugin to read invalid values. + if (stackpointers.find(data_address) != stackpointers.end()) { + guest_address_t paddr = stackpointers[data_address]; + guest_address_t value = 0; + simulator.getMemoryManager().getBytes(paddr, 4, &value); + cpoint.cache_stackpointer_variable(paddr, value); + stackpointer_event = new MemWriteListener(paddr); + stackpointer_event_valid = true; + m_log << "Injected Stackpointer; saved old stackpointer value (" + << hex << "0x" << paddr << "="<< value << dec << ")"<< std::endl; + } + + if (cpoint.in_range(data_address)) { + m_log << "Injected value is in checksummed range, ignore stack digest results" << endl; + check_digest = false; + } + + result->set_original_value(injectBitFlip(data_address, data_width, experiment_id)); + } + + // add listeners + simulator.clearListeners(this); + + if(s_panic_handler.isValid()) { + simulator.addListener(l_panic); + } + i_timeout = 2000 * 1000; + l_timeout.setTimeout(i_timeout); + simulator.addListener(&l_timeout); + simulator.addListener(&l_timeout_hard); + BPSingleListener l_timeout_instr; + l_timeout_instr.setWatchInstructionPointer(ANY_ADDR); + // Simulate only further 500 000 instructions + l_timeout_instr.setCounter(500000); + simulator.addListener(&l_timeout_instr); + + simulator.addListener(&l_fail_trace); + simulator.addListener(&l_trace_end_marker); + + if (stackpointer_event_valid) { + m_log << "Injected Stackpointer; add listener 0x" + << hex << stackpointer_event->getWatchAddress() << dec + << " " << stackpointer_event->getTriggerAccessType() << endl; + simulator.addListener(stackpointer_event); + } + + // resume and wait for results + m_log << "Resuming till the crash" << std::endl; + bool reached_check_start = false; + fail::BaseListener* l = simulator.resume(); + + int checkpoint_count = 0; + + while(l == &l_fail_trace || l == stackpointer_event) { + if (l == stackpointer_event) { + cpoint.uncache_stackpointer_variable(stackpointer_event->getWatchAddress()); + m_log << "Injected Stackpointer; cleared stackpointer cache" << std::endl; + l = simulator.resume(); + continue; + } + + Checkpoint::check_result res = cpoint.check(s_fail_trace, l_fail_trace.getTriggerInstructionPointer()); + checkpoint_count ++; + if(res == Checkpoint::DIFFERENT_IP) { + std::stringstream ss; + ss << "different IP"; + ss << "@ IP 0x" << std::hex << l_fail_trace.getTriggerInstructionPointer(); + ss << " (checkpoint " << std::dec << cpoint.getCount() << ")"; + handleEvent(*result, result->ERR_WRONG_RESULT, ss.str()); + break; + } else if(res == Checkpoint::DIFFERENT_VALUE) { + std::stringstream ss; + ss << "different value"; + ss << "@ IP 0x" << std::hex << l_fail_trace.getTriggerInstructionPointer(); + ss << " (checkpoint " << std::dec << cpoint.getCount() << ")"; + handleEvent(*result, result->ERR_WRONG_RESULT, ss.str()); + break; + } else if(check_digest && res == Checkpoint::DIFFERENT_DIGEST) { + std::stringstream ss; + ss << "different digest"; + ss << "@ IP 0x" << std::hex << l_fail_trace.getTriggerInstructionPointer(); + ss << " (checkpoint " << std::dec << cpoint.getCount() << ")"; + + // IGNORE for ISORC'14 experiments + handleEvent(*result, result->ERR_WRONG_RESULT, ss.str()); + break; + } else if(res == Checkpoint::INVALID) { + std::stringstream ss; + ss << "invalid checkpoint"; + ss << "@ IP 0x" << std::hex << l_fail_trace.getTriggerInstructionPointer(); + ss << " (checkpoint " << std::dec << cpoint.getCount() << ")"; + handleEvent(*result, result->ERR_WRONG_RESULT, ss.str()); + break; + } + m_log << "i " << checkpoint_count << " " << simulator.getTimerTicks() << " " << i_timeout << endl; + + // Reset the soft timeout listener + simulator.removeListener(&l_timeout); + if (checkpoint_count > 5) { + i_timeout = 59 * 1000; + } + l_timeout.setTimeout(i_timeout); + simulator.addListener(&l_timeout); + assert(l_timeout.getTimeout() == i_timeout); + + l = simulator.addListenerAndResume(&l_fail_trace); + } + + if(l == &l_fail_trace) { + // already reported invalid checkpoint + } else if(l == &l_trace_end_marker) { + // trace ended successfully + std::stringstream ss; + ss << "correct end after " << cpoint.getCount() << " checkpoints"; + handleEvent(*result, result->OK, ss.str()); + } else if (l == l_panic) { + // error detected + stringstream sstr; + sstr << "PANIC"; + + // ErikaOS specific trap information + const Register *reg_eax = simulator.getCPU(0).getRegister(RID_CAX); + uint32_t trap = simulator.getCPU(0).getRegisterContent(reg_eax); + sstr << " trap " << dec << trap; + + //address_t sp = simulator.getCPU(0).getStackPointer(); + //if(mm.isMapped(sp)) { + // uint32_t ip; + // mm.getBytes(sp, 4, &ip); + // sstr << " @ 0x" << hex << ip; + + const Register *reg_esi = simulator.getCPU(0).getRegister(RID_CSI); + address_t sp = simulator.getCPU(0).getRegisterContent(reg_esi) + 4; + if(mm.isMapped(sp) && mm.isMapped(sp+1) && mm.isMapped(sp+2) && mm.isMapped(sp+3)) { + uint32_t ip; + mm.getBytes(sp, 4, &ip); + sstr << " from 0x" << hex << ip; + } + + handleEvent(*result, result->OK_DETECTED_ERROR, sstr.str()); + } else if ( l == &l_timeout || l == &l_timeout_hard || l == &l_timeout_instr) { + if (l == &l_timeout) { + handleEvent(*result, result->ERR_TIMEOUT, "soft timeout"); + } else if (l == &l_timeout_hard) { + handleEvent(*result, result->ERR_TIMEOUT, "hard timeout"); + } else if (l == &l_timeout_instr) { + handleEvent(*result, result->ERR_TIMEOUT, "instr timeout"); + } + } else { + // what happened? + handleEvent(*result, result->UNKNOWN, "WTF"); + } + + if (stackpointer_event_valid) { + delete stackpointer_event; + } + + // For RandomJump do only one experiment not 8 + if (pc_injection) break; + } + + // send results + jobclient.sendResult(param); + } + + // explicitly terminate, or the simulator will continue to run + simulator.terminate(); +} + diff --git a/src/experiments/erika-tester/experiment.hpp b/src/experiments/erika-tester/experiment.hpp new file mode 100644 index 00000000..601f5856 --- /dev/null +++ b/src/experiments/erika-tester/experiment.hpp @@ -0,0 +1,33 @@ +#ifndef __ERIKA_TESTER_EXPERIMENT_HPP__ +#define __ERIKA_TESTER_EXPERIMENT_HPP__ + + +#include "sal/SALInst.hpp" +#include "efw/ExperimentFlow.hpp" +#include "efw/JobClient.hpp" +#include "util/Logger.hpp" +#include "util/ElfReader.hpp" +#include +#include +#include "util/llvmdisassembler/LLVMtoFailTranslator.hpp" + +class ErikaTester : public fail::ExperimentFlow { +public: + +private: + fail::Logger m_log; + fail::MemoryManager& m_mm; + fail::ElfReader m_elf; + fail::LLVMtoFailTranslator* m_ltof; + + unsigned injectBitFlip(fail::address_t data_address, unsigned data_width, unsigned bitpos); + void redecodeCurrentInstruction(); + const fail::ElfSymbol& getELFSymbol(const std::string name); + +public: + ErikaTester() : m_log("ErikaTester", false), m_mm(fail::simulator.getMemoryManager()) {} + + bool run(); +}; + +#endif diff --git a/src/experiments/erika-tester/main.cc b/src/experiments/erika-tester/main.cc new file mode 100644 index 00000000..7d1a8791 --- /dev/null +++ b/src/experiments/erika-tester/main.cc @@ -0,0 +1,20 @@ +#include +#include + +#include "cpn/CampaignManager.hpp" +#include "util/CommandLine.hpp" +#include "campaign.hpp" + +int main(int argc, char **argv) +{ + fail::CommandLine &cmd = fail::CommandLine::Inst(); + for (int i = 1; i < argc; ++i) + cmd.add_args(argv[i]); + + ErikaTesterCampaign c; + if (fail::campaignmanager.runCampaign(&c)) { + return 0; + } else { + return 1; + } +} diff --git a/src/experiments/erika-tracing/CMakeLists.txt b/src/experiments/erika-tracing/CMakeLists.txt new file mode 100644 index 00000000..a526eb38 --- /dev/null +++ b/src/experiments/erika-tracing/CMakeLists.txt @@ -0,0 +1,18 @@ +set(EXPERIMENT_NAME erika-tracing) +set(EXPERIMENT_TYPE ErikaTracing) +configure_file(../instantiate-experiment.ah.in + ${CMAKE_CURRENT_BINARY_DIR}/instantiate-${EXPERIMENT_NAME}.ah @ONLY +) + +set(MY_CAMPAIGN_SRCS + experiment.hpp + experiment.cc +) + +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +## Build library +add_library(fail-${EXPERIMENT_NAME} ${PROTO_SRCS} ${PROTO_HDRS} ${MY_CAMPAIGN_SRCS}) + +add_dependencies(fail-${EXPERIMENT_NAME} fail-comm) +target_link_libraries(fail-${EXPERIMENT_NAME} fail-tracing fail-randomgenerator fail-checkpoint fail-comm) diff --git a/src/experiments/erika-tracing/config.cmake b/src/experiments/erika-tracing/config.cmake new file mode 100644 index 00000000..25a685c2 --- /dev/null +++ b/src/experiments/erika-tracing/config.cmake @@ -0,0 +1,12 @@ +SET(bochs_configure_params --enable-a20-pin;--enable-x86-64;--enable-cpu-level=6;--enable-ne2000;--enable-acpi;--enable-pci;--enable-usb;--enable-trace-cache;--enable-fast-function-calls;--enable-host-specific-asms;--enable-readline;--enable-clgd54xx;--enable-fpu;--enable-vmx=2;--enable-monitor-mwait;--enable-cdrom;--enable-sb16=linux;--enable-gdb-stub;--with-nogui CACHE STRING "") + +SET(BUILD_LLVM_DISASSEMBLER ON CACHE BOOL "" FORCE) + +SET(PLUGINS_ACTIVATED "tracing;checkpoint;randomgenerator" CACHE STRING "") + +# Build the import and prune trace tools always +SET(BUILD_IMPORT_TRACE ON CACHE BOOL "" FORCE) +SET(BUILD_PRUNE_TRACE ON CACHE BOOL "" FORCE) +SET(BUILD_CONVERT_TRACE ON CACHE BOOL "" FORCE) +SET(BUILD_DUMP_TRACE ON CACHE BOOL "" FORCE) +SET(BUILD_LLVM_DISASSEMBLER ON CACHE BOOL "" FORCE) diff --git a/src/experiments/erika-tracing/experiment.cc b/src/experiments/erika-tracing/experiment.cc new file mode 100644 index 00000000..38d418d7 --- /dev/null +++ b/src/experiments/erika-tracing/experiment.cc @@ -0,0 +1,291 @@ +#include +#include + +#include "sal/SALInst.hpp" +#include "sal/Register.hpp" +#include "sal/Listener.hpp" +#include "experiment.hpp" +#include "util/CommandLine.hpp" +#include "util/gzstream/gzstream.h" + +// required (enabled) plugins +#include "../plugins/tracing/TracingPlugin.hpp" +#include "../plugins/randomgenerator/RandomGenerator.hpp" +#include "../plugins/checkpoint/Checkpoint.hpp" + +using namespace std; +using namespace fail; + +void ErikaTracing::parseOptions() { + CommandLine &cmd = CommandLine::Inst(); + cmd.addOption("", "", Arg::None, "USAGE: fail-client -Wf,[option] -Wf,[option] ... \n\n"); + CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h,--help \tPrint usage and exit"); + + + CommandLine::option_handle ELF_FILE = cmd.addOption("", "elf-file", Arg::Required, + "--elf-file \tELF Binary File (default: $FAIL_ELF_PATH)"); + CommandLine::option_handle START_SYMBOL = cmd.addOption("s", "start-symbol", Arg::Required, + "-s,--start-symbol \tELF symbol to start tracing (default: main)"); + CommandLine::option_handle STOP_SYMBOL = cmd.addOption("e", "end-symbol", Arg::Required, + "-e,--end-symbol \tELF symbol to end tracing"); + CommandLine::option_handle SAVE_SYMBOL = cmd.addOption("S", "save-symbol", Arg::Required, + "-S,--save-symbol \tELF symbol to save the state of the machine (default: main)\n"); + CommandLine::option_handle STATE_FILE = cmd.addOption("f", "state-file", Arg::Required, + "-f,--state-file \tFile/dir to save the state to (default: state)"); + CommandLine::option_handle TRACE_FILE = cmd.addOption("t", "trace-file", Arg::Required, + "-t,--trace-file \tFile to save the execution trace to (default: trace.pb)\n"); + + CommandLine::option_handle FULL_TRACE = cmd.addOption("", "full-trace", Arg::None, "--full-trace \tDo a full trace (more data, default: off)"); + CommandLine::option_handle MEM_SYMBOL = cmd.addOption("m", "memory-symbol", Arg::Required, + "-m,--memory-symbol \tELF symbol(s) to trace accesses (default: all mem read/writes are traced)"); + CommandLine::option_handle MEM_REGION = cmd.addOption("M", "memory-region", Arg::Required, + "-M,--memory-region \trestrict memory region which is traced" + " (Possible formats: 0x
, 0x
:0x
, 0x
:)"); + + if (!cmd.parse()) { + cerr << "Error parsing arguments." << endl; + exit(-1); + } + + if (cmd[HELP]) { + cmd.printUsage(); + exit(0); + } + + if (cmd[ELF_FILE].count() > 0) + elf_file = cmd[ELF_FILE].first()->arg; + else { + char * elfpath = getenv("FAIL_ELF_PATH"); + if (elfpath == NULL) { + m_log << "FAIL_ELF_PATH not set :( (alternative: --elf-file) " << std::endl; + exit(-1); + } + + elf_file = elfpath; + } + m_elf = new ElfReader(elf_file.c_str()); + + if (cmd[START_SYMBOL].count() > 0) + start_symbol = cmd[START_SYMBOL].first()->arg; + else + start_symbol = "main"; + + if (cmd[STOP_SYMBOL].count() > 0) + stop_symbol = std::string(cmd[STOP_SYMBOL].first()->arg); + else { + m_log << "You have to give an end symbol (-e,--end-symbol)!" << std::endl; + exit(EXIT_FAILURE); + } + + if (cmd[SAVE_SYMBOL].count() > 0) + save_symbol = std::string(cmd[SAVE_SYMBOL].first()->arg); + else + save_symbol = "main"; + + if (cmd[STATE_FILE].count() > 0) + state_file = std::string(cmd[STATE_FILE].first()->arg); + else + state_file = "state"; + + if (cmd[TRACE_FILE].count() > 0) + trace_file = std::string(cmd[TRACE_FILE].first()->arg); + else + trace_file = "trace.pb"; + + use_memory_map = false; + + if (cmd[MEM_SYMBOL].count() > 0) { + use_memory_map = true; + option::Option *opt = cmd[MEM_SYMBOL].first(); + + while (opt != 0) { + const ElfSymbol &symbol = m_elf->getSymbol(opt->arg); + assert(symbol.isValid()); + + m_log << "Adding '" << opt->arg << "' == 0x" << std::hex << symbol.getAddress() + << "+" << std::dec << symbol.getSize() << " to trace map" << std::endl; + traced_memory_map.add(symbol.getAddress(), symbol.getSize()); + + opt = opt->next(); + } + } + + if (cmd[MEM_REGION].count() > 0) { + use_memory_map = true; + option::Option *opt = cmd[MEM_REGION].first(); + + while (opt != 0) { + char *endptr; + guest_address_t begin = strtol(opt->arg, &endptr, 16); + guest_address_t size; + if (endptr == opt->arg) { + m_log << "Couldn't parse " << opt->arg << std::endl; + exit(-1); + } + + char delim = *endptr; + if (delim == 0) { + size = 1; + } else if (delim == ':') { + char *p = endptr +1; + size = strtol(p, &endptr, 16) - begin; + if (p == endptr || *endptr != 0) { + m_log << "Couldn't parse " << opt->arg << std::endl; + exit(-1); + } + } else if (delim == '+') { + char *p = endptr +1; + size = strtol(p, &endptr, 10); + if (p == endptr || *endptr != 0) { + m_log << "Couldn't parse " << opt->arg << std::endl; + exit(-1); + } + } else { + m_log << "Couldn't parse " << opt->arg << std::endl; + exit(-1); + } + + traced_memory_map.add(begin, size); + + m_log << "Adding " << opt->arg << " 0x" << std::hex << begin + << "+" << std::dec << size << " to trace map" << std::endl; + + opt = opt->next(); + } + } + + if (cmd[FULL_TRACE]) { + this->full_trace = true; + } + + assert(m_elf->getSymbol(start_symbol).isValid()); + assert(m_elf->getSymbol(stop_symbol).isValid()); + assert(m_elf->getSymbol(save_symbol).isValid()); + + m_log << "start symbol: " << start_symbol << " 0x" << std::hex << m_elf->getSymbol(start_symbol).getAddress() << std::endl; + m_log << "save symbol: " << save_symbol << " 0x" << std::hex << m_elf->getSymbol(save_symbol).getAddress() << std::endl; + m_log << "stop symbol: " << stop_symbol << " 0x" << std::hex << m_elf->getSymbol(stop_symbol).getAddress() << std::endl; + + m_log << "state file: " << state_file << std::endl; + m_log << "trace file: " << trace_file << std::endl; + m_log << "full-trace: " << this->full_trace << std::endl; + + +} + +bool ErikaTracing::run() +{ + parseOptions(); + + BPSingleListener l_start_symbol(m_elf->getSymbol(start_symbol).getAddress()); + BPSingleListener l_save_symbol (m_elf->getSymbol(save_symbol).getAddress()); + BPSingleListener l_stop_symbol (m_elf->getSymbol(stop_symbol).getAddress()); + + + //////////////////////////////////////////////////////////////// + // STEP 1: run until interesting function starts, start the tracing + simulator.addListenerAndResume(&l_start_symbol); + m_log << start_symbol << " reached, start tracing" << std::endl; + + // restrict memory access logging to injection target + TracingPlugin tp; + tp.setFullTrace(this->full_trace); + + if (use_memory_map) { + m_log << "Use restricted memory map for tracing" << std::endl; + tp.restrictMemoryAddresses(&traced_memory_map); + } + + ogzstream of(trace_file.c_str()); + if (of.bad()) { + m_log << "Couldn't open trace file: " << trace_file << std::endl; + exit(-1); + } + tp.setTraceFile(&of); + + // this must be done *after* configuring the plugin: + simulator.addFlow(&tp); + + //////////////////////////////////////////////////////////////// + // STEP 2: continue to the save point, and save state + if (start_symbol != save_symbol) { + simulator.addListenerAndResume(&l_save_symbol); + } + m_log << start_symbol << " reached, save state" << std::endl; + simulator.save(state_file); + + + //////////////////////////////////////////////////////////////// + // Step 3: add plugins + // symbol to trigger checkpoints + const ElfSymbol &s_fail_trace = m_elf->getSymbol("fail_trace"); + const ElfSymbol &s_tos = m_elf->getSymbol("EE_x86_system_tos"); + + Checkpoint *cpoint; + if(s_fail_trace.isValid() && s_tos.isValid()) { + Checkpoint::range_vector check_ranges; + + const ElfSymbol &s_os_stack = m_elf->getSymbol("os_stack"); + assert (s_os_stack.isValid()); + m_log << "found task stack: " << "os_stack" << std::endl; + + Checkpoint::indirectable_address_t start = std::make_pair(s_tos.getAddress() + 0*4, true); + Checkpoint::indirectable_address_t end = std::make_pair(s_os_stack.getEnd(), false); + check_ranges.push_back(std::make_pair(start, end)); + + for(int i=1; i<255; i++) { + stringstream stackname; + stackname << "EE_x86_stack_"; + stackname << i; + const ElfSymbol &s_stack = m_elf->getSymbol(stackname.str()); + if(!s_stack.isValid()) { + m_log << "found " << std::dec << (i-1) << " task stacks" << std::endl; + break; + } + + m_log << "found task stack: " << stackname.str() << std::endl; + + Checkpoint::indirectable_address_t start = std::make_pair(s_tos.getAddress() + i*4, true); + Checkpoint::indirectable_address_t end = std::make_pair(s_stack.getEnd(), false); + check_ranges.push_back(std::make_pair(start, end)); + } + + cpoint = new Checkpoint(s_fail_trace, check_ranges, "checkpoint.trace"); + simulator.addFlow(cpoint); + } else { + m_log << "Checkpoint plugin NOT added to simulation" << std::endl; + } + + // symbol to read random values from + const ElfSymbol &s_random_source = m_elf->getSymbol("random_source"); + RandomGenerator *rgen; + if (s_random_source.isValid()) { + const unsigned seed = 12342; + rgen = new RandomGenerator(s_random_source, seed); + simulator.addFlow(rgen); + } else { + m_log << "Randomgenerator plugin NOT added to simulation" << std::endl; + } + + + //////////////////////////////////////////////////////////////// + // Step 4: Continue to the stop point + simulator.addListener(&l_stop_symbol); + simulator.resume(); + + //////////////////////////////////////////////////////////////// + // Step 5: tear down the tracing + + simulator.removeFlow(&tp); + // serialize trace to file + if (of.fail()) { + m_log << "failed to write " << trace_file << std::endl; + return false; + } + of.close(); + + simulator.clearListeners(); + simulator.terminate(); + + return true; +} diff --git a/src/experiments/erika-tracing/experiment.hpp b/src/experiments/erika-tracing/experiment.hpp new file mode 100644 index 00000000..d05f4b93 --- /dev/null +++ b/src/experiments/erika-tracing/experiment.hpp @@ -0,0 +1,37 @@ +#ifndef __ERIKA_TRACING_HPP__ +#define __ERIKA_TRACING_HPP__ + +#include "efw/ExperimentFlow.hpp" +#include "util/Logger.hpp" +#include "util/ElfReader.hpp" +#include "util/MemoryMap.hpp" +#include +#include + + + +class ErikaTracing : public fail::ExperimentFlow { + std::string start_symbol; + std::string stop_symbol; + std::string save_symbol; + + std::string state_file; + std::string trace_file; + std::string elf_file; + + bool use_memory_map; + fail::MemoryMap traced_memory_map; + + bool full_trace; + + fail::Logger m_log; + fail::ElfReader *m_elf; + +public: + void parseOptions(); + bool run(); + + ErikaTracing() : full_trace(false), m_log("ErikaTracing", false) {}; +}; + +#endif // __ERIKA_TRACING_HPP__ \ No newline at end of file diff --git a/src/experiments/erika-tracing/fail_experiment.py b/src/experiments/erika-tracing/fail_experiment.py new file mode 100755 index 00000000..9e0441f7 --- /dev/null +++ b/src/experiments/erika-tracing/fail_experiment.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import os,sys +import logging +import imp +import math +import collections +import numpy + +tmp_path = "%s/git/versuchung/src"% os.environ["HOME"] +if os.path.exists(tmp_path): + sys.path.append(tmp_path) + + +from versuchung.experiment import Experiment +from versuchung.types import String, Bool,List +from versuchung.database import Database +from versuchung.files import File, Directory, Executable +from versuchung.archives import GitArchive +from versuchung.execute import shell +from versuchung.tex import DatarefDict +from threading import Thread + +class FailTrace(Experiment): + inputs = { + "erika": GitArchive("gitosis@i4git.informatik.uni-erlangen.de:erika"), + "bochs-runner": Executable("/proj/i4danceos/tools/fail/bochs-experiment-runner.py"), + "erika-tracing": Executable("/proj/i4danceos/tools/fail/erika-tracing"), + } + outputs = { + "trace": Directory("trace"), + "elf": File("erika.elf"), + "iso": File("erika.iso"), + } + + def run(self): + logging.info("Cloning ERIKA...") + + with self.erika as erika_path: + shell("cd %s/examples/x86/coptermock-isorc; make", erika_path) + + self.iso.copy_contents(os.path.join(erika_path, "examples/x86/coptermock-isorc/Debug/erika.iso")) + self.elf.copy_contents(os.path.join(erika_path, "examples/x86/coptermock-isorc/Debug/Debug/out.elf")) + + shell(("cd %(resultdir)s; python %(bochs)s -F 50 -i %(iso)s -e %(elf)s -f %(fail)s" + + " -m 8 -1 -- -Wf,--end-symbol=test_finish -Wf,--start-symbol=EE_oo_StartOS" + + " -Wf,--trace-file=trace.pb -Wf,--save-symbol=EE_oo_StartOS") % { + "resultdir": self.trace.path, + "bochs": self.bochs_runner.path, + "iso": self.iso.path, + "elf": self.elf.path, + "fail": self.erika_tracing.path + } + ) + + +class FailImport(Experiment): + inputs = { + "trace": FailTrace("FailTrace"), + "fail-tool-dir": Directory("/proj/i4danceos/tools/fail"), + } + + def run(self): + variant = "erika/error-hook" + for (label, importer, importer_args) in [\ + ("mem", "MemoryImporter", []), + ("regs", "RegisterImporter", []), + ("ip", "RegisterImporter", ["--no-gp", "--ip"]), + ("flags", "RegisterImporter", ["--no-gp", "--flags"]), + ]: + benchmark = label + logging.info("Importing coredos/%s", benchmark) + cmdline = "%(path)s/import-trace -v %(variant)s -b %(benchmark)s -i %(importer)s "\ + + "-t %(trace)s -e %(elf)s %(args)s" + shell(cmdline %\ + {"path": self.fail_tool_dir.path, + "variant": variant, + "benchmark": benchmark, + "importer": importer, + "trace": os.path.join(self.trace.trace.path, "trace.pb"), + "elf": self.trace.elf.path, + "args": " ".join(importer_args)}) + shell("%s/prune-trace -v %s -b %% -p basic --overwrite", + self.fail_tool_dir.path, variant) + + +if __name__ == "__main__": + import shutil, sys + if len(sys.argv) > 1: + action = sys.argv[1]; del sys.argv[1] + actions = {"trace": FailTrace, + "import": FailImport} + + if action in actions: + experiment = actions[action]() + dirname = experiment(sys.argv) + else: + print "No action to be taken, use: %s" % ", ".join(actions.keys()) +