From 491312bb9103d68ae927dcaf113aa505b5d46952 Mon Sep 17 00:00:00 2001 From: Christian Dietrich Date: Thu, 5 Jun 2014 16:01:20 +0200 Subject: [PATCH] GenericExperiment: A standardized fault injection experiment The GenericExperiment is a standard campaign+experiment pair. It is derived from the DatabaseCampaign+DatabaseExperiment. Its experiment endpoints are set on the command line, therefore it can be used to give users a first impression of FAIL*. Currently it supports different endpoints: --trap: Catch all traps that occur and end the experiment --timeout : kill the experiment after N microseconds --catch-write-text: detect writes on the text segment --catch-write-outerspace: detect writes into nirvana --{ok,fail,detected}-marker: groups of ELF symbols that are used as execution breakpoints Change-Id: Idc7fcf8875953f1007e1a37bacb086eddd29cd10 --- .../generic-experiment/CMakeLists.txt | 37 +++ .../generic-experiment/campaign.cc | 21 ++ .../generic-experiment/campaign.hpp | 16 ++ .../generic-experiment/config.cmake | 2 + .../generic-experiment/experiment.cc | 266 ++++++++++++++++++ .../generic-experiment/experiment.hpp | 111 ++++++++ .../generic-experiment/experimentInfo.hpp | 15 + .../generic-experiment.proto | 36 +++ src/experiments/generic-experiment/main.cc | 20 ++ 9 files changed, 524 insertions(+) create mode 100644 src/experiments/generic-experiment/CMakeLists.txt create mode 100644 src/experiments/generic-experiment/campaign.cc create mode 100644 src/experiments/generic-experiment/campaign.hpp create mode 100644 src/experiments/generic-experiment/config.cmake create mode 100644 src/experiments/generic-experiment/experiment.cc create mode 100644 src/experiments/generic-experiment/experiment.hpp create mode 100644 src/experiments/generic-experiment/experimentInfo.hpp create mode 100644 src/experiments/generic-experiment/generic-experiment.proto create mode 100644 src/experiments/generic-experiment/main.cc diff --git a/src/experiments/generic-experiment/CMakeLists.txt b/src/experiments/generic-experiment/CMakeLists.txt new file mode 100644 index 00000000..2c537ae8 --- /dev/null +++ b/src/experiments/generic-experiment/CMakeLists.txt @@ -0,0 +1,37 @@ +set(EXPERIMENT_NAME generic-experiment) +set(EXPERIMENT_TYPE GenericExperiment) +configure_file(../instantiate-experiment.ah.in + ${CMAKE_CURRENT_BINARY_DIR}/instantiate-${EXPERIMENT_NAME}.ah @ONLY +) + +## Setup desired protobuf descriptions HERE ## +set(MY_PROTOS + generic-experiment.proto +) + +set(MY_CAMPAIGN_SRCS + experiment.hpp + experiment.cc + campaign.hpp + campaign.cc +) + +#### PROTOBUFS #### +find_package(Protobuf REQUIRED) +include_directories(${PROTOBUF_INCLUDE_DIRS}) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +find_package(MySQL REQUIRED) +include_directories(${MYSQL_INCLUDE_DIR}) + +PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS ${MY_PROTOS}) + +## Build library +add_library(fail-${EXPERIMENT_NAME} ${PROTO_SRCS} ${PROTO_HDRS} ${MY_CAMPAIGN_SRCS}) +add_dependencies(fail-${EXPERIMENT_NAME} fail-comm) +target_link_libraries(fail-${EXPERIMENT_NAME} ${PROTOBUF_LIBRARY} fail-sal) + +## This is the example's campaign server distributing experiment parameters +add_executable(${EXPERIMENT_NAME}-server main.cc) +target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} ${MYSQL_LIBRARIES} -Wl,--end-group) +install(TARGETS ${EXPERIMENT_NAME}-server RUNTIME DESTINATION bin) diff --git a/src/experiments/generic-experiment/campaign.cc b/src/experiments/generic-experiment/campaign.cc new file mode 100644 index 00000000..1286e9da --- /dev/null +++ b/src/experiments/generic-experiment/campaign.cc @@ -0,0 +1,21 @@ +#include +#include + +#include "campaign.hpp" +#include "experimentInfo.hpp" +#include "cpn/CampaignManager.hpp" +#include "util/Logger.hpp" +#include "util/ProtoStream.hpp" +#include "sal/SALConfig.hpp" + +#include "experimentInfo.hpp" + +using namespace std; +using namespace fail; +using namespace google::protobuf; + +void GenericExperimentCampaign::cb_send_pilot(DatabaseCampaignMessage pilot) { + GenericExperimentData *data = new GenericExperimentData; + data->msg.mutable_fsppilot()->CopyFrom(pilot); + campaignmanager.addParam(data); +} diff --git a/src/experiments/generic-experiment/campaign.hpp b/src/experiments/generic-experiment/campaign.hpp new file mode 100644 index 00000000..8a49a8a4 --- /dev/null +++ b/src/experiments/generic-experiment/campaign.hpp @@ -0,0 +1,16 @@ +#ifndef __KESOGCCAMPAIGN_HPP__ + #define __KESOGCCAMPAIGN_HPP__ + +#include "cpn/DatabaseCampaign.hpp" +#include + +class GenericExperimentCampaign : public fail::DatabaseCampaign { + virtual const google::protobuf::Descriptor * cb_result_message() { + return google::protobuf::DescriptorPool::generated_pool() + ->FindMessageTypeByName("GenericExperimentMessage"); + } + + virtual void cb_send_pilot(DatabaseCampaignMessage pilot); +}; + +#endif // __KESOGCCAMPAIGN_HPP__ diff --git a/src/experiments/generic-experiment/config.cmake b/src/experiments/generic-experiment/config.cmake new file mode 100644 index 00000000..63d3927a --- /dev/null +++ b/src/experiments/generic-experiment/config.cmake @@ -0,0 +1,2 @@ +SET(bochs_configure_params "--enable-a20-pin;--enable-x86-64;--enable-cpu-level=6;--enable-ne2000;--enable-acpi;--enable-pci;--enable-usb;--enable-trace-cache;--enable-fast-function-calls;--enable-host-specific-asms;--enable-readline;--enable-clgd54xx;--enable-fpu;--enable-vmx=2;--enable-monitor-mwait;--enable-cdrom;--enable-sb16=linux;--enable-gdb-stub;--with-nogui" CACHE STRING "") + diff --git a/src/experiments/generic-experiment/experiment.cc b/src/experiments/generic-experiment/experiment.cc new file mode 100644 index 00000000..c11cb690 --- /dev/null +++ b/src/experiments/generic-experiment/experiment.cc @@ -0,0 +1,266 @@ +#include +#include + +// getpid +#include +#include + + +#include +#include "experiment.hpp" +#include "experimentInfo.hpp" +#include "sal/SALConfig.hpp" +#include "sal/SALInst.hpp" +#include "sal/Memory.hpp" +#include "sal/Listener.hpp" + +#include "sal/bochs/BochsListener.hpp" +#include +#include + +#include "campaign.hpp" +#include "generic-experiment.pb.h" +#include "util/CommandLine.hpp" + +using namespace std; +using namespace fail; + + +GenericExperiment::~GenericExperiment() {} + +static GenericExperimentData space_for_param; +ExperimentData* GenericExperiment::cb_allocate_experiment_data() { + return &space_for_param; +} + +/** + * Allocate a new result slot in the given experiment data + */ +google::protobuf::Message* GenericExperiment::cb_new_result(ExperimentData* data) { + GenericExperimentData *param = static_cast(data); + GenericExperimentMessage_Result *result = param->msg.add_result(); + return result; +} + + +void handleEvent(GenericExperimentMessage_Result& result, + GenericExperimentMessage_Result_ResultType restype, + unsigned int details) { + cout << "Result details: " << restype << " "<< details << endl; + result.set_resulttype(restype); + result.set_details(details); +} + +void GenericExperiment::parseSymbols(const std::string &args, std::set * into) { + std::vector elems; + std::stringstream ss(args); + std::string item; + while (std::getline(ss, item, ',')) { + const ElfSymbol * symbol = &m_elf.getSymbol(item); + if (!symbol->isValid()) { + m_log << "ELF Symbol not found: " << item << endl; + simulator.terminate(1); + } + m_log << "Adding symbol " << item << " at 0x" << hex << symbol->getAddress() << endl; + BPSingleListener *l = new BPSingleListener(symbol->getAddress()); + into->insert(l); + end_markers.insert(l); + listener_to_symbol[l] = symbol; + } +} + + + +bool GenericExperiment::cb_start_experiment() { + CommandLine &cmd = CommandLine::Inst(); + cmd.addOption("", "", Arg::None, "USAGE: fail-client -Wf,[option] -Wf,[option] ... \n\n"); + CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h,--help \tPrint usage and exit"); + + CommandLine::option_handle STATE_DIR = cmd.addOption("", "state-dir", Arg::Required, + "--state-dir \t Path to the state directory"); + + // catch any trap + CommandLine::option_handle TRAP = cmd.addOption("", "trap", Arg::None, + "--trap \tCatch Traps"); + CommandLine::option_handle WRITE_MEM_TEXT = cmd.addOption("", "catch-write-textsegment", Arg::None, + "--catch-write-textsegment \tCatch writes to the text segment"); + + CommandLine::option_handle WRITE_MEM_OUTERSPACE + = cmd.addOption("", "catch-write-outerspace", Arg::None, + "--catch-write-outerspace \tCatch writes to the outerspace"); + + CommandLine::option_handle TIMEOUT = cmd.addOption("", "timeout", Arg::Required, + "--timeout \t Experiment Timeout in uS"); + + + std::map option_handles; + for (std::map::iterator it = end_marker_groups.begin(); + it != end_marker_groups.end(); ++it) { + CommandLine::option_handle handle = + cmd.addOption("", it->first, Arg::Required, + "--" + it->first + " \tList of symbols (comma separated)"); + option_handles[it->first] = handle; + } + + if (!cmd.parse()) { + cerr << "Error parsing arguments." << endl; + exit(-1); + } + + if (cmd[HELP]) { + cmd.printUsage(); + exit(0); + } + + address_t minimal_ip = INT_MAX; // Every address is lower + address_t maximal_ip = 0; + address_t minimal_data = 0x100000; // 1 Mbyte + address_t maximal_data = 0; + + for (ElfReader::section_iterator it = m_elf.sec_begin(); + it != m_elf.sec_end(); ++it) { + const ElfSymbol &symbol = *it; + std::string prefix(".text"); + if (symbol.getName().compare(0, prefix.size(), prefix) == 0) { + minimal_ip = std::min(minimal_ip, symbol.getStart()); + maximal_ip = std::max(maximal_ip, symbol.getEnd()); + } else { + minimal_data = std::min(minimal_data, symbol.getStart()); + maximal_data = std::max(maximal_data, symbol.getEnd()); + } + } + + if (cmd[WRITE_MEM_TEXT]) { + m_log << "Catch writes to text segment from " << hex << minimal_ip << " to " << maximal_ip << std::endl; + enabled_mem_text = true; + + l_mem_text.setWatchAddress(minimal_ip); + l_mem_text.setTriggerAccessType(MemAccessEvent::MEM_WRITE); + l_mem_text.setWatchWidth(maximal_ip - minimal_ip); + } + + if (cmd[WRITE_MEM_OUTERSPACE]) { + m_log << "Catch writes to outerspace from " << hex << " from " << maximal_data << std::endl; + enabled_mem_outerspace = true; + + l_mem_outerspace.setWatchAddress(maximal_data); + l_mem_outerspace.setTriggerAccessType(MemAccessEvent::MEM_WRITE); + l_mem_outerspace.setWatchWidth(0xfffffff0); + } + + if (cmd[TRAP]) { + m_log << "Catch all traps" << endl; + enabled_trap = true; + } + + if (cmd[STATE_DIR]) { + std::string value(cmd[STATE_DIR].first()->arg); + m_state_dir = value; + m_log << "Set state dir to " << value << endl; + } + + if (cmd[TIMEOUT]) { + std::string value(cmd[TIMEOUT].first()->arg); + std::stringstream ss(value); + ss >> m_Timeout; + if (ss.bad()) { + m_log << "Could not parse --timeout argument" << endl; + return false; // Initialization failed + } + l_timeout.setTimeout(m_Timeout); + enabled_timeout = true; + m_log << "Enabled Experiment Timeout of " << dec << m_Timeout << " microseconds" << endl; + } + + for (std::map::iterator it = option_handles.begin(); + it != option_handles.end(); ++it) { + if (cmd[option_handles[it->first]]) { + option::Option *opt = cmd[option_handles[it->first]].first(); + while (opt != 0) { + parseSymbols(std::string(opt->arg), end_marker_groups[it->first]); + opt = opt->next(); + } + } + } + + return true; // Everything OK +} + + +bool GenericExperiment::cb_before_resume() { + if (enabled_trap) + simulator.addListener(&l_trap); + + if (enabled_mem_text) + simulator.addListener(&l_mem_text); + + if (enabled_mem_outerspace) { + std::cout << "enabled mem outerspace " << endl; + simulator.addListener(&l_mem_outerspace); + } + + if (enabled_timeout) + simulator.addListener(&l_timeout); + + for (std::set::iterator it = end_markers.begin(); + it != end_markers.end(); ++it) { + simulator.addListener(*it); + } + + return true; // everything OK +} + +void GenericExperiment::cb_after_resume(fail::BaseListener *event) { + GenericExperimentMessage_Result * result = static_cast(this->get_current_result()); + + // Record the crash time + result->set_crash_time(simulator.getTimerTicks()); + + + if (event == &l_timeout) { + handleEvent(*result, result->TIMEOUT, m_Timeout); + } else if (event == &l_trap) { + handleEvent(*result, result->TRAP, l_trap.getTriggerNumber()); + } else if (event == &l_mem_text) { + handleEvent(*result, result->WRITE_TEXTSEGMENT, + l_mem_text.getTriggerAddress()); + + } else if (event == &l_mem_outerspace){ + handleEvent(*result, result->WRITE_OUTERSPACE, + l_mem_outerspace.getTriggerAddress()); + + ////////////////////////////////////////////////// + // End Marker Groups + ////////////////////////////////////////////////// + } else if (OK_marker.find(event) != OK_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->OK_MARKER, symbol->getAddress()); + + } else if (FAIL_marker.find(event) != FAIL_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->FAIL_MARKER, symbol->getAddress()); + + } else if (DETECTED_marker.find(event) != DETECTED_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->DETECTED_MARKER, symbol->getAddress()); + + } else if (GROUP1_marker.find(event) != GROUP1_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->GROUP1_MARKER, symbol->getAddress()); + + } else if (GROUP2_marker.find(event) != GROUP2_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->GROUP2_MARKER, symbol->getAddress()); + + } else if (GROUP3_marker.find(event) != GROUP3_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->GROUP3_MARKER, symbol->getAddress()); + + } else if (GROUP4_marker.find(event) != GROUP4_marker.end()) { + const ElfSymbol *symbol = listener_to_symbol[event]; + handleEvent(*result, result->GROUP4_MARKER, symbol->getAddress()); + + } else { + handleEvent(*result, result->UNKNOWN, 0); + } +} diff --git a/src/experiments/generic-experiment/experiment.hpp b/src/experiments/generic-experiment/experiment.hpp new file mode 100644 index 00000000..72780e7b --- /dev/null +++ b/src/experiments/generic-experiment/experiment.hpp @@ -0,0 +1,111 @@ +#ifndef __GENERIC_EXPERIMENT_EXPERIMENT_HPP__ +#define __GENERIC_EXPERIMENT_EXPERIMENT_HPP__ + +#include "sal/SALInst.hpp" +#include "efw/DatabaseExperiment.hpp" +#include "sal/Listener.hpp" +#include "efw/JobClient.hpp" +#include "util/Logger.hpp" +#include "util/ElfReader.hpp" +#include +#include +#include +#include + + +class GenericExperiment : public fail::DatabaseExperiment { + fail::ElfReader m_elf; + + std::string m_state_dir; + + bool enabled_mem_text; + fail::MemAccessListener l_mem_text; + + bool enabled_mem_outerspace; + fail::MemAccessListener l_mem_outerspace; + + bool enabled_trap; + fail::TrapListener l_trap; + + bool enabled_timeout; + unsigned m_Timeout; + fail::TimerListener l_timeout; + + std::map listener_to_symbol; + + typedef std::set ListenerSet; + + ListenerSet end_markers; + ListenerSet OK_marker; + ListenerSet FAIL_marker; + ListenerSet DETECTED_marker; + ListenerSet GROUP1_marker; + ListenerSet GROUP2_marker; + ListenerSet GROUP3_marker; + ListenerSet GROUP4_marker; + + std::map end_marker_groups; + + void parseSymbols(const std::string &args, std::set *into); + +public: + GenericExperiment() : DatabaseExperiment("GenericExperiment"), + m_state_dir("state"), + l_trap(fail::ANY_TRAP), l_timeout(0) { + enabled_mem_text = false; + enabled_mem_outerspace = false; + enabled_trap = false; + enabled_timeout = false; + + end_marker_groups["ok-marker"] = &OK_marker; + end_marker_groups["fail-marker"] = &FAIL_marker; + end_marker_groups["detected-marker"] = &FAIL_marker; + end_marker_groups["group1-marker"] = &GROUP1_marker; + end_marker_groups["group2-marker"] = &GROUP2_marker; + end_marker_groups["group3-marker"] = &GROUP3_marker; + end_marker_groups["group4-marker"] = &GROUP4_marker; + } + virtual ~GenericExperiment(); + + /** + * Get path to the state directory + */ + virtual std::string cb_state_directory() { return m_state_dir; } + + /** + * Allocate enough space to hold the incoming ExperimentData message. + */ + virtual fail::ExperimentData* cb_allocate_experiment_data(); + + /** + * Allocate a new result slot in the given experiment data + */ + virtual google::protobuf::Message* cb_new_result(fail::ExperimentData* data); + + /** + * Callback that is called, before the actual experiment + * starts. Simulation is terminated on false. + * @param The current result message + * @return \c true on success, \c false otherwise + */ + virtual bool cb_start_experiment(); + + /** + * Callback that is called before the resuming till crash has + * started. This is called after the fault was injected. Here the + * end listeners should be installed. Returns true on + * success. Otherwise the experiment is canceled. + + * @return \c true on success, \c false otherwise + */ + virtual bool cb_before_resume(); + + /** + * Callback that is called after the resume-till-crash phase with + * the last triggered listener. This callback should collect all + * data and fill up the result message. + */ + virtual void cb_after_resume(fail::BaseListener *event); +}; + +#endif // __GENERIC_EXPERIMENT_EXPERIMENT_HPP__ diff --git a/src/experiments/generic-experiment/experimentInfo.hpp b/src/experiments/generic-experiment/experimentInfo.hpp new file mode 100644 index 00000000..c371b250 --- /dev/null +++ b/src/experiments/generic-experiment/experimentInfo.hpp @@ -0,0 +1,15 @@ +#ifndef __EXPERIMENT_INFO_HPP__ +#define __EXPERIMENT_INFO_HPP__ + +#include "comm/ExperimentData.hpp" +#include "generic-experiment.pb.h" + +class GenericExperimentData : public fail::ExperimentData { +public: + GenericExperimentMessage msg; + GenericExperimentData() : fail::ExperimentData(&msg) {} +}; + + + +#endif // __EXPERIMENT_INFO_HPP__ diff --git a/src/experiments/generic-experiment/generic-experiment.proto b/src/experiments/generic-experiment/generic-experiment.proto new file mode 100644 index 00000000..09b25fc6 --- /dev/null +++ b/src/experiments/generic-experiment/generic-experiment.proto @@ -0,0 +1,36 @@ +import "DatabaseCampaignMessage.proto"; + +message GenericExperimentMessage { + required DatabaseCampaignMessage fsppilot = 1; + + repeated group Result = 2 { + // This submessage is required by the database experiment and + // is filled with standard experiment result values + required DatabaseExperimentMessage base_result = 1; + + // make these optional to reduce overhead for server->client communication + enum ResultType { + OK_MARKER = 1; + FAIL_MARKER = 2; + DETECTED_MARKER = 3; + + GROUP1_MARKER = 4; + GROUP2_MARKER = 5; + GROUP3_MARKER = 6; + GROUP4_MARKER = 7; + + TIMEOUT = 8; + TRAP = 9; + WRITE_TEXTSEGMENT = 10; + WRITE_OUTERSPACE = 11; + + UNKNOWN = 100; + } + // result type, see above + required ResultType resulttype = 4; + + required uint64 crash_time = 5; + + optional uint64 details = 7; + } +} diff --git a/src/experiments/generic-experiment/main.cc b/src/experiments/generic-experiment/main.cc new file mode 100644 index 00000000..73501382 --- /dev/null +++ b/src/experiments/generic-experiment/main.cc @@ -0,0 +1,20 @@ +#include +#include + +#include "cpn/CampaignManager.hpp" +#include "util/CommandLine.hpp" +#include "campaign.hpp" + +int main(int argc, char **argv) +{ + fail::CommandLine &cmd = fail::CommandLine::Inst(); + for (int i = 1; i < argc; ++i) + cmd.add_args(argv[i]); + + GenericExperimentCampaign c; + if (fail::campaignmanager.runCampaign(&c)) { + return 0; + } else { + return 1; + } +}