GenericExperiment: A standardized fault injection experiment

The GenericExperiment is a standard campaign+experiment pair. It is
derived from the DatabaseCampaign+DatabaseExperiment. Its experiment
endpoints are set on the command line, therefore it can be used to give
users a first impression of FAIL*. Currently it supports different
endpoints:

--trap: Catch all traps that occur and end the experiment
--timeout <N>: kill the experiment after N microseconds
--catch-write-text: detect writes on the text segment
--catch-write-outerspace: detect writes into nirvana
--{ok,fail,detected}-marker: groups of ELF symbols that are used as
     execution breakpoints

Change-Id: Idc7fcf8875953f1007e1a37bacb086eddd29cd10

src/experiments/generic-experiment/CMakeLists.txt

@@ -0,0 +1,37 @@
+set(EXPERIMENT_NAME generic-experiment)
+set(EXPERIMENT_TYPE GenericExperiment)
+configure_file(../instantiate-experiment.ah.in
+               ${CMAKE_CURRENT_BINARY_DIR}/instantiate-${EXPERIMENT_NAME}.ah @ONLY
+)
+
+## Setup desired protobuf descriptions HERE ##
+set(MY_PROTOS
+	generic-experiment.proto
+)
+
+set(MY_CAMPAIGN_SRCS
+	experiment.hpp
+	experiment.cc
+	campaign.hpp
+	campaign.cc
+)
+
+#### PROTOBUFS ####
+find_package(Protobuf REQUIRED)
+include_directories(${PROTOBUF_INCLUDE_DIRS})
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+find_package(MySQL REQUIRED)
+include_directories(${MYSQL_INCLUDE_DIR})
+
+PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS ${MY_PROTOS})
+
+## Build library
+add_library(fail-${EXPERIMENT_NAME} ${PROTO_SRCS} ${PROTO_HDRS} ${MY_CAMPAIGN_SRCS})
+add_dependencies(fail-${EXPERIMENT_NAME} fail-comm)
+target_link_libraries(fail-${EXPERIMENT_NAME} ${PROTOBUF_LIBRARY} fail-sal)
+
+## This is the example's campaign server distributing experiment parameters
+add_executable(${EXPERIMENT_NAME}-server main.cc)
+target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} ${MYSQL_LIBRARIES} -Wl,--end-group)
+install(TARGETS ${EXPERIMENT_NAME}-server RUNTIME DESTINATION bin)

src/experiments/generic-experiment/campaign.cc

@@ -0,0 +1,21 @@
+#include <iostream>
+#include <fstream>
+
+#include "campaign.hpp"
+#include "experimentInfo.hpp"
+#include "cpn/CampaignManager.hpp"
+#include "util/Logger.hpp"
+#include "util/ProtoStream.hpp"
+#include "sal/SALConfig.hpp"
+
+#include "experimentInfo.hpp"
+
+using namespace std;
+using namespace fail;
+using namespace google::protobuf;
+
+void GenericExperimentCampaign::cb_send_pilot(DatabaseCampaignMessage pilot) {
+	GenericExperimentData *data = new GenericExperimentData;
+	data->msg.mutable_fsppilot()->CopyFrom(pilot);
+	campaignmanager.addParam(data);
+}

src/experiments/generic-experiment/campaign.hpp

@@ -0,0 +1,16 @@
+#ifndef __KESOGCCAMPAIGN_HPP__
+  #define __KESOGCCAMPAIGN_HPP__
+
+#include "cpn/DatabaseCampaign.hpp"
+#include <google/protobuf/descriptor.h>
+
+class GenericExperimentCampaign : public fail::DatabaseCampaign {
+	virtual const google::protobuf::Descriptor * cb_result_message() {
+		return google::protobuf::DescriptorPool::generated_pool()
+			->FindMessageTypeByName("GenericExperimentMessage");
+	}
+
+	virtual void cb_send_pilot(DatabaseCampaignMessage pilot);
+};
+
+#endif // __KESOGCCAMPAIGN_HPP__

src/experiments/generic-experiment/config.cmake

@@ -0,0 +1,2 @@
+SET(bochs_configure_params "--enable-a20-pin;--enable-x86-64;--enable-cpu-level=6;--enable-ne2000;--enable-acpi;--enable-pci;--enable-usb;--enable-trace-cache;--enable-fast-function-calls;--enable-host-specific-asms;--enable-readline;--enable-clgd54xx;--enable-fpu;--enable-vmx=2;--enable-monitor-mwait;--enable-cdrom;--enable-sb16=linux;--enable-gdb-stub;--with-nogui" CACHE STRING "")
+

src/experiments/generic-experiment/experiment.cc

@@ -0,0 +1,266 @@
+#include <iostream>
+#include <fstream>
+
+// getpid
+#include <sys/types.h>
+#include <unistd.h>
+
+
+#include <stdlib.h>
+#include "experiment.hpp"
+#include "experimentInfo.hpp"
+#include "sal/SALConfig.hpp"
+#include "sal/SALInst.hpp"
+#include "sal/Memory.hpp"
+#include "sal/Listener.hpp"
+
+#include "sal/bochs/BochsListener.hpp"
+#include <string>
+#include <vector>
+
+#include "campaign.hpp"
+#include "generic-experiment.pb.h"
+#include "util/CommandLine.hpp"
+
+using namespace std;
+using namespace fail;
+
+
+GenericExperiment::~GenericExperiment() {}
+
+static GenericExperimentData space_for_param;
+ExperimentData* GenericExperiment::cb_allocate_experiment_data() {
+    return &space_for_param;
+}
+
+/**
+ * Allocate a new result slot in the given experiment data
+ */
+google::protobuf::Message* GenericExperiment::cb_new_result(ExperimentData* data) {
+    GenericExperimentData *param = static_cast<GenericExperimentData *>(data);
+    GenericExperimentMessage_Result *result = param->msg.add_result();
+    return result;
+}
+
+
+void handleEvent(GenericExperimentMessage_Result& result,
+				 GenericExperimentMessage_Result_ResultType restype,
+				 unsigned int details) {
+    cout << "Result details: " << restype << " "<< details << endl;
+    result.set_resulttype(restype);
+    result.set_details(details);
+}
+
+void GenericExperiment::parseSymbols(const std::string &args, std::set<fail::BaseListener *> * into) {
+	std::vector<std::string> elems;
+	std::stringstream ss(args);
+	std::string item;
+	while (std::getline(ss, item, ',')) {
+		const ElfSymbol * symbol = &m_elf.getSymbol(item);
+		if (!symbol->isValid()) {
+			m_log << "ELF Symbol not found: " << item << endl;
+			simulator.terminate(1);
+		}
+		m_log << "Adding symbol " << item <<  " at 0x" << hex << symbol->getAddress() << endl;
+		BPSingleListener *l = new BPSingleListener(symbol->getAddress());
+		into->insert(l);
+		end_markers.insert(l);
+		listener_to_symbol[l] = symbol;
+	}
+}
+
+
+
+bool GenericExperiment::cb_start_experiment() {
+	CommandLine &cmd = CommandLine::Inst();
+	cmd.addOption("", "", Arg::None, "USAGE: fail-client -Wf,[option] -Wf,[option] ... <BochsOptions...>\n\n");
+	CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h,--help \tPrint usage and exit");
+
+	CommandLine::option_handle STATE_DIR = cmd.addOption("", "state-dir", Arg::Required,
+														 "--state-dir \t Path to the state directory");
+
+	// catch any trap
+	CommandLine::option_handle TRAP = cmd.addOption("", "trap", Arg::None,
+														 "--trap \tCatch Traps");
+	CommandLine::option_handle WRITE_MEM_TEXT = cmd.addOption("", "catch-write-textsegment", Arg::None,
+													"--catch-write-textsegment \tCatch writes to the text segment");
+
+	CommandLine::option_handle WRITE_MEM_OUTERSPACE
+		= cmd.addOption("", "catch-write-outerspace", Arg::None,
+						"--catch-write-outerspace \tCatch writes to the outerspace");
+
+	CommandLine::option_handle TIMEOUT = cmd.addOption("", "timeout", Arg::Required,
+													   "--timeout \t Experiment Timeout in uS");
+
+
+	std::map<std::string, CommandLine::option_handle> option_handles;
+	for (std::map<std::string, ListenerSet *>::iterator it = end_marker_groups.begin();
+		 it != end_marker_groups.end(); ++it) {
+		CommandLine::option_handle handle =
+			cmd.addOption("", it->first, Arg::Required,
+						  "--" + it->first + " \tList of symbols (comma separated)");
+		option_handles[it->first] = handle;
+	}
+
+	if (!cmd.parse()) {
+		cerr << "Error parsing arguments." << endl;
+		exit(-1);
+	}
+
+	if (cmd[HELP]) {
+		cmd.printUsage();
+		exit(0);
+	}
+
+	address_t minimal_ip = INT_MAX; // Every address is lower
+	address_t maximal_ip = 0;
+	address_t minimal_data = 0x100000; // 1 Mbyte
+	address_t maximal_data = 0;
+
+	for (ElfReader::section_iterator it = m_elf.sec_begin();
+		 it != m_elf.sec_end(); ++it) {
+		const ElfSymbol &symbol = *it;
+		std::string prefix(".text");
+		if (symbol.getName().compare(0, prefix.size(), prefix) == 0) {
+			minimal_ip = std::min(minimal_ip, symbol.getStart());
+			maximal_ip = std::max(maximal_ip, symbol.getEnd());
+		} else {
+			minimal_data = std::min(minimal_data, symbol.getStart());
+			maximal_data = std::max(maximal_data, symbol.getEnd());
+		}
+	}
+
+	if (cmd[WRITE_MEM_TEXT]) {
+		m_log << "Catch writes to text segment from " << hex << minimal_ip << " to " << maximal_ip << std::endl;
+		enabled_mem_text = true;
+
+		l_mem_text.setWatchAddress(minimal_ip);
+		l_mem_text.setTriggerAccessType(MemAccessEvent::MEM_WRITE);
+		l_mem_text.setWatchWidth(maximal_ip - minimal_ip);
+	}
+
+	if (cmd[WRITE_MEM_OUTERSPACE]) {
+		m_log << "Catch writes to outerspace from " << hex << " from " << maximal_data << std::endl;
+		enabled_mem_outerspace = true;
+
+		l_mem_outerspace.setWatchAddress(maximal_data);
+		l_mem_outerspace.setTriggerAccessType(MemAccessEvent::MEM_WRITE);
+		l_mem_outerspace.setWatchWidth(0xfffffff0);
+	}
+
+	if (cmd[TRAP]) {
+		m_log << "Catch all traps" << endl;
+		enabled_trap = true;
+	}
+
+	if (cmd[STATE_DIR]) {
+		std::string value(cmd[STATE_DIR].first()->arg);
+		m_state_dir = value;
+		m_log << "Set state dir to " << value << endl;
+	}
+
+	if (cmd[TIMEOUT]) {
+		std::string value(cmd[TIMEOUT].first()->arg);
+		std::stringstream ss(value);
+		ss >> m_Timeout;
+		if (ss.bad()) {
+			m_log << "Could not parse --timeout argument" << endl;
+			return false; // Initialization failed
+		}
+		l_timeout.setTimeout(m_Timeout);
+		enabled_timeout = true;
+		m_log << "Enabled Experiment Timeout of " << dec << m_Timeout << " microseconds" << endl;
+	}
+
+	for (std::map<std::string, CommandLine::option_handle>::iterator it = option_handles.begin();
+		 it != option_handles.end(); ++it) {
+		if (cmd[option_handles[it->first]]) {
+			option::Option *opt = cmd[option_handles[it->first]].first();
+			while (opt != 0) {
+				parseSymbols(std::string(opt->arg), end_marker_groups[it->first]);
+				opt = opt->next();
+			}
+		}
+	}
+
+	return true; // Everything OK
+}
+
+
+bool GenericExperiment::cb_before_resume() {
+	if (enabled_trap)
+		simulator.addListener(&l_trap);
+
+	if (enabled_mem_text)
+		simulator.addListener(&l_mem_text);
+
+	if (enabled_mem_outerspace) {
+		std::cout << "enabled mem outerspace " << endl;
+		simulator.addListener(&l_mem_outerspace);
+	}
+
+	if (enabled_timeout)
+		simulator.addListener(&l_timeout);
+
+	for (std::set<BaseListener *>::iterator it = end_markers.begin();
+		 it != end_markers.end(); ++it) {
+		simulator.addListener(*it);
+	}
+
+	return true; // everything OK
+}
+
+void GenericExperiment::cb_after_resume(fail::BaseListener *event) {
+	GenericExperimentMessage_Result * result = static_cast<GenericExperimentMessage_Result *>(this->get_current_result());
+
+	// Record the crash time
+	result->set_crash_time(simulator.getTimerTicks());
+
+
+	if (event == &l_timeout) {
+		handleEvent(*result, result->TIMEOUT, m_Timeout);
+	}  else if (event == &l_trap) {
+		handleEvent(*result, result->TRAP, l_trap.getTriggerNumber());
+	} else if (event == &l_mem_text) {
+		handleEvent(*result, result->WRITE_TEXTSEGMENT,
+					l_mem_text.getTriggerAddress());
+
+	} else if (event == &l_mem_outerspace){
+		handleEvent(*result, result->WRITE_OUTERSPACE,
+					l_mem_outerspace.getTriggerAddress());
+
+		//////////////////////////////////////////////////
+		// End Marker Groups
+		//////////////////////////////////////////////////
+	} else if (OK_marker.find(event) != OK_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->OK_MARKER, symbol->getAddress());
+
+	} else if (FAIL_marker.find(event) != FAIL_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->FAIL_MARKER, symbol->getAddress());
+
+	} else if (DETECTED_marker.find(event) != DETECTED_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->DETECTED_MARKER, symbol->getAddress());
+
+	} else if (GROUP1_marker.find(event) != GROUP1_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->GROUP1_MARKER, symbol->getAddress());
+
+	} else if (GROUP2_marker.find(event) != GROUP2_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->GROUP2_MARKER, symbol->getAddress());
+
+	} else if (GROUP3_marker.find(event) != GROUP3_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->GROUP3_MARKER, symbol->getAddress());
+
+	} else if (GROUP4_marker.find(event) != GROUP4_marker.end()) {
+		const ElfSymbol *symbol = listener_to_symbol[event];
+		handleEvent(*result, result->GROUP4_MARKER, symbol->getAddress());
+
+	} else {
+		handleEvent(*result, result->UNKNOWN, 0);
+	}
+}

src/experiments/generic-experiment/experiment.hpp

@@ -0,0 +1,111 @@
+#ifndef __GENERIC_EXPERIMENT_EXPERIMENT_HPP__
+#define __GENERIC_EXPERIMENT_EXPERIMENT_HPP__
+
+#include "sal/SALInst.hpp"
+#include "efw/DatabaseExperiment.hpp"
+#include "sal/Listener.hpp"
+#include "efw/JobClient.hpp"
+#include "util/Logger.hpp"
+#include "util/ElfReader.hpp"
+#include <string>
+#include <stdlib.h>
+#include <map>
+#include <set>
+
+
+class GenericExperiment : public fail::DatabaseExperiment {
+	fail::ElfReader m_elf;
+
+	std::string m_state_dir;
+
+	bool enabled_mem_text;
+	fail::MemAccessListener l_mem_text;
+
+	bool enabled_mem_outerspace;
+	fail::MemAccessListener l_mem_outerspace;
+
+	bool enabled_trap;
+	fail::TrapListener l_trap;
+
+	bool enabled_timeout;
+	unsigned m_Timeout;
+	fail::TimerListener l_timeout;
+
+	std::map<fail::BaseListener *, const fail::ElfSymbol *> listener_to_symbol;
+
+	typedef std::set<fail::BaseListener *> ListenerSet;
+
+	ListenerSet end_markers;
+	ListenerSet OK_marker;
+	ListenerSet FAIL_marker;
+	ListenerSet DETECTED_marker;
+	ListenerSet GROUP1_marker;
+	ListenerSet GROUP2_marker;
+	ListenerSet GROUP3_marker;
+	ListenerSet GROUP4_marker;
+
+	std::map<std::string, ListenerSet * >  end_marker_groups;
+
+	void parseSymbols(const std::string &args, std::set<fail::BaseListener *> *into);
+
+public:
+	GenericExperiment() : DatabaseExperiment("GenericExperiment"),
+						  m_state_dir("state"),
+						  l_trap(fail::ANY_TRAP), l_timeout(0) {
+		enabled_mem_text = false;
+		enabled_mem_outerspace = false;
+		enabled_trap = false;
+		enabled_timeout = false;
+
+		end_marker_groups["ok-marker"] = &OK_marker;
+		end_marker_groups["fail-marker"] = &FAIL_marker;
+		end_marker_groups["detected-marker"] = &FAIL_marker;
+		end_marker_groups["group1-marker"] = &GROUP1_marker;
+		end_marker_groups["group2-marker"] = &GROUP2_marker;
+		end_marker_groups["group3-marker"] = &GROUP3_marker;
+		end_marker_groups["group4-marker"] = &GROUP4_marker;
+	}
+	virtual ~GenericExperiment();
+
+	/**
+	 * Get path to the state directory
+	 */
+	virtual std::string cb_state_directory() { return m_state_dir; }
+
+	/**
+	 * Allocate enough space to hold the incoming ExperimentData message.
+	 */
+	virtual fail::ExperimentData* cb_allocate_experiment_data();
+
+	/**
+	 * Allocate a new result slot in the given experiment data
+	 */
+	virtual google::protobuf::Message* cb_new_result(fail::ExperimentData* data);
+
+	/**
+	 * Callback that is called, before the actual experiment
+	 * starts. Simulation is terminated on false.
+	 * @param The current result message
+	 * @return \c true on success, \c false otherwise
+	 */
+	virtual bool cb_start_experiment();
+
+	/**
+	 * Callback that is called before the resuming till crash has
+	 * started. This is called after the fault was injected. Here the
+	 * end listeners should be installed. Returns true on
+	 * success. Otherwise the experiment is canceled.
+
+	 * @return \c true on success, \c false otherwise
+	 */
+	virtual bool cb_before_resume();
+
+	/**
+	 * Callback that is called after the resume-till-crash phase with
+	 * the last triggered listener. This callback should collect all
+	 * data and fill up the result message.
+	 */
+	virtual void cb_after_resume(fail::BaseListener *event);
+};
+
+#endif // __GENERIC_EXPERIMENT_EXPERIMENT_HPP__

src/experiments/generic-experiment/experimentInfo.hpp

@@ -0,0 +1,15 @@
+#ifndef __EXPERIMENT_INFO_HPP__
+#define __EXPERIMENT_INFO_HPP__
+
+#include "comm/ExperimentData.hpp"
+#include "generic-experiment.pb.h"
+
+class GenericExperimentData : public fail::ExperimentData {
+public:
+	GenericExperimentMessage msg;
+	GenericExperimentData() : fail::ExperimentData(&msg) {}
+};
+
+
+
+#endif // __EXPERIMENT_INFO_HPP__

src/experiments/generic-experiment/generic-experiment.proto

@@ -0,0 +1,36 @@
+import "DatabaseCampaignMessage.proto";
+
+message GenericExperimentMessage {
+    required DatabaseCampaignMessage fsppilot = 1;
+
+    repeated group Result = 2 {
+        // This submessage is required by the database experiment and
+        // is filled with standard experiment result values
+        required DatabaseExperimentMessage base_result = 1;
+
+		// make these optional to reduce overhead for server->client communication
+		enum ResultType {
+            OK_MARKER = 1;
+            FAIL_MARKER = 2;
+            DETECTED_MARKER = 3;
+            
+            GROUP1_MARKER = 4;
+            GROUP2_MARKER = 5;
+            GROUP3_MARKER = 6;
+            GROUP4_MARKER = 7;
+
+            TIMEOUT = 8;
+            TRAP    = 9;
+            WRITE_TEXTSEGMENT = 10;
+            WRITE_OUTERSPACE  = 11;
+
+            UNKNOWN = 100;
+		}
+		// result type, see above
+		required ResultType resulttype = 4;
+
+        required uint64 crash_time = 5;
+
+		optional uint64 details = 7;
+	}
+}

src/experiments/generic-experiment/main.cc

@@ -0,0 +1,20 @@
+#include <iostream>
+#include <cstdlib>
+
+#include "cpn/CampaignManager.hpp"
+#include "util/CommandLine.hpp"
+#include "campaign.hpp"
+
+int main(int argc, char **argv)
+{
+	fail::CommandLine &cmd = fail::CommandLine::Inst();
+	for (int i = 1; i < argc; ++i)
+		cmd.add_args(argv[i]);
+
+	GenericExperimentCampaign c;
+	if (fail::campaignmanager.runCampaign(&c)) {
+		return 0;
+	} else {
+		return 1;
+	}
+}