tools: added compute-hops and dump-hops tools

As these tools work closely together with fail components, its
easiest, to build them in this context. As these tools don't
really matter for fail use, they might never be pushed to the
master branch.

Change-Id: I8c8bd80376d0475f08a531a995d829e85032371b

src/core/comm/InjectionPointHopsMessage.proto

@@ -6,6 +6,11 @@ message InjectionPointMessage {
 	// If checkpoint must be used for this hop chain, id is set properly
 	optional uint32 checkpoint_id = 1;
 
+    // If we need to knwo the target dynamic instruction offset,
+    // here it is
+    optional uint32 target_trace_position = 4; 
+
+
 	// Repeated groups can't be defined as notempty, so a manual
 	// non-empty check is required at usage 
 	

src/core/cpn/CMakeLists.txt

@@ -16,7 +16,7 @@ add_library(fail-cpn ${SRCS})
 # if hop-chains need to be calculated by the server, we 
 # the smarthopping module
 if(CONFIG_INJECTIONPOINT_HOPS)
-    add_dependencies(fail-cpn fail-smarthopping)
+    add_dependencies(fail-cpn fail-smarthops)
 endif(CONFIG_INJECTIONPOINT_HOPS)
 
 add_dependencies(fail-cpn fail-comm)

src/core/util/smarthops/TraceReader.cc

@@ -41,16 +41,25 @@ std::istream& openStream(const char *input_file,
 	return normal_stream;
 }
 
-void TraceReader::openTraceFile(const char *filename)
+bool TraceReader::openTraceFile(const char *filename, unsigned int num_inst)
 {
 	normal_stream = new std::ifstream();
 	gz_stream = new igzstream();
 	ps = new fail::ProtoIStream(&openStream(filename, *normal_stream, *gz_stream, m_log));
+
+	m_max_num_inst = num_inst;
+
+	return true;
 }
 
 bool TraceReader::getNextTraceEvents(trace_pos_t& trace_pos,
 							std::vector<trace_event_tuple_t >& trace_events)
 {
+	// Stop after fixed number of instructions, if given as command line argument
+	if ((m_max_num_inst > 0) && (m_current_position > m_max_num_inst)) {
+		return false;
+	}
+
 	trace_pos = m_current_position;
 
 	// Delivered trace_events vector does not have to be

src/core/util/smarthops/TraceReader.hpp

@@ -28,8 +28,13 @@ typedef std::pair<address_t, mem_access_type_e> trace_event_tuple_t;
 class TraceReader {
     public:
 
-		TraceReader() : m_current_position(1), ps(0), normal_stream(0),
+		TraceReader() : m_current_position(1),
-	        		gz_stream(0), ev_avail(false), m_log("TraceReader", false) {}
+						ps(0),
+						normal_stream(0),
+						gz_stream(0),
+						m_max_num_inst(0),
+						ev_avail(false),
+						m_log("TraceReader", false) {}
 
 		~TraceReader();
 
@@ -37,12 +42,13 @@ class TraceReader {
         bool getNextTraceEvents(trace_pos_t& trace_pos,
         					std::vector<trace_event_tuple_t >& trace_events);
 
-        void openTraceFile(const char *filename);
+        bool openTraceFile(const char *filename, unsigned int num_inst = 0);
     private:
         unsigned int m_current_position;
         ProtoIStream* ps;
         std::ifstream *normal_stream;
         igzstream *gz_stream;
+        unsigned int m_max_num_inst;
         Trace_Event ev;
         bool ev_avail;
 

src/experiments/lra-simple-panda/CMakeLists.txt

@@ -32,5 +32,5 @@ target_link_libraries(fail-${EXPERIMENT_NAME} ${PROTOBUF_LIBRARY})
 
 ## This is the example's campaign server distributing experiment parameters
 add_executable(${EXPERIMENT_NAME}-server main.cc)
-target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-smarthopping fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} -lmysqlclient -Wl,--end-group)
+target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-smarthops fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} -lmysqlclient -Wl,--end-group)
 install(TARGETS ${EXPERIMENT_NAME}-server RUNTIME DESTINATION bin)

tools/CMakeLists.txt

@@ -3,6 +3,8 @@ option(BUILD_IMPORT_TRACE "Build the trace import tool?" OFF)
 option(BUILD_PRUNE_TRACE  "Build the trace prune tool?" OFF)
 option(BUILD_CONVERT_TRACE "Build the trace converter tool?" OFF)
 
+option(BUILD_COMPUTE_HOPS  "Build the compute hops tool?" OFF)
+option(BUILD_DUMP_HOPS  "Build the hops dump tool?" OFF)
 
 ### Setup search paths for headers ##
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../src/core)
@@ -23,3 +25,11 @@ endif(BUILD_DUMP_TRACE)
 if(BUILD_CONVERT_TRACE)
 	add_subdirectory(convert-trace)
 endif(BUILD_CONVERT_TRACE)
+
+if(BUILD_COMPUTE_HOPS)
+      add_subdirectory(compute-hops)
+endif(BUILD_COMPUTE_HOPS)
+
+if(BUILD_DUMP_HOPS)
+      add_subdirectory(dump-hops)
+endif(BUILD_DUMP_HOPS)

tools/compute-hops/BasicAlgorithm.hpp

@@ -0,0 +1,23 @@
+#ifndef __BASIC_ALGORITHM__HPP
+#define __BASIC_ALGORITHM__HPP
+
+#include "../../src/core/util/smarthops/TraceReader.hpp"
+class ResultCollector;
+
+class BasicAlgorithm {
+    public:
+		/**
+		 *
+		 * @returns boolean value for calculation success
+		 */
+        virtual bool calculateAllHops(fail::TraceReader& trace) = 0;
+
+        BasicAlgorithm(ResultCollector *rc) {m_resultCollector = rc;}
+        virtual ~BasicAlgorithm() {}
+
+    protected:
+        ResultCollector *m_resultCollector;
+};
+
+#endif // __BASIC_ALGORITHM__HPP
+

tools/compute-hops/CMakeLists.txt

@@ -0,0 +1,13 @@
+set(SRCS
+  ResultCollector.cc
+  SimpleAlgorithm.cc
+  SmartAlgorithm.cc
+)
+
+if(NOT CONFIG_INJECTIONPOINT_HOPS)
+    message (FATAL_ERROR "For building the compute-hops tool you need to enable CONFIG_INJECTIONPOINT_HOPS")
+endif()
+
+add_executable(compute-hops main.cc ${SRCS})
+target_link_libraries(compute-hops ${PROTOBUF_LIBRARY} fail-util fail-comm)
+install(TARGETS compute-hops RUNTIME DESTINATION bin)

tools/compute-hops/ResultCollector.cc

@@ -0,0 +1,172 @@
+/*
+ * StatisticsCollector.cc
+ *
+ *  Created on: 21.08.2013
+ *      Author: lrade
+ */
+
+#include "ResultCollector.hpp"
+
+#include <iostream>
+#include <fstream>
+#include <cmath>
+#include <numeric>
+#include <algorithm>
+
+#include <vector>
+
+#include "comm/InjectionPointHopsMessage.pb.h"
+
+// #include <proc/readproc.h>
+
+using std::hex;
+using std::dec;
+
+char separator = ' ';
+
+extern std::ofstream g_cp_ofstream;
+
+extern fail::Logger LOG;
+
+void
+ResultCollector::setProtoOStream(ProtoOStream* protoOStream)
+{
+	ps = protoOStream;
+}
+
+unsigned int calculate_costs(std::vector<result_tuple >& res)
+{
+	std::vector<result_tuple>::iterator it = res.begin();
+	std::vector<result_tuple>::iterator it_ahead = res.begin();
+	it_ahead++;
+
+	// Cost COST_CHANGE for first hop
+	// Costs of all following hops are calculated only on basis of
+	// difference between two consecutive hop_positions
+	unsigned int costs = COST_CHANGE;
+
+	while (it != res.end() && it_ahead != res.end()) {
+		if (it->first == it_ahead->first) {
+			costs += COST_NO_CHANGE;
+		} else {
+			costs += COST_CHANGE;
+		}
+		it_ahead++;
+		it++;
+	}
+
+	return costs;
+}
+
+void
+ResultCollector::addResult(std::vector<result_tuple >& res, unsigned int costs)
+{
+	m_result_size++;
+
+	// Costs not calculated in hop calculator (e.g. dijkstra)? => Do it here
+	if (costs == 0) {
+		costs = calculate_costs(res);
+	}
+
+	if (m_output_mode == OUTPUT_COSTS) {
+		m_ostream << m_res_count++ << separator << costs << '\n';
+	} else if (m_output_mode == OUTPUT_RESULT) {
+		if (ps) {
+			InjectionPointMessage hc;
+			hc.set_costs(costs);
+            hc.set_target_trace_position(res.back().second);
+			// If checkpoint at beginning of hop-chain, add its id to InjectionPointMessage
+			std::vector<result_tuple >::iterator it_hop = res.begin();
+			if (it_hop != res.end() && it_hop->first.second == ACCESS_CHECKPOINT) {
+				hc.set_checkpoint_id(it_hop->first.first);
+				it_hop++;
+			}
+
+			for(;it_hop != res.end();
+				it_hop++) {
+				InjectionPointMessage_Hops *hop = hc.add_hops();
+				hop->set_address(it_hop->first.first);
+
+				enum InjectionPointMessage_Hops_AccessType at;
+				switch (it_hop->first.second) {
+				case ACCESS_NONE:
+					at = InjectionPointMessage_Hops_AccessType_EXECUTE;
+					break;
+				case ACCESS_READ:
+					at = InjectionPointMessage_Hops_AccessType_READ;
+					break;
+				case ACCESS_WRITE:
+					at = InjectionPointMessage_Hops_AccessType_WRITE;
+					break;
+				case ACCESS_READORWRITE:
+					LOG << "ReadOrWrite memory access event not yet"
+					" covered" << std::endl;
+					exit(-1);
+					break;
+				case ACCESS_CHECKPOINT:
+					LOG << "Checkpoint not allowed after beginning of hop chain" << std::endl;
+					exit(-1);
+				}
+				hop->set_accesstype(at);
+			}
+			ps->writeMessage(&hc);
+		} else {
+			for(std::vector<result_tuple >::iterator it_hop = res.begin();
+					it_hop != res.end();
+					it_hop++) {
+				address_t add = it_hop->first.first;
+				mem_access_type_e mem_acc_type = it_hop->first.second;
+				std::string prefix = (mem_acc_type == ACCESS_READ)?"R":((mem_acc_type == ACCESS_WRITE)?"W":((mem_acc_type == ACCESS_NONE)?"X":((mem_acc_type == ACCESS_CHECKPOINT)?"C":"")));
+				m_ostream << prefix << hex << add << dec << separator;
+			}
+
+			if (res.size() > 0)
+				m_ostream << '\n';
+		}
+	} else if (m_output_mode == OUTPUT_STATISTICS) {
+		// Calculate mean
+		// http://www.heikohoffmann.de/htmlthesis/node134.html
+		// c(t+1) = c(t) + (1/(t+1))*(x - c(t))
+		m_it_mean_costs = m_it_mean_costs + (1/(double)m_result_size)*((double)costs-m_it_mean_costs);
+
+	}
+}
+
+void
+ResultCollector::addCheckpoint(unsigned int pos)
+{
+	g_cp_ofstream << m_checkpoint_count++ << separator << pos << '\n';
+}
+
+void
+ResultCollector::startTimer()
+{
+	m_timer.startTimer();
+}
+
+void
+ResultCollector::stopTimer()
+{
+	m_timer.stopTimer();
+}
+
+void
+ResultCollector::setMaxMemUsage()
+{
+//	struct proc_t usage;
+//	look_up_our_self(&usage);
+//	m_mem_usage = usage.vsize;
+}
+
+void
+ResultCollector::finish()
+{
+	// Print results if buffered
+
+	// Print statistics
+	if (m_output_mode == OUTPUT_STATISTICS) {
+
+		m_ostream << m_result_size << separator << m_timer.getRuntimeAsDouble() << separator
+				<< m_mem_usage << separator << m_it_mean_costs << separator << m_checkpoint_count << '\n';
+	}
+}

tools/compute-hops/ResultCollector.hpp

@@ -0,0 +1,91 @@
+/*
+ * ResultCollector.hpp
+ *
+ *  Created on: 23.08.2013
+ *      Author: Lars Rademacher
+ */
+
+#ifndef RESULTCOLLECTOR_HPP_
+#define RESULTCOLLECTOR_HPP_
+
+#include <vector>
+#include <fstream>
+#include <iostream>
+
+#include "../../src/core/util/smarthops/TraceReader.hpp"
+
+#include "util/WallclockTimer.hpp"
+
+using namespace fail;
+
+typedef unsigned int trace_pos_t;
+
+typedef std::pair<trace_event_tuple_t, trace_pos_t > result_tuple;
+
+typedef enum {
+	OUTPUT_RESULT		,
+	OUTPUT_COSTS		,
+	OUTPUT_STATISTICS	,
+} output_mode_e;
+
+using fail::ProtoOStream;
+
+#define COST_CHANGE 	1
+#define COST_NO_CHANGE 	2
+
+class ResultCollector {
+public:
+
+	ResultCollector(std::ostream& out_stream, output_mode_e output_mode) :
+		m_ostream(out_stream),
+		m_res_count(1),
+		m_output_mode(output_mode),
+		m_mem_usage(0),
+		m_result_size(0),
+		m_checkpoint_count(1),
+		m_it_mean_costs(0),
+		m_max_costs(0),
+		ps(0){}
+
+	void
+	addResult(std::vector<result_tuple >& res, unsigned int costs);
+
+	void
+	addCheckpoint(unsigned int pos);
+
+	void
+	startTimer();
+
+	void
+	stopTimer();
+
+	void
+	setMaxMemUsage();
+
+	void
+	setProtoOStream(ProtoOStream* protoOStream);
+
+	// Prints buffered results on output stream
+	void
+	finish();
+
+private:
+
+	std::ostream& m_ostream;
+	unsigned int m_res_count;
+	output_mode_e m_output_mode;
+	unsigned long m_mem_usage;
+	unsigned int m_result_size;
+	unsigned int m_checkpoint_count;
+
+	double m_it_mean_costs;
+	unsigned int m_max_costs;
+
+	fail::WallclockTimer m_timer;
+
+	fail::ProtoOStream *ps;
+};
+
+#endif /* STATISTICSCOLLECTOR_HPP_ */
+
+

tools/compute-hops/SimpleAlgorithm.cc

@@ -0,0 +1,125 @@
+#include "SimpleAlgorithm.hpp"
+
+#include <limits>
+//#include <utility>
+#include <cmath>
+#include <algorithm>
+
+#include "../../src/core/util/smarthops/TraceReader.hpp"
+#include "ResultCollector.hpp"
+
+using namespace std;
+using namespace fail;
+
+// Idea for CPs
+// for each new trace_event: costs > THRESH?
+//   => Use latest CP
+//      Costs still > THRESH?
+//		=> New CP
+
+// Additional Map, which tracks down, how often a instruction occured
+// since creation of the latest CP
+// => MEM USAGE x 2
+
+extern unsigned int g_cp_thresh;
+extern unsigned int g_cost_cp;
+
+extern bool g_use_watchpoints;
+extern bool g_use_checkpoints;
+extern fail::Logger LOG;
+
+bool SimpleAlgorithm::calculateAllHops(TraceReader& trace)
+{
+	trace_pos_t trace_pos;
+	vector<trace_event_tuple_t > trace_events;
+
+	vector<result_tuple > empty_result;
+
+	// Parse all executed instructions
+	while (trace.getNextTraceEvents(trace_pos, trace_events)) {
+
+		// Policy when multiple trace events: Choose the most rare one
+		trace_event_tuple_t *candidate = NULL;
+		//vector<trace_pos_t> *cand_pos = NULL;
+		unsigned long num_hops = numeric_limits<unsigned long>::max();
+
+		for (vector<trace_event_tuple_t >::iterator it_te = trace_events.begin();
+								it_te != trace_events.end();
+								it_te++) {
+			// Don't use watchpoints?
+			if (!g_use_watchpoints && (it_te->second != ACCESS_NONE)) {
+				continue;
+			}
+
+			// Find last pos of current trace event
+			//map<trace_event_tuple_t, vector<trace_pos_t> >::iterator it_oc = m_occurences.find(*it_te);
+			map<trace_event_tuple_t, unsigned long >::iterator it_oc = m_occurences.find(*it_te);
+
+			// Not known? => Add
+			if (it_oc == m_occurences.end()) {
+				//m_occurences.insert(pair<trace_event_tuple_t, vector<trace_pos_t > >(*it_te, vector<trace_pos_t>(1, trace_pos)));
+				m_occurences.insert(pair<trace_event_tuple_t, unsigned long >(*it_te, 1));
+				num_hops = 1;
+				it_oc = m_occurences.find(*it_te);
+				candidate = (trace_event_tuple_t*)&(it_oc->first);
+				//cand_pos = (vector<trace_pos_t>*)&(it_oc->second);
+			} else {
+				it_oc->second++;
+				if (it_oc->second < num_hops) {
+					candidate = (trace_event_tuple_t*)&(it_oc->first);
+					//cand_pos = (vector<trace_pos_t>*)&(it_oc->second);
+					num_hops = it_oc->second;
+				}
+			}
+
+			// Do the same thing for the occ_cp map
+			map<trace_event_tuple_t, unsigned long >::iterator it_oc_cp = m_occ_cp.find(*it_te);
+
+			// Not known? => Add
+			if (it_oc_cp == m_occ_cp.end()) {
+				m_occ_cp.insert(pair<trace_event_tuple_t, unsigned long >(*it_te, 1));
+			} else {
+				it_oc_cp->second++;
+			}
+
+		}
+
+		if (candidate == NULL ) { //|| cand_pos == NULL
+			cout << "Error while calculating" << endl;
+			exit(1);
+		}
+
+		/*vector<result_tuple > res;
+		for (vector<trace_pos_t>::iterator it_res = cand_pos->begin();
+				it_res != cand_pos->end();
+				it_res++) {
+			res.push_back(result_tuple(*candidate, *it_res));
+		}*/
+
+		unsigned int costs = 1 + 2*(num_hops - 1);
+
+		// Costs too high?
+		if (g_use_checkpoints && costs > g_cp_thresh) {
+			// costs acceptable with usage of latest CP?
+			map<trace_event_tuple_t, unsigned long >::iterator occ_cp_find = m_occ_cp.find(*candidate);
+			if (occ_cp_find == m_occ_cp.end()) {
+				cout << "Error in data structure!"<< endl;
+				exit(1);
+			}
+			costs = 1 + 2*(occ_cp_find->second - 1) + g_cost_cp;
+			// Costs still too hight? Add new CP
+			if (costs > g_cp_thresh) {
+				m_resultCollector->addCheckpoint(trace_pos);
+				m_occ_cp.clear();
+				costs = g_cost_cp;
+			}
+		}
+
+		m_resultCollector->addResult(empty_result, costs);
+	}
+
+	// MAX MEM USAGE
+	m_resultCollector->setMaxMemUsage();
+
+	return true;
+}

tools/compute-hops/SimpleAlgorithm.hpp

@@ -0,0 +1,30 @@
+#ifndef SIMPLE_ALGORITHM__HPP
+#define SIMPLE_ALGORITHM__HPP
+
+//#include <vector>
+#include <map>
+
+#include "BasicAlgorithm.hpp"
+#include "../../src/core/util/smarthops/TraceReader.hpp"
+
+using namespace fail;
+
+class ResultCollector;
+
+class SimpleAlgorithm : public BasicAlgorithm {
+public:
+	SimpleAlgorithm(ResultCollector *rc) : BasicAlgorithm(rc) {}
+	virtual ~SimpleAlgorithm() {}
+
+    bool calculateAllHops(TraceReader& trace);
+
+private:
+    // Count occurences
+    //map<trace_event_tuple_t, vector<trace_pos_t> > m_occurences;
+    std::map<trace_event_tuple_t, unsigned long > m_occurences;
+
+    std::map<trace_event_tuple_t, unsigned long > m_occ_cp;
+};
+
+#endif // SIMPLE_ALGORITHM__HPP
+ 

tools/compute-hops/SmartAlgorithm.cc

@@ -0,0 +1,218 @@
+#include "SmartAlgorithm.hpp"
+
+#include <algorithm>
+#include <limits>
+
+#include "ResultCollector.hpp"
+
+extern bool g_use_weights;
+extern bool g_use_watchpoints;
+extern bool g_use_checkpoints;
+
+extern unsigned int g_cp_thresh;
+extern unsigned int g_cost_cp;
+
+// Must be smaller than ((COST_CP_THRESH - COST_CP) / 2)
+// If too high: Costs per dynamic instruction will approach
+// COST_CP_THRESH but number of CPs can be reduced drastically
+extern unsigned int g_rollback_thresh;
+
+extern fail::Logger LOG;
+
+bool SmartAlgorithm::calculateAllHops(TraceReader& trace)
+{
+	unsigned int trace_pos = 0;
+	unsigned int costs = 0;
+
+	std::vector<trace_event_tuple_t > trace_events;
+	std::vector<result_tuple > result;
+
+	while (trace.getNextTraceEvents(trace_pos, trace_events)) {
+		// Policy when multiple trace events: Choose the one with smallest last_pos
+		trace_event_tuple_t *best_te = NULL;
+		trace_pos_t last_pos = std::numeric_limits<trace_pos_t>::max();
+
+		bool hop_found = false;
+		bool checkpoint_forbidden = false;
+
+		for (std::vector<trace_event_tuple_t >::iterator it_te = trace_events.begin();
+						it_te != trace_events.end();
+						it_te++) {
+
+			// Don't use watchpoints?
+			if (!g_use_watchpoints && it_te->second != ACCESS_NONE) {
+				continue;
+			}
+
+			// Find last pos of current trace event
+			std::map<trace_event_tuple_t, trace_pos_t>::iterator it_lp = m_last_positions.find(*it_te);
+
+			// Not known? => Single hop, result calculation complete
+			if (it_lp == m_last_positions.end()) {
+				m_last_positions.insert(std::pair<trace_event_tuple_t, trace_pos_t>(*it_te, trace_pos));
+				// New instruction => single hop
+				if (!hop_found) {
+					result.clear();
+					result.push_back(result_tuple(*it_te, trace_pos));
+					costs = COST_CHANGE;
+					m_resultCollector->addResult(result, costs);
+					hop_found = true;
+				}
+			} else {
+				if (it_lp->second < last_pos) {
+					last_pos = it_lp->second;
+					best_te = (trace_event_tuple_t*)&(it_lp->first);
+				}
+				it_lp->second = trace_pos;
+			}
+		}
+
+		if (hop_found) {
+			continue;
+		}
+
+		// Deltion of unnecessary hops
+
+		/*
+		 *   |----------------|
+		 *   |pos bigger than |
+		 *   |last pos of new | --> delete -->-|
+		 *   |trace_event?    |                |
+		 *   |----------------|                |
+		 *         ^                           |
+		 *         |                           v
+		 * A   B   A                           C       <<< Last result
+		 *         ^                           ^
+		 *      Reverse                     Reverse
+		 *      iterator                    iterator
+		 *    rit_pre_last                  rit_last
+		 *
+		 */
+
+		std::vector<result_tuple >::reverse_iterator rit_pre_last, rit_last, end;
+
+
+		rit_last = result.rbegin();
+		rit_pre_last = result.rbegin();
+		rit_pre_last++;
+
+		while (rit_last != result.rend()) {
+
+			trace_pos_t left_pos;
+
+			// Policy switch:
+			// No weights 	=> 	last pos <= pos of rit_pre_last
+			// Weights 		=> 	last pos < pos of rit_pre_last
+
+			if ((rit_pre_last == result.rend())) {
+				// First node in hop list is Checkpoint? => Look at result before CP-Creation
+				if (rit_last->first.second == ACCESS_CHECKPOINT) {
+					rit_pre_last = (m_checkpoints[rit_last->first.first]).second.rbegin();
+
+					// This should never happen:
+					if (rit_pre_last == (m_checkpoints[rit_last->first.first]).second.rend()) {
+						break;
+					}
+
+					// Don't allow deletion of CP, if afterwards
+					// too few of the olf hops would be deleted.
+					// (past before to be deleted CP)
+
+					// Should not exceed vector boundaries, if
+					// g_rollback_thresh*2 < (g_cp_thresh - g_cost_cp)
+					if ((*(rit_pre_last + g_rollback_thresh)).second < last_pos) {
+						checkpoint_forbidden = true;
+						break;
+					}
+				} else {
+					break;
+				}
+			}
+
+			left_pos = rit_pre_last->second;
+
+			if ((!g_use_weights && !(last_pos <= left_pos))
+					|| (g_use_weights && !(last_pos < left_pos))) {
+				break;
+			}
+
+			// Hop a->b is not allowed if last pos of b is at a
+			// but a is not b
+			// e.g. instruction x with mem access at y
+			// a is WP y, and b is BP x
+			// OR a is BP x, and b is WP y but instruction
+			// at position a has also memory access at y
+			if (rit_pre_last != result.rend()) {
+				if ((last_pos == rit_pre_last->second) &&
+						rit_pre_last->first != *best_te) {
+					break;
+				}
+			}
+
+			// Right hop will be deleted
+			// => Recalculate costs
+
+			// If to be deleted hop is a checkpoint:
+			// There will be no past in the trace
+			// Reconfigure result and iterators
+			if (rit_last->first.second == ACCESS_CHECKPOINT) {
+				costs = m_checkpoints[rit_last->first.first].first;
+
+				result.clear();
+				std::vector<result_tuple > *tmp = &(m_checkpoints[rit_last->first.first].second);
+				result.insert(result.end(), tmp->begin(), tmp->end());
+
+				rit_last = result.rbegin();
+				rit_pre_last = result.rbegin();
+				rit_pre_last++;
+
+				// CP could be removed from m_checkpoints, but to do this
+				// m_checkpoints needs to be a map (CP-ID is implicit as vector index)
+
+				continue;
+			}
+
+			if (rit_pre_last->first == rit_last->first) {
+				costs -= COST_NO_CHANGE;
+			} else {
+				costs -= COST_CHANGE;
+			}
+
+			// Delete element described by rit_last
+			// Deletion of elements described by reverse iterator works like this
+			result.erase((rit_last+1).base());
+			rit_last = rit_pre_last;
+			rit_pre_last++;
+		}
+
+
+		// Add costs for new hop
+		if (*best_te == (result.back()).first) {
+			costs += COST_NO_CHANGE;
+		} else {
+			costs += COST_CHANGE;
+		}
+
+		// Check if Checkpoint needed
+		if (g_use_checkpoints && (costs > g_cp_thresh) && !checkpoint_forbidden) {
+			checkpoint_tuple_t new_cp(costs, std::vector<result_tuple >(result));
+			m_checkpoints.push_back(new_cp);
+			result.clear();
+			result.push_back(result_tuple(trace_event_tuple_t(m_next_cp_id++,
+																ACCESS_CHECKPOINT),
+											trace_pos));
+			costs = g_cost_cp;
+
+			m_resultCollector->addCheckpoint(trace_pos);
+		} else {
+			// Add new hop
+			result.push_back(result_tuple(*best_te, trace_pos));
+		}
+
+		m_resultCollector->addResult(result, costs);
+	}
+	// MAX MEM USAGE
+	m_resultCollector->setMaxMemUsage();
+
+	return true;
+}

tools/compute-hops/SmartAlgorithm.hpp

@@ -0,0 +1,40 @@
+#ifndef SMART_ALGORITHM__HPP
+#define SMART_ALGORITHM__HPP
+
+#include <map>
+#include <vector>
+
+#include "BasicAlgorithm.hpp"
+#include "../../src/core/util/smarthops/TraceReader.hpp"
+#include "ResultCollector.hpp"
+
+class ResultCollector;
+
+using namespace fail;
+
+// 			       COSTS         NAVIGATIONAL RECORD
+typedef std::pair<unsigned int, std::vector<result_tuple > > checkpoint_tuple_t;
+
+class SmartAlgorithm : public BasicAlgorithm {
+public:
+
+	SmartAlgorithm(ResultCollector *rc) : BasicAlgorithm(rc), m_next_cp_id(0) {}
+	virtual ~SmartAlgorithm() {}
+
+    bool calculateAllHops(TraceReader& trace);
+
+private:
+
+    bool
+	occurenceInPosIntervall(unsigned int intervall_low,
+							unsigned int intervall_high,
+							address_t add,
+							mem_access_type_e acc);
+
+    std::map<trace_event_tuple_t, trace_pos_t> m_last_positions;
+    std::vector<checkpoint_tuple_t > m_checkpoints;
+    unsigned int m_next_cp_id;
+};
+
+#endif // SMART_ALGORITHM__HPP
+

tools/compute-hops/main.cc

@@ -0,0 +1,313 @@
+// #include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <cstring>
+
+#include "../../src/core/util/smarthops/TraceReader.hpp"
+#include "BasicAlgorithm.hpp"
+#include "SmartAlgorithm.hpp"
+#include "SimpleAlgorithm.hpp"
+#include "ResultCollector.hpp"
+
+#include "../../src/core/util/gzstream/gzstream.h"
+
+#include "util/CommandLine.hpp"
+#include "util/Logger.hpp"
+
+#define STDOUT_CMD_STRING "-"
+
+using namespace fail;
+
+typedef enum {
+	ALGO_SIMPLE,
+	ALGO_SMART,
+} algorithm_e;
+
+bool g_use_weights;
+bool g_use_watchpoints;
+bool g_use_checkpoints;
+
+unsigned int g_cp_thresh;
+unsigned int g_cost_cp;
+unsigned int g_rollback_thresh;
+
+std::ofstream g_cp_ofstream;
+
+Logger LOG("hop-calculator", false);
+
+int main(int argc, char *argv[])
+{
+
+	// Manually fill the command line option parser
+	CommandLine &cmd = CommandLine::Inst();
+
+	cmd.addOption("", "", Arg::None, "USAGE: compute-hops [options]");
+	CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h,--help \tPrint usage and exit");
+
+	CommandLine::option_handle ALGORITHM =
+		cmd.addOption("a", "algorithm", Arg::Required,
+			"-a,--algorithm \tHop algorithm (\"simple\"/\"smart\", default: \"none\")");
+
+	CommandLine::option_handle OUTPUT_MODE =
+		cmd.addOption("m", "output-mode", Arg::Required,
+			"-m,--output-mode \tOutput mode (\"results\"/\"costs\"/\"statistics\", default: \"results\")");
+
+	CommandLine::option_handle TRACE_FILE =
+		cmd.addOption("i", "input-file", Arg::Required,
+			"-i,--input-file \tInput trace file path");
+
+	CommandLine::option_handle OUTPUT_FILE =
+		cmd.addOption("o", "output-file", Arg::Required,
+			"-o,--output-file \tOutput file, default: stdout");
+
+	CommandLine::option_handle OUTPUT_FILE_PROTOBUF =
+		cmd.addOption("b", "protobuf-output", Arg::None,
+			"-b,--protobuf-output \tOutput file will be created in protobuf (HopChain) format");
+
+	CommandLine::option_handle USE_COSTS =
+		cmd.addOption("c", "use-costs", Arg::None,
+			"-c,--use-costs \tUse hop costs for calculations of Smart-Hopping algorithm");
+
+	CommandLine::option_handle USE_WATCHPOINTS =
+		cmd.addOption("w", "use-watchpoints", Arg::None,
+			"-w,--use-watchpoints \tUse watchpoints as additional hop candidates");
+
+	CommandLine::option_handle USE_CHECKPOINTS =
+		cmd.addOption("", "use-checkpoints", Arg::None,
+			"--use-checkpoints \tUse checkpoints to cap costs in case of long hop chains");
+
+	CommandLine::option_handle CP_THRESH =
+		cmd.addOption("", "cp-costs-threshold", Arg::Required,
+			"--cp-costs-threshold \tIf costs of a hop-chain reach this threshold, a checkpoint will be created");
+
+	CommandLine::option_handle CP_COSTS =
+		cmd.addOption("", "cp-costs", Arg::Required,
+			"--cp-costs \tThe costs for reloading a checkpoint into the system");
+
+	CommandLine::option_handle CP_ROLLBACK_THRESH =
+		cmd.addOption("", "cp-rollback-threshold", Arg::Required,
+			"--cp-rollback-threshold \tMinial number of hops to roll back current solution beyond checkpoint. Must be smaller than ((cp_cost_thresh - cp_costs) / 2)");
+
+	CommandLine::option_handle CHECKPOINT_OUTPUT_FILE =
+		cmd.addOption("", "cp-output", Arg::Required,
+			"--cp-output \tCheckpoint output file");
+
+	for (int i = 1; i < argc; ++i) {
+		cmd.add_args(argv[i]);
+	}
+	if (!cmd.parse()) {
+		std::cerr << "Error parsing arguments." << std::endl;
+		return 1;
+	}
+
+	if (cmd[HELP]) {
+		cmd.printUsage();
+		if (cmd[HELP]) {
+			exit(0);
+		} else {
+			exit(1);
+		}
+	}
+
+	if (cmd[USE_CHECKPOINTS]) {
+		g_use_checkpoints = true;
+	} else {
+		g_use_checkpoints = false;
+	}
+
+	if (cmd[USE_WATCHPOINTS]) {
+		g_use_watchpoints = true;
+	} else {
+		g_use_watchpoints = false;
+	}
+
+	if (cmd[USE_COSTS]) {
+		g_use_weights = true;
+	} else {
+		g_use_weights = false;
+	}
+
+	algorithm_e algorithm;
+
+	if (cmd[ALGORITHM].count() > 0) {
+		std::string alg(cmd[ALGORITHM].first()->arg);
+		if (alg == "simple") {
+			algorithm = ALGO_SIMPLE;
+		} else if (alg == "smart") {
+			algorithm = ALGO_SMART;
+		} else {
+			exit(-1);
+		}
+	} else {
+		// default
+		algorithm = ALGO_SMART;
+	}
+
+	output_mode_e output_mode;
+
+	if (cmd[OUTPUT_MODE].count() > 0) {
+		std::string outp(cmd[OUTPUT_MODE].first()->arg);
+		if (outp == "results") {
+			output_mode = OUTPUT_RESULT;
+		} else if (outp == "costs") {
+			output_mode = OUTPUT_COSTS;
+		} else if (outp == "statistics") {
+			output_mode = OUTPUT_STATISTICS;
+		} else {
+			LOG << "Unkown output mode: " << outp << std::endl;
+			exit(-1);
+		}
+	} else {
+		// default
+		output_mode = OUTPUT_RESULT;
+	}
+
+	fail::TraceReader trace;
+
+	if (cmd[TRACE_FILE].count() > 0) {
+		const char *filename = cmd[TRACE_FILE].first()->arg;
+		if (!trace.openTraceFile(filename, 0)) {
+			LOG << "Unable to open and parse input trace file " << filename << std::endl;
+			return 1;
+		}
+	} else {
+		LOG << "Input trace file path not defined" << std::endl;
+		exit(-1);
+	}
+
+	std::ostream *outFile;
+
+	if (cmd[OUTPUT_FILE].count() > 0) {
+		std::string filename(cmd[OUTPUT_FILE].first()->arg);
+
+		if (filename.compare(STDOUT_CMD_STRING) == 0) {
+			outFile = &std::cout;
+		} else {
+			std::ofstream *of = new std::ofstream(filename.c_str());
+			if (!of->is_open()) {
+				LOG << "Unable to open output file " << filename << std::endl;
+				exit(-1);
+			}
+			outFile = of;
+		}
+	} else {
+		outFile = &std::cout;
+	}
+
+	if (cmd[OUTPUT_FILE_PROTOBUF] && outFile == &std::cout) {
+		LOG << "If protobuf output format is selected, output file must be defined" << std::endl;
+		exit(-1);
+	}
+
+	if (cmd[USE_COSTS] && !(algorithm == ALGO_SMART)) {
+		LOG << "Using costs for calculations is only possible with Smart-Hopping algorithm" << std::endl;
+		exit(-1);
+	}
+
+	if (cmd[CP_THRESH]) {
+		if (cmd[CP_THRESH].count() != 1) {
+			LOG << "Could not parse cp-costs-threshold" << std::endl;
+			exit(-1);
+		}
+		g_cp_thresh = strtoul(cmd[CP_THRESH].first()->arg, NULL, 0);
+	}
+
+	if (cmd[CP_COSTS]) {
+		if (cmd[CP_COSTS].count() != 1) {
+			LOG << "Could not parse cp-costs" << std::endl;
+			exit(-1);
+		}
+		g_cost_cp = strtoul(cmd[CP_COSTS].first()->arg, NULL, 0);
+	}
+
+	if (cmd[CP_ROLLBACK_THRESH]) {
+		if (cmd[CP_ROLLBACK_THRESH].count()!=1) {
+			LOG << "Could not parse cp-rollback-threshold" << std::endl;
+			exit(-1);
+		}
+//		if (strcmp(argv[9],"max")==0) {
+//			g_rollback_thresh = (g_cp_thresh - g_cost_cp - 5)/2;
+//		}
+		g_cost_cp = strtoul(cmd[CP_ROLLBACK_THRESH].first()->arg, NULL, 0);
+	}
+
+	if (cmd[USE_CHECKPOINTS] && !(cmd[CHECKPOINT_OUTPUT_FILE].count() > 0)) {
+		LOG << "If checkpointing is enabled, --cp-output must be defined" << std::endl;
+		exit(-1);
+	}
+
+	if (cmd[CHECKPOINT_OUTPUT_FILE].count() > 0) {
+		std::string filename(cmd[CHECKPOINT_OUTPUT_FILE].first()->arg);
+		g_cp_ofstream.open(filename.c_str());
+		if(!g_cp_ofstream.is_open()) {
+			LOG << "Unable to open cp_out_file " << filename << std::endl;
+			exit(-1);
+		}
+	}
+
+	if (cmd[USE_CHECKPOINTS] && (!cmd[CP_THRESH] ||
+			!cmd[CP_COSTS]|| !cmd[CP_ROLLBACK_THRESH])) {
+		LOG << "If using checkpointing is enabled, also cp-costs-threshold, "
+				"cp-costs and cp-rollback-threshold must be defined" << std::endl;
+		exit(-1);
+	}
+
+	if (!cmd[USE_CHECKPOINTS] && (cmd[CP_THRESH].count() ||
+			(cmd[CP_COSTS].count() > 0) || (cmd[CP_ROLLBACK_THRESH].count() > 0))) {
+		LOG << "If using checkpointing is disabled, cp-costs-threshold, "
+				"cp-costs and cp-rollback-threshold must not be defined" << std::endl;
+		exit(-1);
+	}
+
+	if (g_use_checkpoints && !(g_cp_thresh > g_cost_cp)) {
+		LOG << "cp_cost_thresh needs to be bigger than cp_costs" << std::endl;
+		exit(1);
+	}
+
+	if (g_use_checkpoints && !(g_rollback_thresh < ((g_cp_thresh - g_cost_cp) / 2))) {
+		LOG << "cp_rollback_hop_thresh needs to be smaller than "
+				"((cp_cost_thresh - cp_costs) / 2)" << std::endl;
+		exit(1);
+	}
+
+
+	ogzstream zipstream;
+
+	ResultCollector rc(*outFile, output_mode);
+
+	if (cmd[OUTPUT_FILE_PROTOBUF]) {
+		zipstream.open(cmd[OUTPUT_FILE].first()->arg);
+		rc.setProtoOStream(new ProtoOStream(&zipstream));
+	}
+
+	BasicAlgorithm *algo;
+
+   	switch (algorithm) {
+		case ALGO_SMART:
+			algo = new SmartAlgorithm(&rc);
+			break;
+		case ALGO_SIMPLE:
+			algo = new SimpleAlgorithm(&rc);
+			break;
+		default:
+			break;
+	}
+
+   	rc.startTimer();
+   	algo->calculateAllHops(trace);
+    rc.stopTimer();
+
+    rc.finish();
+
+    // ToDo: close output file if not stdout
+	if (outFile != &std::cout) {
+		((std::ofstream*)outFile)->close();
+		delete (std::ofstream*)outFile;
+	}
+
+    if (g_use_checkpoints) {
+    	g_cp_ofstream.close();
+    }
+
+    return 0;
+}

tools/dump-hops/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(SRCS
+  DumpHops.cc
+)
+
+add_executable(dump-hops ${SRCS})
+target_link_libraries(dump-hops ${PROTOBUF_LIBRARY} fail-util fail-comm)
+install(TARGETS dump-hops RUNTIME DESTINATION bin)

tools/dump-hops/DumpHops.cc

@@ -0,0 +1,110 @@
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include "comm/InjectionPointHopsMessage.pb.h"
+#include "util/ProtoStream.hpp"
+#include "../../src/core/util/Logger.hpp"
+#include "../../src/core/util/gzstream/gzstream.h"
+#include "util/CommandLine.hpp"
+
+using namespace fail;
+using std::stringstream;
+using std::endl;
+using std::cout;
+using std::cerr;
+using std::hex;
+using std::dec;
+
+Logger LOG("dump-hops", true);
+
+std::istream& openStream(const char *input_file,
+	std::ifstream& normal_stream, igzstream& gz_stream) {
+	normal_stream.open(input_file);
+	if (!normal_stream) {
+		LOG << "couldn't open " << input_file << endl;
+		exit(-1);
+	}
+	unsigned char b1, b2;
+	normal_stream >> b1 >> b2;
+
+	if (b1 == 0x1f && b2 == 0x8b) {
+		normal_stream.close();
+		gz_stream.open(input_file);
+		if (!gz_stream) {
+			LOG << "couldn't open " << input_file << endl;
+			exit(-1);
+		}
+		//LOG << "opened file " << input_file << " in GZip mode" << endl;
+		return gz_stream;
+	}
+
+	normal_stream.seekg(0);
+
+	//LOG << "opened file " << input_file << " in normal mode" << endl;
+	return normal_stream;
+}
+
+int main(int argc, char *argv[])
+{
+	InjectionPointMessage ev;
+
+	CommandLine &cmd = CommandLine::Inst();
+	cmd.addOption("", "", Arg::None, "usage: dump-hops [options] hopfile.hp");
+	CommandLine::option_handle HELP =
+		cmd.addOption("h", "help", Arg::None, "-h/--help \tPrint usage and exit");
+//	CommandLine::option_handle STATS =
+//		cmd.addOption("s", "stats", Arg::None,
+//			"-s/--stats \tShow trace stats");
+
+	for (int i = 1; i < argc; ++i) {
+		cmd.add_args(argv[i]);
+	}
+	if (!cmd.parse()) {
+		std::cerr << "Error parsing arguments." << std::endl;
+		return 1;
+	}
+
+	if (cmd[HELP] || cmd.parser()->nonOptionsCount() != 1) {
+		cmd.printUsage();
+		if (cmd[HELP]) {
+			exit(0);
+		} else {
+			exit(1);
+		}
+	}
+
+//	bool stats_only = cmd[STATS];
+//	bool extended = cmd[EXTENDED_TRACE];
+
+	std::ifstream normal_stream;
+	igzstream gz_stream;
+	ProtoIStream ps(&openStream(cmd.parser()->nonOption(0), normal_stream, gz_stream));
+
+//	uint64_t stats_instr = 0, stats_reads = 0, stats_writes = 0, starttime = 0;
+
+	while (ps.getNext(&ev)) {
+		cout << "T" << ev.target_trace_position() ;
+
+		if (ev.has_checkpoint_id()) {
+			cout << " CP" << ev.checkpoint_id();
+		}
+
+		for (int i = 0; i < ev.hops_size(); i++) {
+			const InjectionPointMessage_Hops &hops = ev.hops(i);
+			cout << " " <<
+				((hops.accesstype()==InjectionPointMessage_Hops_AccessType_EXECUTE)?"X":
+						((hops.accesstype()==InjectionPointMessage_Hops_AccessType_READ)?"R":"W"))
+						<< hex << hops.address() << dec;
+		}
+		cout << "\n";
+	}
+
+//	if (stats_only) {
+//		cout << "#instructions: " << stats_instr << "\n"
+//		     << "#memR:         " << stats_reads << "\n"
+//		     << "#memW:         " << stats_writes << "\n"
+//		     << "duration:      " << (acctime - starttime + 1) << endl;
+//	}
+
+	return 0;
+}