From 8b5098abdd11c1d65efbd403c778634cccd9167b Mon Sep 17 00:00:00 2001 From: Lars Rademacher Date: Sat, 30 Nov 2013 22:26:17 +0100 Subject: [PATCH] tools: added compute-hops and dump-hops tools As these tools work closely together with fail components, its easiest, to build them in this context. As these tools don't really matter for fail use, they might never be pushed to the master branch. Change-Id: I8c8bd80376d0475f08a531a995d829e85032371b --- src/core/comm/InjectionPointHopsMessage.proto | 9 +- src/core/cpn/CMakeLists.txt | 2 +- src/core/util/smarthops/TraceReader.cc | 11 +- src/core/util/smarthops/TraceReader.hpp | 12 +- .../lra-simple-panda/CMakeLists.txt | 2 +- tools/CMakeLists.txt | 10 + tools/compute-hops/BasicAlgorithm.hpp | 23 ++ tools/compute-hops/CMakeLists.txt | 13 + tools/compute-hops/ResultCollector.cc | 172 ++++++++++ tools/compute-hops/ResultCollector.hpp | 91 +++++ tools/compute-hops/SimpleAlgorithm.cc | 125 +++++++ tools/compute-hops/SimpleAlgorithm.hpp | 30 ++ tools/compute-hops/SmartAlgorithm.cc | 218 ++++++++++++ tools/compute-hops/SmartAlgorithm.hpp | 40 +++ tools/compute-hops/main.cc | 313 ++++++++++++++++++ tools/dump-hops/CMakeLists.txt | 7 + tools/dump-hops/DumpHops.cc | 110 ++++++ 17 files changed, 1180 insertions(+), 8 deletions(-) create mode 100644 tools/compute-hops/BasicAlgorithm.hpp create mode 100644 tools/compute-hops/CMakeLists.txt create mode 100644 tools/compute-hops/ResultCollector.cc create mode 100644 tools/compute-hops/ResultCollector.hpp create mode 100644 tools/compute-hops/SimpleAlgorithm.cc create mode 100644 tools/compute-hops/SimpleAlgorithm.hpp create mode 100644 tools/compute-hops/SmartAlgorithm.cc create mode 100644 tools/compute-hops/SmartAlgorithm.hpp create mode 100644 tools/compute-hops/main.cc create mode 100644 tools/dump-hops/CMakeLists.txt create mode 100644 tools/dump-hops/DumpHops.cc diff --git a/src/core/comm/InjectionPointHopsMessage.proto b/src/core/comm/InjectionPointHopsMessage.proto index 6aa59370..a97e9968 100644 --- a/src/core/comm/InjectionPointHopsMessage.proto +++ b/src/core/comm/InjectionPointHopsMessage.proto @@ -5,7 +5,12 @@ message InjectionPointMessage { // If checkpoint must be used for this hop chain, id is set properly optional uint32 checkpoint_id = 1; - + + // If we need to knwo the target dynamic instruction offset, + // here it is + optional uint32 target_trace_position = 4; + + // Repeated groups can't be defined as notempty, so a manual // non-empty check is required at usage @@ -20,4 +25,4 @@ message InjectionPointMessage { } required AccessType accesstype = 2; } -} \ No newline at end of file +} diff --git a/src/core/cpn/CMakeLists.txt b/src/core/cpn/CMakeLists.txt index 94dba088..eaeee8a8 100644 --- a/src/core/cpn/CMakeLists.txt +++ b/src/core/cpn/CMakeLists.txt @@ -16,7 +16,7 @@ add_library(fail-cpn ${SRCS}) # if hop-chains need to be calculated by the server, we # the smarthopping module if(CONFIG_INJECTIONPOINT_HOPS) - add_dependencies(fail-cpn fail-smarthopping) + add_dependencies(fail-cpn fail-smarthops) endif(CONFIG_INJECTIONPOINT_HOPS) add_dependencies(fail-cpn fail-comm) diff --git a/src/core/util/smarthops/TraceReader.cc b/src/core/util/smarthops/TraceReader.cc index ceac1787..96eec1c8 100644 --- a/src/core/util/smarthops/TraceReader.cc +++ b/src/core/util/smarthops/TraceReader.cc @@ -41,16 +41,25 @@ std::istream& openStream(const char *input_file, return normal_stream; } -void TraceReader::openTraceFile(const char *filename) +bool TraceReader::openTraceFile(const char *filename, unsigned int num_inst) { normal_stream = new std::ifstream(); gz_stream = new igzstream(); ps = new fail::ProtoIStream(&openStream(filename, *normal_stream, *gz_stream, m_log)); + + m_max_num_inst = num_inst; + + return true; } bool TraceReader::getNextTraceEvents(trace_pos_t& trace_pos, std::vector& trace_events) { + // Stop after fixed number of instructions, if given as command line argument + if ((m_max_num_inst > 0) && (m_current_position > m_max_num_inst)) { + return false; + } + trace_pos = m_current_position; // Delivered trace_events vector does not have to be diff --git a/src/core/util/smarthops/TraceReader.hpp b/src/core/util/smarthops/TraceReader.hpp index 3273e448..4cb30d39 100644 --- a/src/core/util/smarthops/TraceReader.hpp +++ b/src/core/util/smarthops/TraceReader.hpp @@ -28,8 +28,13 @@ typedef std::pair trace_event_tuple_t; class TraceReader { public: - TraceReader() : m_current_position(1), ps(0), normal_stream(0), - gz_stream(0), ev_avail(false), m_log("TraceReader", false) {} + TraceReader() : m_current_position(1), + ps(0), + normal_stream(0), + gz_stream(0), + m_max_num_inst(0), + ev_avail(false), + m_log("TraceReader", false) {} ~TraceReader(); @@ -37,12 +42,13 @@ class TraceReader { bool getNextTraceEvents(trace_pos_t& trace_pos, std::vector& trace_events); - void openTraceFile(const char *filename); + bool openTraceFile(const char *filename, unsigned int num_inst = 0); private: unsigned int m_current_position; ProtoIStream* ps; std::ifstream *normal_stream; igzstream *gz_stream; + unsigned int m_max_num_inst; Trace_Event ev; bool ev_avail; diff --git a/src/experiments/lra-simple-panda/CMakeLists.txt b/src/experiments/lra-simple-panda/CMakeLists.txt index 995eb4c7..d7912abc 100644 --- a/src/experiments/lra-simple-panda/CMakeLists.txt +++ b/src/experiments/lra-simple-panda/CMakeLists.txt @@ -32,5 +32,5 @@ target_link_libraries(fail-${EXPERIMENT_NAME} ${PROTOBUF_LIBRARY}) ## This is the example's campaign server distributing experiment parameters add_executable(${EXPERIMENT_NAME}-server main.cc) -target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-smarthopping fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} -lmysqlclient -Wl,--end-group) +target_link_libraries(${EXPERIMENT_NAME}-server -Wl,--start-group fail-${EXPERIMENT_NAME} fail-sal fail-util fail-cpn fail-smarthops fail-comm ${PROTOBUF_LIBRARY} ${Boost_THREAD_LIBRARY} -lmysqlclient -Wl,--end-group) install(TARGETS ${EXPERIMENT_NAME}-server RUNTIME DESTINATION bin) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f11196d8..737f53de 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -3,6 +3,8 @@ option(BUILD_IMPORT_TRACE "Build the trace import tool?" OFF) option(BUILD_PRUNE_TRACE "Build the trace prune tool?" OFF) option(BUILD_CONVERT_TRACE "Build the trace converter tool?" OFF) +option(BUILD_COMPUTE_HOPS "Build the compute hops tool?" OFF) +option(BUILD_DUMP_HOPS "Build the hops dump tool?" OFF) ### Setup search paths for headers ## include_directories(${CMAKE_CURRENT_BINARY_DIR}/../src/core) @@ -23,3 +25,11 @@ endif(BUILD_DUMP_TRACE) if(BUILD_CONVERT_TRACE) add_subdirectory(convert-trace) endif(BUILD_CONVERT_TRACE) + +if(BUILD_COMPUTE_HOPS) + add_subdirectory(compute-hops) +endif(BUILD_COMPUTE_HOPS) + +if(BUILD_DUMP_HOPS) + add_subdirectory(dump-hops) +endif(BUILD_DUMP_HOPS) diff --git a/tools/compute-hops/BasicAlgorithm.hpp b/tools/compute-hops/BasicAlgorithm.hpp new file mode 100644 index 00000000..e1891771 --- /dev/null +++ b/tools/compute-hops/BasicAlgorithm.hpp @@ -0,0 +1,23 @@ +#ifndef __BASIC_ALGORITHM__HPP +#define __BASIC_ALGORITHM__HPP + +#include "../../src/core/util/smarthops/TraceReader.hpp" +class ResultCollector; + +class BasicAlgorithm { + public: + /** + * + * @returns boolean value for calculation success + */ + virtual bool calculateAllHops(fail::TraceReader& trace) = 0; + + BasicAlgorithm(ResultCollector *rc) {m_resultCollector = rc;} + virtual ~BasicAlgorithm() {} + + protected: + ResultCollector *m_resultCollector; +}; + +#endif // __BASIC_ALGORITHM__HPP + diff --git a/tools/compute-hops/CMakeLists.txt b/tools/compute-hops/CMakeLists.txt new file mode 100644 index 00000000..cfc69e51 --- /dev/null +++ b/tools/compute-hops/CMakeLists.txt @@ -0,0 +1,13 @@ +set(SRCS + ResultCollector.cc + SimpleAlgorithm.cc + SmartAlgorithm.cc +) + +if(NOT CONFIG_INJECTIONPOINT_HOPS) + message (FATAL_ERROR "For building the compute-hops tool you need to enable CONFIG_INJECTIONPOINT_HOPS") +endif() + +add_executable(compute-hops main.cc ${SRCS}) +target_link_libraries(compute-hops ${PROTOBUF_LIBRARY} fail-util fail-comm) +install(TARGETS compute-hops RUNTIME DESTINATION bin) diff --git a/tools/compute-hops/ResultCollector.cc b/tools/compute-hops/ResultCollector.cc new file mode 100644 index 00000000..11c5ccec --- /dev/null +++ b/tools/compute-hops/ResultCollector.cc @@ -0,0 +1,172 @@ +/* + * StatisticsCollector.cc + * + * Created on: 21.08.2013 + * Author: lrade + */ + +#include "ResultCollector.hpp" + +#include +#include +#include +#include +#include + +#include + +#include "comm/InjectionPointHopsMessage.pb.h" + +// #include + +using std::hex; +using std::dec; + +char separator = ' '; + +extern std::ofstream g_cp_ofstream; + +extern fail::Logger LOG; + +void +ResultCollector::setProtoOStream(ProtoOStream* protoOStream) +{ + ps = protoOStream; +} + +unsigned int calculate_costs(std::vector& res) +{ + std::vector::iterator it = res.begin(); + std::vector::iterator it_ahead = res.begin(); + it_ahead++; + + // Cost COST_CHANGE for first hop + // Costs of all following hops are calculated only on basis of + // difference between two consecutive hop_positions + unsigned int costs = COST_CHANGE; + + while (it != res.end() && it_ahead != res.end()) { + if (it->first == it_ahead->first) { + costs += COST_NO_CHANGE; + } else { + costs += COST_CHANGE; + } + it_ahead++; + it++; + } + + return costs; +} + +void +ResultCollector::addResult(std::vector& res, unsigned int costs) +{ + m_result_size++; + + // Costs not calculated in hop calculator (e.g. dijkstra)? => Do it here + if (costs == 0) { + costs = calculate_costs(res); + } + + if (m_output_mode == OUTPUT_COSTS) { + m_ostream << m_res_count++ << separator << costs << '\n'; + } else if (m_output_mode == OUTPUT_RESULT) { + if (ps) { + InjectionPointMessage hc; + hc.set_costs(costs); + hc.set_target_trace_position(res.back().second); + // If checkpoint at beginning of hop-chain, add its id to InjectionPointMessage + std::vector::iterator it_hop = res.begin(); + if (it_hop != res.end() && it_hop->first.second == ACCESS_CHECKPOINT) { + hc.set_checkpoint_id(it_hop->first.first); + it_hop++; + } + + for(;it_hop != res.end(); + it_hop++) { + InjectionPointMessage_Hops *hop = hc.add_hops(); + hop->set_address(it_hop->first.first); + + enum InjectionPointMessage_Hops_AccessType at; + switch (it_hop->first.second) { + case ACCESS_NONE: + at = InjectionPointMessage_Hops_AccessType_EXECUTE; + break; + case ACCESS_READ: + at = InjectionPointMessage_Hops_AccessType_READ; + break; + case ACCESS_WRITE: + at = InjectionPointMessage_Hops_AccessType_WRITE; + break; + case ACCESS_READORWRITE: + LOG << "ReadOrWrite memory access event not yet" + " covered" << std::endl; + exit(-1); + break; + case ACCESS_CHECKPOINT: + LOG << "Checkpoint not allowed after beginning of hop chain" << std::endl; + exit(-1); + } + hop->set_accesstype(at); + } + ps->writeMessage(&hc); + } else { + for(std::vector::iterator it_hop = res.begin(); + it_hop != res.end(); + it_hop++) { + address_t add = it_hop->first.first; + mem_access_type_e mem_acc_type = it_hop->first.second; + std::string prefix = (mem_acc_type == ACCESS_READ)?"R":((mem_acc_type == ACCESS_WRITE)?"W":((mem_acc_type == ACCESS_NONE)?"X":((mem_acc_type == ACCESS_CHECKPOINT)?"C":""))); + m_ostream << prefix << hex << add << dec << separator; + } + + if (res.size() > 0) + m_ostream << '\n'; + } + } else if (m_output_mode == OUTPUT_STATISTICS) { + // Calculate mean + // http://www.heikohoffmann.de/htmlthesis/node134.html + // c(t+1) = c(t) + (1/(t+1))*(x - c(t)) + m_it_mean_costs = m_it_mean_costs + (1/(double)m_result_size)*((double)costs-m_it_mean_costs); + + } +} + +void +ResultCollector::addCheckpoint(unsigned int pos) +{ + g_cp_ofstream << m_checkpoint_count++ << separator << pos << '\n'; +} + +void +ResultCollector::startTimer() +{ + m_timer.startTimer(); +} + +void +ResultCollector::stopTimer() +{ + m_timer.stopTimer(); +} + +void +ResultCollector::setMaxMemUsage() +{ +// struct proc_t usage; +// look_up_our_self(&usage); +// m_mem_usage = usage.vsize; +} + +void +ResultCollector::finish() +{ + // Print results if buffered + + // Print statistics + if (m_output_mode == OUTPUT_STATISTICS) { + + m_ostream << m_result_size << separator << m_timer.getRuntimeAsDouble() << separator + << m_mem_usage << separator << m_it_mean_costs << separator << m_checkpoint_count << '\n'; + } +} diff --git a/tools/compute-hops/ResultCollector.hpp b/tools/compute-hops/ResultCollector.hpp new file mode 100644 index 00000000..67205b35 --- /dev/null +++ b/tools/compute-hops/ResultCollector.hpp @@ -0,0 +1,91 @@ +/* + * ResultCollector.hpp + * + * Created on: 23.08.2013 + * Author: Lars Rademacher + */ + +#ifndef RESULTCOLLECTOR_HPP_ +#define RESULTCOLLECTOR_HPP_ + +#include +#include +#include + +#include "../../src/core/util/smarthops/TraceReader.hpp" + +#include "util/WallclockTimer.hpp" + +using namespace fail; + +typedef unsigned int trace_pos_t; + +typedef std::pair result_tuple; + +typedef enum { + OUTPUT_RESULT , + OUTPUT_COSTS , + OUTPUT_STATISTICS , +} output_mode_e; + +using fail::ProtoOStream; + +#define COST_CHANGE 1 +#define COST_NO_CHANGE 2 + +class ResultCollector { +public: + + ResultCollector(std::ostream& out_stream, output_mode_e output_mode) : + m_ostream(out_stream), + m_res_count(1), + m_output_mode(output_mode), + m_mem_usage(0), + m_result_size(0), + m_checkpoint_count(1), + m_it_mean_costs(0), + m_max_costs(0), + ps(0){} + + void + addResult(std::vector& res, unsigned int costs); + + void + addCheckpoint(unsigned int pos); + + void + startTimer(); + + void + stopTimer(); + + void + setMaxMemUsage(); + + void + setProtoOStream(ProtoOStream* protoOStream); + + // Prints buffered results on output stream + void + finish(); + +private: + + std::ostream& m_ostream; + unsigned int m_res_count; + output_mode_e m_output_mode; + unsigned long m_mem_usage; + unsigned int m_result_size; + unsigned int m_checkpoint_count; + + double m_it_mean_costs; + unsigned int m_max_costs; + + fail::WallclockTimer m_timer; + + fail::ProtoOStream *ps; +}; + +#endif /* STATISTICSCOLLECTOR_HPP_ */ + + diff --git a/tools/compute-hops/SimpleAlgorithm.cc b/tools/compute-hops/SimpleAlgorithm.cc new file mode 100644 index 00000000..963cfd92 --- /dev/null +++ b/tools/compute-hops/SimpleAlgorithm.cc @@ -0,0 +1,125 @@ +#include "SimpleAlgorithm.hpp" + +#include +//#include +#include +#include + +#include "../../src/core/util/smarthops/TraceReader.hpp" +#include "ResultCollector.hpp" + +using namespace std; +using namespace fail; + +// Idea for CPs +// for each new trace_event: costs > THRESH? +// => Use latest CP +// Costs still > THRESH? +// => New CP + +// Additional Map, which tracks down, how often a instruction occured +// since creation of the latest CP +// => MEM USAGE x 2 + +extern unsigned int g_cp_thresh; +extern unsigned int g_cost_cp; + +extern bool g_use_watchpoints; +extern bool g_use_checkpoints; +extern fail::Logger LOG; + +bool SimpleAlgorithm::calculateAllHops(TraceReader& trace) +{ + trace_pos_t trace_pos; + vector trace_events; + + vector empty_result; + + // Parse all executed instructions + while (trace.getNextTraceEvents(trace_pos, trace_events)) { + + // Policy when multiple trace events: Choose the most rare one + trace_event_tuple_t *candidate = NULL; + //vector *cand_pos = NULL; + unsigned long num_hops = numeric_limits::max(); + + for (vector::iterator it_te = trace_events.begin(); + it_te != trace_events.end(); + it_te++) { + // Don't use watchpoints? + if (!g_use_watchpoints && (it_te->second != ACCESS_NONE)) { + continue; + } + + // Find last pos of current trace event + //map >::iterator it_oc = m_occurences.find(*it_te); + map::iterator it_oc = m_occurences.find(*it_te); + + // Not known? => Add + if (it_oc == m_occurences.end()) { + //m_occurences.insert(pair >(*it_te, vector(1, trace_pos))); + m_occurences.insert(pair(*it_te, 1)); + num_hops = 1; + it_oc = m_occurences.find(*it_te); + candidate = (trace_event_tuple_t*)&(it_oc->first); + //cand_pos = (vector*)&(it_oc->second); + } else { + it_oc->second++; + if (it_oc->second < num_hops) { + candidate = (trace_event_tuple_t*)&(it_oc->first); + //cand_pos = (vector*)&(it_oc->second); + num_hops = it_oc->second; + } + } + + // Do the same thing for the occ_cp map + map::iterator it_oc_cp = m_occ_cp.find(*it_te); + + // Not known? => Add + if (it_oc_cp == m_occ_cp.end()) { + m_occ_cp.insert(pair(*it_te, 1)); + } else { + it_oc_cp->second++; + } + + } + + if (candidate == NULL ) { //|| cand_pos == NULL + cout << "Error while calculating" << endl; + exit(1); + } + + /*vector res; + for (vector::iterator it_res = cand_pos->begin(); + it_res != cand_pos->end(); + it_res++) { + res.push_back(result_tuple(*candidate, *it_res)); + }*/ + + unsigned int costs = 1 + 2*(num_hops - 1); + + // Costs too high? + if (g_use_checkpoints && costs > g_cp_thresh) { + // costs acceptable with usage of latest CP? + map::iterator occ_cp_find = m_occ_cp.find(*candidate); + if (occ_cp_find == m_occ_cp.end()) { + cout << "Error in data structure!"<< endl; + exit(1); + } + costs = 1 + 2*(occ_cp_find->second - 1) + g_cost_cp; + // Costs still too hight? Add new CP + if (costs > g_cp_thresh) { + m_resultCollector->addCheckpoint(trace_pos); + m_occ_cp.clear(); + costs = g_cost_cp; + } + } + + m_resultCollector->addResult(empty_result, costs); + } + + // MAX MEM USAGE + m_resultCollector->setMaxMemUsage(); + + return true; +} diff --git a/tools/compute-hops/SimpleAlgorithm.hpp b/tools/compute-hops/SimpleAlgorithm.hpp new file mode 100644 index 00000000..cfcb832d --- /dev/null +++ b/tools/compute-hops/SimpleAlgorithm.hpp @@ -0,0 +1,30 @@ +#ifndef SIMPLE_ALGORITHM__HPP +#define SIMPLE_ALGORITHM__HPP + +//#include +#include + +#include "BasicAlgorithm.hpp" +#include "../../src/core/util/smarthops/TraceReader.hpp" + +using namespace fail; + +class ResultCollector; + +class SimpleAlgorithm : public BasicAlgorithm { +public: + SimpleAlgorithm(ResultCollector *rc) : BasicAlgorithm(rc) {} + virtual ~SimpleAlgorithm() {} + + bool calculateAllHops(TraceReader& trace); + +private: + // Count occurences + //map > m_occurences; + std::map m_occurences; + + std::map m_occ_cp; +}; + +#endif // SIMPLE_ALGORITHM__HPP + diff --git a/tools/compute-hops/SmartAlgorithm.cc b/tools/compute-hops/SmartAlgorithm.cc new file mode 100644 index 00000000..16f4f2b2 --- /dev/null +++ b/tools/compute-hops/SmartAlgorithm.cc @@ -0,0 +1,218 @@ +#include "SmartAlgorithm.hpp" + +#include +#include + +#include "ResultCollector.hpp" + +extern bool g_use_weights; +extern bool g_use_watchpoints; +extern bool g_use_checkpoints; + +extern unsigned int g_cp_thresh; +extern unsigned int g_cost_cp; + +// Must be smaller than ((COST_CP_THRESH - COST_CP) / 2) +// If too high: Costs per dynamic instruction will approach +// COST_CP_THRESH but number of CPs can be reduced drastically +extern unsigned int g_rollback_thresh; + +extern fail::Logger LOG; + +bool SmartAlgorithm::calculateAllHops(TraceReader& trace) +{ + unsigned int trace_pos = 0; + unsigned int costs = 0; + + std::vector trace_events; + std::vector result; + + while (trace.getNextTraceEvents(trace_pos, trace_events)) { + // Policy when multiple trace events: Choose the one with smallest last_pos + trace_event_tuple_t *best_te = NULL; + trace_pos_t last_pos = std::numeric_limits::max(); + + bool hop_found = false; + bool checkpoint_forbidden = false; + + for (std::vector::iterator it_te = trace_events.begin(); + it_te != trace_events.end(); + it_te++) { + + // Don't use watchpoints? + if (!g_use_watchpoints && it_te->second != ACCESS_NONE) { + continue; + } + + // Find last pos of current trace event + std::map::iterator it_lp = m_last_positions.find(*it_te); + + // Not known? => Single hop, result calculation complete + if (it_lp == m_last_positions.end()) { + m_last_positions.insert(std::pair(*it_te, trace_pos)); + // New instruction => single hop + if (!hop_found) { + result.clear(); + result.push_back(result_tuple(*it_te, trace_pos)); + costs = COST_CHANGE; + m_resultCollector->addResult(result, costs); + hop_found = true; + } + } else { + if (it_lp->second < last_pos) { + last_pos = it_lp->second; + best_te = (trace_event_tuple_t*)&(it_lp->first); + } + it_lp->second = trace_pos; + } + } + + if (hop_found) { + continue; + } + + // Deltion of unnecessary hops + + /* + * |----------------| + * |pos bigger than | + * |last pos of new | --> delete -->-| + * |trace_event? | | + * |----------------| | + * ^ | + * | v + * A B A C <<< Last result + * ^ ^ + * Reverse Reverse + * iterator iterator + * rit_pre_last rit_last + * + */ + + std::vector::reverse_iterator rit_pre_last, rit_last, end; + + + rit_last = result.rbegin(); + rit_pre_last = result.rbegin(); + rit_pre_last++; + + while (rit_last != result.rend()) { + + trace_pos_t left_pos; + + // Policy switch: + // No weights => last pos <= pos of rit_pre_last + // Weights => last pos < pos of rit_pre_last + + if ((rit_pre_last == result.rend())) { + // First node in hop list is Checkpoint? => Look at result before CP-Creation + if (rit_last->first.second == ACCESS_CHECKPOINT) { + rit_pre_last = (m_checkpoints[rit_last->first.first]).second.rbegin(); + + // This should never happen: + if (rit_pre_last == (m_checkpoints[rit_last->first.first]).second.rend()) { + break; + } + + // Don't allow deletion of CP, if afterwards + // too few of the olf hops would be deleted. + // (past before to be deleted CP) + + // Should not exceed vector boundaries, if + // g_rollback_thresh*2 < (g_cp_thresh - g_cost_cp) + if ((*(rit_pre_last + g_rollback_thresh)).second < last_pos) { + checkpoint_forbidden = true; + break; + } + } else { + break; + } + } + + left_pos = rit_pre_last->second; + + if ((!g_use_weights && !(last_pos <= left_pos)) + || (g_use_weights && !(last_pos < left_pos))) { + break; + } + + // Hop a->b is not allowed if last pos of b is at a + // but a is not b + // e.g. instruction x with mem access at y + // a is WP y, and b is BP x + // OR a is BP x, and b is WP y but instruction + // at position a has also memory access at y + if (rit_pre_last != result.rend()) { + if ((last_pos == rit_pre_last->second) && + rit_pre_last->first != *best_te) { + break; + } + } + + // Right hop will be deleted + // => Recalculate costs + + // If to be deleted hop is a checkpoint: + // There will be no past in the trace + // Reconfigure result and iterators + if (rit_last->first.second == ACCESS_CHECKPOINT) { + costs = m_checkpoints[rit_last->first.first].first; + + result.clear(); + std::vector *tmp = &(m_checkpoints[rit_last->first.first].second); + result.insert(result.end(), tmp->begin(), tmp->end()); + + rit_last = result.rbegin(); + rit_pre_last = result.rbegin(); + rit_pre_last++; + + // CP could be removed from m_checkpoints, but to do this + // m_checkpoints needs to be a map (CP-ID is implicit as vector index) + + continue; + } + + if (rit_pre_last->first == rit_last->first) { + costs -= COST_NO_CHANGE; + } else { + costs -= COST_CHANGE; + } + + // Delete element described by rit_last + // Deletion of elements described by reverse iterator works like this + result.erase((rit_last+1).base()); + rit_last = rit_pre_last; + rit_pre_last++; + } + + + // Add costs for new hop + if (*best_te == (result.back()).first) { + costs += COST_NO_CHANGE; + } else { + costs += COST_CHANGE; + } + + // Check if Checkpoint needed + if (g_use_checkpoints && (costs > g_cp_thresh) && !checkpoint_forbidden) { + checkpoint_tuple_t new_cp(costs, std::vector(result)); + m_checkpoints.push_back(new_cp); + result.clear(); + result.push_back(result_tuple(trace_event_tuple_t(m_next_cp_id++, + ACCESS_CHECKPOINT), + trace_pos)); + costs = g_cost_cp; + + m_resultCollector->addCheckpoint(trace_pos); + } else { + // Add new hop + result.push_back(result_tuple(*best_te, trace_pos)); + } + + m_resultCollector->addResult(result, costs); + } + // MAX MEM USAGE + m_resultCollector->setMaxMemUsage(); + + return true; +} diff --git a/tools/compute-hops/SmartAlgorithm.hpp b/tools/compute-hops/SmartAlgorithm.hpp new file mode 100644 index 00000000..7d1321e5 --- /dev/null +++ b/tools/compute-hops/SmartAlgorithm.hpp @@ -0,0 +1,40 @@ +#ifndef SMART_ALGORITHM__HPP +#define SMART_ALGORITHM__HPP + +#include +#include + +#include "BasicAlgorithm.hpp" +#include "../../src/core/util/smarthops/TraceReader.hpp" +#include "ResultCollector.hpp" + +class ResultCollector; + +using namespace fail; + +// COSTS NAVIGATIONAL RECORD +typedef std::pair > checkpoint_tuple_t; + +class SmartAlgorithm : public BasicAlgorithm { +public: + + SmartAlgorithm(ResultCollector *rc) : BasicAlgorithm(rc), m_next_cp_id(0) {} + virtual ~SmartAlgorithm() {} + + bool calculateAllHops(TraceReader& trace); + +private: + + bool + occurenceInPosIntervall(unsigned int intervall_low, + unsigned int intervall_high, + address_t add, + mem_access_type_e acc); + + std::map m_last_positions; + std::vector m_checkpoints; + unsigned int m_next_cp_id; +}; + +#endif // SMART_ALGORITHM__HPP + diff --git a/tools/compute-hops/main.cc b/tools/compute-hops/main.cc new file mode 100644 index 00000000..e87d750f --- /dev/null +++ b/tools/compute-hops/main.cc @@ -0,0 +1,313 @@ +// #include +#include +#include +#include + +#include "../../src/core/util/smarthops/TraceReader.hpp" +#include "BasicAlgorithm.hpp" +#include "SmartAlgorithm.hpp" +#include "SimpleAlgorithm.hpp" +#include "ResultCollector.hpp" + +#include "../../src/core/util/gzstream/gzstream.h" + +#include "util/CommandLine.hpp" +#include "util/Logger.hpp" + +#define STDOUT_CMD_STRING "-" + +using namespace fail; + +typedef enum { + ALGO_SIMPLE, + ALGO_SMART, +} algorithm_e; + +bool g_use_weights; +bool g_use_watchpoints; +bool g_use_checkpoints; + +unsigned int g_cp_thresh; +unsigned int g_cost_cp; +unsigned int g_rollback_thresh; + +std::ofstream g_cp_ofstream; + +Logger LOG("hop-calculator", false); + +int main(int argc, char *argv[]) +{ + + // Manually fill the command line option parser + CommandLine &cmd = CommandLine::Inst(); + + cmd.addOption("", "", Arg::None, "USAGE: compute-hops [options]"); + CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h,--help \tPrint usage and exit"); + + CommandLine::option_handle ALGORITHM = + cmd.addOption("a", "algorithm", Arg::Required, + "-a,--algorithm \tHop algorithm (\"simple\"/\"smart\", default: \"none\")"); + + CommandLine::option_handle OUTPUT_MODE = + cmd.addOption("m", "output-mode", Arg::Required, + "-m,--output-mode \tOutput mode (\"results\"/\"costs\"/\"statistics\", default: \"results\")"); + + CommandLine::option_handle TRACE_FILE = + cmd.addOption("i", "input-file", Arg::Required, + "-i,--input-file \tInput trace file path"); + + CommandLine::option_handle OUTPUT_FILE = + cmd.addOption("o", "output-file", Arg::Required, + "-o,--output-file \tOutput file, default: stdout"); + + CommandLine::option_handle OUTPUT_FILE_PROTOBUF = + cmd.addOption("b", "protobuf-output", Arg::None, + "-b,--protobuf-output \tOutput file will be created in protobuf (HopChain) format"); + + CommandLine::option_handle USE_COSTS = + cmd.addOption("c", "use-costs", Arg::None, + "-c,--use-costs \tUse hop costs for calculations of Smart-Hopping algorithm"); + + CommandLine::option_handle USE_WATCHPOINTS = + cmd.addOption("w", "use-watchpoints", Arg::None, + "-w,--use-watchpoints \tUse watchpoints as additional hop candidates"); + + CommandLine::option_handle USE_CHECKPOINTS = + cmd.addOption("", "use-checkpoints", Arg::None, + "--use-checkpoints \tUse checkpoints to cap costs in case of long hop chains"); + + CommandLine::option_handle CP_THRESH = + cmd.addOption("", "cp-costs-threshold", Arg::Required, + "--cp-costs-threshold \tIf costs of a hop-chain reach this threshold, a checkpoint will be created"); + + CommandLine::option_handle CP_COSTS = + cmd.addOption("", "cp-costs", Arg::Required, + "--cp-costs \tThe costs for reloading a checkpoint into the system"); + + CommandLine::option_handle CP_ROLLBACK_THRESH = + cmd.addOption("", "cp-rollback-threshold", Arg::Required, + "--cp-rollback-threshold \tMinial number of hops to roll back current solution beyond checkpoint. Must be smaller than ((cp_cost_thresh - cp_costs) / 2)"); + + CommandLine::option_handle CHECKPOINT_OUTPUT_FILE = + cmd.addOption("", "cp-output", Arg::Required, + "--cp-output \tCheckpoint output file"); + + for (int i = 1; i < argc; ++i) { + cmd.add_args(argv[i]); + } + if (!cmd.parse()) { + std::cerr << "Error parsing arguments." << std::endl; + return 1; + } + + if (cmd[HELP]) { + cmd.printUsage(); + if (cmd[HELP]) { + exit(0); + } else { + exit(1); + } + } + + if (cmd[USE_CHECKPOINTS]) { + g_use_checkpoints = true; + } else { + g_use_checkpoints = false; + } + + if (cmd[USE_WATCHPOINTS]) { + g_use_watchpoints = true; + } else { + g_use_watchpoints = false; + } + + if (cmd[USE_COSTS]) { + g_use_weights = true; + } else { + g_use_weights = false; + } + + algorithm_e algorithm; + + if (cmd[ALGORITHM].count() > 0) { + std::string alg(cmd[ALGORITHM].first()->arg); + if (alg == "simple") { + algorithm = ALGO_SIMPLE; + } else if (alg == "smart") { + algorithm = ALGO_SMART; + } else { + exit(-1); + } + } else { + // default + algorithm = ALGO_SMART; + } + + output_mode_e output_mode; + + if (cmd[OUTPUT_MODE].count() > 0) { + std::string outp(cmd[OUTPUT_MODE].first()->arg); + if (outp == "results") { + output_mode = OUTPUT_RESULT; + } else if (outp == "costs") { + output_mode = OUTPUT_COSTS; + } else if (outp == "statistics") { + output_mode = OUTPUT_STATISTICS; + } else { + LOG << "Unkown output mode: " << outp << std::endl; + exit(-1); + } + } else { + // default + output_mode = OUTPUT_RESULT; + } + + fail::TraceReader trace; + + if (cmd[TRACE_FILE].count() > 0) { + const char *filename = cmd[TRACE_FILE].first()->arg; + if (!trace.openTraceFile(filename, 0)) { + LOG << "Unable to open and parse input trace file " << filename << std::endl; + return 1; + } + } else { + LOG << "Input trace file path not defined" << std::endl; + exit(-1); + } + + std::ostream *outFile; + + if (cmd[OUTPUT_FILE].count() > 0) { + std::string filename(cmd[OUTPUT_FILE].first()->arg); + + if (filename.compare(STDOUT_CMD_STRING) == 0) { + outFile = &std::cout; + } else { + std::ofstream *of = new std::ofstream(filename.c_str()); + if (!of->is_open()) { + LOG << "Unable to open output file " << filename << std::endl; + exit(-1); + } + outFile = of; + } + } else { + outFile = &std::cout; + } + + if (cmd[OUTPUT_FILE_PROTOBUF] && outFile == &std::cout) { + LOG << "If protobuf output format is selected, output file must be defined" << std::endl; + exit(-1); + } + + if (cmd[USE_COSTS] && !(algorithm == ALGO_SMART)) { + LOG << "Using costs for calculations is only possible with Smart-Hopping algorithm" << std::endl; + exit(-1); + } + + if (cmd[CP_THRESH]) { + if (cmd[CP_THRESH].count() != 1) { + LOG << "Could not parse cp-costs-threshold" << std::endl; + exit(-1); + } + g_cp_thresh = strtoul(cmd[CP_THRESH].first()->arg, NULL, 0); + } + + if (cmd[CP_COSTS]) { + if (cmd[CP_COSTS].count() != 1) { + LOG << "Could not parse cp-costs" << std::endl; + exit(-1); + } + g_cost_cp = strtoul(cmd[CP_COSTS].first()->arg, NULL, 0); + } + + if (cmd[CP_ROLLBACK_THRESH]) { + if (cmd[CP_ROLLBACK_THRESH].count()!=1) { + LOG << "Could not parse cp-rollback-threshold" << std::endl; + exit(-1); + } +// if (strcmp(argv[9],"max")==0) { +// g_rollback_thresh = (g_cp_thresh - g_cost_cp - 5)/2; +// } + g_cost_cp = strtoul(cmd[CP_ROLLBACK_THRESH].first()->arg, NULL, 0); + } + + if (cmd[USE_CHECKPOINTS] && !(cmd[CHECKPOINT_OUTPUT_FILE].count() > 0)) { + LOG << "If checkpointing is enabled, --cp-output must be defined" << std::endl; + exit(-1); + } + + if (cmd[CHECKPOINT_OUTPUT_FILE].count() > 0) { + std::string filename(cmd[CHECKPOINT_OUTPUT_FILE].first()->arg); + g_cp_ofstream.open(filename.c_str()); + if(!g_cp_ofstream.is_open()) { + LOG << "Unable to open cp_out_file " << filename << std::endl; + exit(-1); + } + } + + if (cmd[USE_CHECKPOINTS] && (!cmd[CP_THRESH] || + !cmd[CP_COSTS]|| !cmd[CP_ROLLBACK_THRESH])) { + LOG << "If using checkpointing is enabled, also cp-costs-threshold, " + "cp-costs and cp-rollback-threshold must be defined" << std::endl; + exit(-1); + } + + if (!cmd[USE_CHECKPOINTS] && (cmd[CP_THRESH].count() || + (cmd[CP_COSTS].count() > 0) || (cmd[CP_ROLLBACK_THRESH].count() > 0))) { + LOG << "If using checkpointing is disabled, cp-costs-threshold, " + "cp-costs and cp-rollback-threshold must not be defined" << std::endl; + exit(-1); + } + + if (g_use_checkpoints && !(g_cp_thresh > g_cost_cp)) { + LOG << "cp_cost_thresh needs to be bigger than cp_costs" << std::endl; + exit(1); + } + + if (g_use_checkpoints && !(g_rollback_thresh < ((g_cp_thresh - g_cost_cp) / 2))) { + LOG << "cp_rollback_hop_thresh needs to be smaller than " + "((cp_cost_thresh - cp_costs) / 2)" << std::endl; + exit(1); + } + + + ogzstream zipstream; + + ResultCollector rc(*outFile, output_mode); + + if (cmd[OUTPUT_FILE_PROTOBUF]) { + zipstream.open(cmd[OUTPUT_FILE].first()->arg); + rc.setProtoOStream(new ProtoOStream(&zipstream)); + } + + BasicAlgorithm *algo; + + switch (algorithm) { + case ALGO_SMART: + algo = new SmartAlgorithm(&rc); + break; + case ALGO_SIMPLE: + algo = new SimpleAlgorithm(&rc); + break; + default: + break; + } + + rc.startTimer(); + algo->calculateAllHops(trace); + rc.stopTimer(); + + rc.finish(); + + // ToDo: close output file if not stdout + if (outFile != &std::cout) { + ((std::ofstream*)outFile)->close(); + delete (std::ofstream*)outFile; + } + + if (g_use_checkpoints) { + g_cp_ofstream.close(); + } + + return 0; +} diff --git a/tools/dump-hops/CMakeLists.txt b/tools/dump-hops/CMakeLists.txt new file mode 100644 index 00000000..930dbb01 --- /dev/null +++ b/tools/dump-hops/CMakeLists.txt @@ -0,0 +1,7 @@ +set(SRCS + DumpHops.cc +) + +add_executable(dump-hops ${SRCS}) +target_link_libraries(dump-hops ${PROTOBUF_LIBRARY} fail-util fail-comm) +install(TARGETS dump-hops RUNTIME DESTINATION bin) diff --git a/tools/dump-hops/DumpHops.cc b/tools/dump-hops/DumpHops.cc new file mode 100644 index 00000000..dd3c88c0 --- /dev/null +++ b/tools/dump-hops/DumpHops.cc @@ -0,0 +1,110 @@ +#include +#include +#include +#include "comm/InjectionPointHopsMessage.pb.h" +#include "util/ProtoStream.hpp" +#include "../../src/core/util/Logger.hpp" +#include "../../src/core/util/gzstream/gzstream.h" +#include "util/CommandLine.hpp" + +using namespace fail; +using std::stringstream; +using std::endl; +using std::cout; +using std::cerr; +using std::hex; +using std::dec; + +Logger LOG("dump-hops", true); + +std::istream& openStream(const char *input_file, + std::ifstream& normal_stream, igzstream& gz_stream) { + normal_stream.open(input_file); + if (!normal_stream) { + LOG << "couldn't open " << input_file << endl; + exit(-1); + } + unsigned char b1, b2; + normal_stream >> b1 >> b2; + + if (b1 == 0x1f && b2 == 0x8b) { + normal_stream.close(); + gz_stream.open(input_file); + if (!gz_stream) { + LOG << "couldn't open " << input_file << endl; + exit(-1); + } + //LOG << "opened file " << input_file << " in GZip mode" << endl; + return gz_stream; + } + + normal_stream.seekg(0); + + //LOG << "opened file " << input_file << " in normal mode" << endl; + return normal_stream; +} + +int main(int argc, char *argv[]) +{ + InjectionPointMessage ev; + + CommandLine &cmd = CommandLine::Inst(); + cmd.addOption("", "", Arg::None, "usage: dump-hops [options] hopfile.hp"); + CommandLine::option_handle HELP = + cmd.addOption("h", "help", Arg::None, "-h/--help \tPrint usage and exit"); +// CommandLine::option_handle STATS = +// cmd.addOption("s", "stats", Arg::None, +// "-s/--stats \tShow trace stats"); + + for (int i = 1; i < argc; ++i) { + cmd.add_args(argv[i]); + } + if (!cmd.parse()) { + std::cerr << "Error parsing arguments." << std::endl; + return 1; + } + + if (cmd[HELP] || cmd.parser()->nonOptionsCount() != 1) { + cmd.printUsage(); + if (cmd[HELP]) { + exit(0); + } else { + exit(1); + } + } + +// bool stats_only = cmd[STATS]; +// bool extended = cmd[EXTENDED_TRACE]; + + std::ifstream normal_stream; + igzstream gz_stream; + ProtoIStream ps(&openStream(cmd.parser()->nonOption(0), normal_stream, gz_stream)); + +// uint64_t stats_instr = 0, stats_reads = 0, stats_writes = 0, starttime = 0; + + while (ps.getNext(&ev)) { + cout << "T" << ev.target_trace_position() ; + + if (ev.has_checkpoint_id()) { + cout << " CP" << ev.checkpoint_id(); + } + + for (int i = 0; i < ev.hops_size(); i++) { + const InjectionPointMessage_Hops &hops = ev.hops(i); + cout << " " << + ((hops.accesstype()==InjectionPointMessage_Hops_AccessType_EXECUTE)?"X": + ((hops.accesstype()==InjectionPointMessage_Hops_AccessType_READ)?"R":"W")) + << hex << hops.address() << dec; + } + cout << "\n"; + } + +// if (stats_only) { +// cout << "#instructions: " << stats_instr << "\n" +// << "#memR: " << stats_reads << "\n" +// << "#memW: " << stats_writes << "\n" +// << "duration: " << (acctime - starttime + 1) << endl; +// } + + return 0; +}