introduce convert-trace tool

The input (taken on stdin) is a gem5.opt --debug-flags=MemoryAccess
trace (--format gem5), or a dump-trace output (--format dump) for easy
trace synthesis (for testing purposes).

gem5 format: Currently imports physical, not virtual memory addresses.
dump format: Currently ignores extended trace information.

Change-Id: Ic26a996d6fb9ce4175c855fadcbcff9ac9263888

tools/CMakeLists.txt

@@ -1,6 +1,7 @@
 option(BUILD_DUMP_TRACE   "Build the trace dump tool?" OFF)
 option(BUILD_IMPORT_TRACE "Build the trace import tool?" OFF)
 option(BUILD_PRUNE_TRACE  "Build the trace prune tool?" OFF)
+option(BUILD_CONVERT_TRACE "Build the trace converter tool?" OFF)
 
 
 ### Setup search paths for headers ##
@@ -18,3 +19,7 @@ endif(BUILD_PRUNE_TRACE)
 if(BUILD_DUMP_TRACE)
 	add_subdirectory(dump-trace)
 endif(BUILD_DUMP_TRACE)
+
+if(BUILD_CONVERT_TRACE)
+	add_subdirectory(convert-trace)
+endif(BUILD_CONVERT_TRACE)

tools/convert-trace/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(SRCS
+  main.cc
+  Gem5Converter.cc
+  DumpConverter.cc
+)
+
+add_executable(convert-trace ${SRCS})
+target_link_libraries(convert-trace fail-util fail-comm)
+
+install(TARGETS convert-trace RUNTIME DESTINATION bin)

tools/convert-trace/DumpConverter.cc

@@ -0,0 +1,73 @@
+#include <sstream>
+#include <string>
+#include "util/Logger.hpp"
+#include "comm/TracePlugin.pb.h"
+#include "DumpConverter.hpp"
+
+using namespace fail;
+using std::stringstream;
+using std::endl;
+using std::hex;
+using std::dec;
+
+static Logger LOG("DumpConverter", true);
+
+// TODO: convert extended trace information, too
+bool DumpConverter::convert()
+{
+	char buf[2048], dummy[1024];
+	int64_t prev_cycle = 0, cycle;
+
+	while (m_input.getline(buf, sizeof(buf))) {
+		Trace_Event ev;
+		// IP 1035c9 t=0
+		// MEM R 123abc width 4 IP 1035c9 t=1
+		// MEM W 123abc width 4 IP 1035c9 t=1
+
+		std::stringstream ss(buf);
+		std::string ev_type;
+		ss >> ev_type;
+
+		if (ev_type == "MEM") {
+			std::string accesstype;
+			uint64_t data_address;
+			uint32_t data_width;
+			ss >> accesstype >> hex >> data_address >> dummy >> dec >> data_width >> ev_type;
+			if (!ss) {
+				LOG << "input mismatch (1), input = " << buf << endl;
+				continue;
+			}
+			ev.set_memaddr(data_address);
+			ev.set_width(data_width);
+			ev.set_accesstype(accesstype == "R" ? ev.READ : ev.WRITE);
+		}
+		assert(ev_type == "IP");
+
+		uint64_t instr_address;
+		ss >> hex >> instr_address >> dummy;
+		if (!ss || dummy[0] != 't' || dummy[1] != '=') {
+			LOG << "input mismatch (2), input = " << buf << endl;
+			continue;
+		}
+		ev.set_ip(instr_address);
+		ss.clear();
+		ss.str(dummy + 2);
+		ss >> dec >> cycle;
+		if (!ss) {
+			LOG << "input mismatch (3), input = " << buf << endl;
+			continue;
+		}
+
+		if (cycle - prev_cycle > 0) {
+			ev.set_time_delta(cycle - prev_cycle);
+			prev_cycle = cycle;
+		} else if (cycle - prev_cycle < 0) {
+			LOG << "ignoring time discontinuity " << dec << prev_cycle << " -> " << cycle << endl;
+			prev_cycle = cycle;
+		}
+
+		m_ps.writeMessage(&ev);
+	}
+
+	return true;
+}

tools/convert-trace/DumpConverter.hpp

@@ -0,0 +1,11 @@
+#ifndef __DUMPCONVERTER_HPP__
+#define __DUMPCONVERTER_HPP__
+
+#include "FormatConverter.hpp"
+
+class DumpConverter : public FormatConverter {
+public:
+	DumpConverter(std::istream& input, fail::ProtoOStream& ps) : FormatConverter(input, ps) {}
+	bool convert();
+};
+#endif

tools/convert-trace/FormatConverter.hpp

@@ -0,0 +1,16 @@
+#ifndef __FORMATCONVERTER_HPP__
+#define __FORMATCONVERTER_HPP__
+
+#include <istream>
+#include "util/ProtoStream.hpp"
+
+class FormatConverter {
+public:
+	FormatConverter(std::istream& input, fail::ProtoOStream& ps) : m_input(input), m_ps(ps) {}
+	virtual bool convert() = 0;
+
+protected:
+	std::istream& m_input;
+	fail::ProtoOStream& m_ps;
+};
+#endif

tools/convert-trace/Gem5Converter.cc

@@ -0,0 +1,93 @@
+#include <sstream>
+#include <string>
+#include "util/Logger.hpp"
+#include "comm/TracePlugin.pb.h"
+#include "Gem5Converter.hpp"
+
+using namespace fail;
+using std::stringstream;
+using std::endl;
+using std::hex;
+using std::dec;
+
+static Logger LOG("Gem5Converter", true);
+
+bool Gem5Converter::convert()
+{
+	char buf[2048], dummy[2048];
+	int64_t prev_cycle = 0, cycle;
+	uint64_t cur_ip = 0;
+	bool ifetch_seen = false;
+
+	while (m_input.getline(buf, sizeof(buf))) {
+		Trace_Event ev;
+		// reset ifetch_seen in case we're getting multiple concatenated gem5 traces
+		if (strstr(buf, "http://gem5.org")) {
+			ifetch_seen = false;
+			continue;
+		}
+		if (!strstr(buf, "system.physmem") || strstr(buf, "access wrote")) {
+			continue;
+		}
+		//    5000: system.physmem: access wrote 4 bytes to address f7ee8
+		//    5000: system.physmem: Write of size 4 on address 0xf7ee8 data 0x61190
+		//    6000: system.physmem: IFetch of size 4 on address 0xd58 data 0xe59f000c
+		//    6000: system.physmem: Read of size 4 on address 0xd6c data 0x8e2c
+		//  161000: system.physmem: 00000000  4c 69 6e 75 78 00 00 00  00 00 00 00 00 00 00 00   Linux
+
+		std::stringstream ss(buf);
+		std::string access_type;
+		unsigned access_width;
+		uint64_t access_address;
+		ss >> dec >> cycle >> dummy >> dummy >> access_type
+		   >> dummy >> dummy >> dec >> access_width
+		   >> dummy >> dummy >> hex >> access_address;
+		if (!ss) {
+			if (ifetch_seen && !access_type.c_str()[0] == '0') {
+				LOG << "input mismatch, input = " << buf << endl;
+			}
+			continue;
+		}
+
+		if (cycle - prev_cycle > 0) {
+			ev.set_time_delta(cycle - prev_cycle);
+			prev_cycle = cycle;
+		} else if (cycle - prev_cycle < 0) {
+			LOG << "ignoring time discontinuity " << dec << prev_cycle << " -> " << cycle << endl;
+			prev_cycle = cycle;
+		}
+
+		if (access_type == "IFetch") {
+			ifetch_seen = true;
+			cur_ip = access_address;
+			ev.set_ip(cur_ip);
+			//LOG << "ip = " << hex << cur_ip << endl;
+
+		} else {
+			bool is_read;
+			if (access_type == "Read") {
+				is_read = true;
+			} else if (access_type == "Write") {
+				is_read = false;
+			} else if (access_type.c_str()[0] == '0') {
+				continue;
+			} else {
+				LOG << "input mismatch, input = " << buf << endl;
+				continue;
+			}
+
+			if (!ifetch_seen) {
+				// skip all accesses before the first ifetch
+				continue;
+			}
+			ev.set_ip(cur_ip);
+			ev.set_memaddr(access_address);
+			ev.set_width(access_width);
+			ev.set_accesstype(is_read ? ev.READ : ev.WRITE);
+			//cout << "ip = " << hex << cur_ip << " mem " << access_address << " " << access_width << " " << (is_read ? 'R' : 'W') << endl;
+		}
+		m_ps.writeMessage(&ev);
+	}
+
+	return true;
+}

tools/convert-trace/Gem5Converter.hpp

@@ -0,0 +1,11 @@
+#ifndef __GEM5CONVERTER_HPP__
+#define __GEM5CONVERTER_HPP__
+
+#include "FormatConverter.hpp"
+
+class Gem5Converter : public FormatConverter {
+public:
+	Gem5Converter(std::istream& input, fail::ProtoOStream& ps) : FormatConverter(input, ps) {}
+	bool convert();
+};
+#endif

tools/convert-trace/main.cc

@@ -0,0 +1,65 @@
+#include <fstream>
+#include <string>
+
+#include "FormatConverter.hpp"
+#include "Gem5Converter.hpp"
+#include "DumpConverter.hpp"
+
+#include "util/CommandLine.hpp"
+#include "util/gzstream/gzstream.h"
+#include "util/Logger.hpp"
+
+using namespace fail;
+using std::cin;
+using std::endl;
+
+static Logger LOG("convert-trace", true);
+
+int main(int argc, char *argv[]) {
+	CommandLine &cmd = CommandLine::Inst();
+	cmd.addOption("", "", Arg::None, "usage: convert-trace -f dump|gem5 -t tracefile.tc");
+	CommandLine::option_handle HELP =
+		cmd.addOption("h", "help", Arg::None, "-h/--help \tPrint usage and exit");
+	CommandLine::option_handle FORMAT =
+		cmd.addOption("f", "format", Arg::Required, "-f/--format FORMAT \tInput format (dump|gem5)");
+	CommandLine::option_handle OUTFILE =
+		cmd.addOption("t", "trace", Arg::Required, "-t/--trace FILE \tOutput file");
+	for (int i = 1; i < argc; ++i) {
+		cmd.add_args(argv[i]);
+	}
+	if (!cmd.parse()) {
+		LOG << "Error parsing arguments." << endl;
+		return 1;
+	}
+
+	if (cmd[HELP] || !cmd[OUTFILE] || !cmd[FORMAT] || cmd.parser()->nonOptionsCount() != 0) {
+		cmd.printUsage();
+		if (cmd[HELP]) {
+			exit(0);
+		} else {
+			exit(1);
+		}
+	}
+
+	std::string format = cmd[FORMAT].first()->arg;
+	std::string trace_file = cmd[OUTFILE].first()->arg;
+
+	ogzstream gz_stream(trace_file.c_str());
+	std::ostream *os = &gz_stream;
+	ProtoOStream ps(os);
+
+	FormatConverter *converter;
+	if (format == "gem5") {
+		converter = new Gem5Converter(cin, ps);
+	} else if (format == "dump") {
+		converter = new DumpConverter(cin, ps);
+	} else {
+		LOG << "unknown input format '" << format << "'" << endl;
+		return 1;
+	}
+
+	if (!converter->convert()) {
+		LOG << "converter failed" << endl;
+		return 1;
+	}
+}

tools/dump-trace/DumpTrace.cc

@@ -122,7 +122,7 @@ int main(int argc, char *argv[])
 			if (!stats_only) {
 				cout << "MEM "
 				     << (ev.accesstype() == Trace_Event_AccessType_READ ? "R" : "W") << " "
-				     << ev.memaddr()
+				     << hex << ev.memaddr()
 				     << dec << " width " << ev.width()
 				     << hex << " IP " << ev.ip()
 				     << dec << " t=" << acctime