import-trace: import extended traces

This tool can now import extended trace information with the
--extended-trace command-line parameter.  The existing importers cease
using artificial access_info_t objects in favor of passing through the
original Trace_Event wherever possible.  This allows us to import
extended trace information for all importers.

Change-Id: I3613e9d05d5e69ad49e96f4dc5ba0b1c4ef95a11

src/core/sal/Architecture.hpp

@@ -0,0 +1,15 @@
+// Architecture.hpp: wraps architecture definition headers
+#ifndef __ARCHITECTURE_HPP__
+  #define __ARCHITECTURE_HPP__
+
+#include "config/FailConfig.hpp"
+
+#ifdef BUILD_X86
+#include "x86/X86Architecture.hpp"
+#endif
+
+#ifdef BUILD_ARM
+#include "arm/ARMArchitecture.hpp"
+#endif
+
+#endif

src/core/sal/arm/ArmArchitecture.hpp

@@ -17,6 +17,10 @@ public:
 	~ArmArchitecture();
 };
 
+#ifdef BUILD_ARM
+typedef ArmArchitecture Architecture;
+#endif
+
 /**
  * \enum GPRegIndex
  * Defines the general purpose (GP) register identifier for the ARM

src/core/sal/x86/X86Architecture.hpp

@@ -18,6 +18,10 @@ public:
 	~X86Architecture();
 };
 
+#ifdef BUILD_X86
+typedef X86Architecture Architecture;
+#endif
+
 /**
  * \enum GPRegisterId
  * Symbolic identifier to access the x86 general purpose register

tools/import-trace/Importer.cc

@@ -13,13 +13,16 @@ bool Importer::init(const std::string &variant, const std::string &benchmark, Da
 	if (!m_variant_id) {
 		return false;
 	}
+	m_extended_trace_regs =
+		m_arch.getRegisterSetOfType(RT_TRACE);
 	LOG << "Importing to variant " << variant << "/" << benchmark
 		<< " (ID: " << m_variant_id << ")" << std::endl;
 	return true;
 }
 
 bool Importer::create_database() {
-	std::string create_statement = "CREATE TABLE IF NOT EXISTS trace ("
+	std::stringstream create_statement;
+	create_statement << "CREATE TABLE IF NOT EXISTS trace ("
 		"	variant_id int(11) NOT NULL,"
 		"	instr1 int(10) unsigned NOT NULL,"
 		"	instr1_absolute int(10) unsigned DEFAULT NULL,"
@@ -29,10 +32,19 @@ bool Importer::create_database() {
 		"	time2 bigint(10) unsigned NOT NULL,"
 		"	data_address int(10) unsigned NOT NULL,"
 		"	width tinyint(3) unsigned NOT NULL,"
-		"	accesstype enum('R','W') NOT NULL,"
+		"	accesstype enum('R','W') NOT NULL,";
+	if (m_extended_trace) {
+		create_statement << "   data_value int(10) unsigned NULL,";
+		for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin();
+			it != m_extended_trace_regs->end(); ++it) {
+			create_statement << " r" << (*it)->getId() << " int(10) unsigned NULL,";
+			create_statement << " r" << (*it)->getId() << "_deref int(10) unsigned NULL,";
+		}
+	}
+	create_statement <<
 		"	PRIMARY KEY (variant_id,data_address,instr2)"
 		") engine=MyISAM ";
-	return db->query(create_statement.c_str());
+	return db->query(create_statement.str().c_str());
 }
 
 
@@ -82,12 +94,14 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) {
 			/* Another instruction was executed, handle it in the
 			   subclass */
 			if (!handle_ip_event(curtime, instr, ev)) {
+				LOG << "error: handle_ip_event() failed at instr=" << instr << std::endl;
 				return false;
 			}
 
 			instr++;
 		} else {
 			if (!handle_mem_event(curtime, instr, ev)) {
+				LOG << "error: handle_mem_event() failed at instr=" << instr << std::endl;
 				return false;
 			}
 		}
@@ -219,60 +233,144 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) {
 
 bool Importer::add_trace_event(margin_info_t &begin, margin_info_t &end,
 							   access_info_t &access, bool is_fake) {
-	static MYSQL_STMT *stmt = 0;
+	Trace_Event e;
-	if (!stmt) {
+	e.set_ip(end.ip);
-		std::string sql("INSERT INTO trace (variant_id, instr1, instr1_absolute, instr2, instr2_absolute, time1, time2, data_address, width,"
+	e.set_memaddr(access.data_address);
-		                "					accesstype)"
+	e.set_width(access.data_width);
-		                "VALUES (?,?,?,?,?,?,?,?,?,?)");
+	e.set_accesstype(access.access_type == 'R' ? e.READ : e.WRITE);
-		stmt = mysql_stmt_init(db->getHandle());
+	return add_trace_event(begin, end, e, is_fake);
-		if (mysql_stmt_prepare(stmt, sql.c_str(), sql.length())) {
+}
-			LOG << "query '" << sql << "' failed: " << mysql_error(db->getHandle()) << std::endl;
+
+bool Importer::add_trace_event(margin_info_t &begin, margin_info_t &end,
+							   Trace_Event &event, bool is_fake) {
+	// insert extended trace info if configured and available
+	bool extended = m_extended_trace && event.has_trace_ext();
+
+	MYSQL_STMT **stmt;
+	unsigned *columns;
+	static MYSQL_STMT *stmt_basic = 0;
+	static unsigned columns_basic = 0;
+	static MYSQL_STMT *stmt_extended = 0;
+	static unsigned columns_extended = 0;
+	stmt = extended ? &stmt_extended : &stmt_basic;
+	columns = extended ? &columns_extended : &columns_basic;
+
+	if (!*stmt) {
+		std::stringstream sql;
+		sql << "INSERT INTO trace (variant_id, instr1, instr1_absolute, instr2, instr2_absolute, time1, time2, "
+		       "data_address, width, accesstype";
+		*columns = 10;
+		if (extended) {
+			sql << ", data_value";
+			(*columns)++;
+			for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin();
+				it != m_extended_trace_regs->end(); ++it) {
+				sql << ", r" << (*it)->getId() << ", r" << (*it)->getId() << "_deref";
+			}
+		}
+
+		sql << ") VALUES (?";
+		for (unsigned i = 1; i < *columns + (extended ? m_extended_trace_regs->count() * 2 : 0); ++i) {
+			sql << ",?";
+		}
+		sql << ")";
+
+		*stmt = mysql_stmt_init(db->getHandle());
+		if (mysql_stmt_prepare(*stmt, sql.str().c_str(), sql.str().length())) {
+			LOG << "query '" << sql.str() << "' failed: " << mysql_error(db->getHandle()) << std::endl;
 			return false;
 		}
 	}
 
-	MYSQL_BIND bind[10];
+	// extended trace:
-	my_bool is_null = is_fake;
+	// retrieve register / register-dereferenced values
-	my_bool null = true;
+	std::map<int, uint32_t> register_values;
+	std::map<int, uint32_t> register_values_deref;
+	unsigned data_value = 0;
+	const Trace_Event_Extended& ev_ext = event.trace_ext();
+	if (extended) {
+		data_value = ev_ext.data();
+		for (int i = 0; i < ev_ext.registers_size(); i++) {
+			const Trace_Event_Extended_Registers& reg = ev_ext.registers(i);
+			register_values[reg.id()] = reg.value();
+			register_values_deref[reg.id()] = reg.value_deref();
+		}
+	}
+
+	// C99 / g++ extension VLA to the rescue:
+	MYSQL_BIND bind[*columns + m_extended_trace_regs->count() * 2];
+	my_bool fake_null = is_fake;
+	my_bool null = true, not_null = false;
 	long unsigned accesstype_len = 1;
 
+	unsigned data_address = event.memaddr();
+	unsigned width = event.width();
+	char accesstype = event.accesstype() == event.READ ? 'R' : 'W';
+
 	// LOG << m_variant_id  << "-" << ":" << begin.dyninstr << ":" << end.dyninstr << "-" << data_address << " " << accesstype << std::endl;
 
-
+	// sizeof works fine for VLAs
 	memset(bind, 0, sizeof(bind));
-	for (unsigned i = 0; i < sizeof(bind)/sizeof(*bind); ++i) {
+	for (unsigned i = 0; i < *columns; ++i) {
 		bind[i].buffer_type = MYSQL_TYPE_LONG;
 		bind[i].is_unsigned = 1;
 		switch (i) {
 		case 0: bind[i].buffer = &m_variant_id; break;
 		case 1: bind[i].buffer = &begin.dyninstr; break;
 		case 2: bind[i].buffer = &begin.ip;
-			bind[i].is_null = begin.ip == 0 ? &null : &is_null;
+			bind[i].is_null = begin.ip == 0 ? &null : &fake_null;
 			break;
 		case 3: bind[i].buffer = &end.dyninstr; break;
 		case 4: bind[i].buffer = &end.ip;
-			bind[i].is_null = &is_null; break;
+			bind[i].is_null = &fake_null; break;
 		case 5: bind[i].buffer = &begin.time;
 			bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
 			break;
 		case 6: bind[i].buffer = &end.time;
 			bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
 			break;
-		case 7: bind[i].buffer = &access.data_address; break;
+		case 7: bind[i].buffer = &data_address; break;
-		case 8: bind[i].buffer = &access.data_width; break;
+		case 8: bind[i].buffer = &width; break;
-		case 9: bind[i].buffer = &access.access_type;
+		case 9: bind[i].buffer = &accesstype;
 			bind[i].buffer_type = MYSQL_TYPE_STRING;
 			bind[i].buffer_length = accesstype_len;
 			bind[i].length = &accesstype_len;
 			break;
+		// only visited in extended mode:
+		case 10: bind[i].buffer = &data_value;
+			bind[i].is_null = ev_ext.has_data() ? &not_null : &null; break;
 		}
 	}
-	if (mysql_stmt_bind_param(stmt, bind)) {
+	if (extended) {
-		LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(stmt) << std::endl;
+		unsigned i = 0;
+		for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin();
+			it != m_extended_trace_regs->end(); ++it, ++i) {
+			assert(*columns + i*2 + 1 < sizeof(bind)/sizeof(*bind));
+			bind[*columns + i*2    ].buffer_type = MYSQL_TYPE_LONG;
+			bind[*columns + i*2    ].is_unsigned = 1;
+			if (register_values.count((*it)->getId())) {
+				bind[*columns + i*2    ].buffer = &register_values[(*it)->getId()];
+			} else {
+				bind[*columns + i*2    ].buffer = &width; // arbitrary
+				bind[*columns + i*2    ].is_null = &null;
+			}
+
+			bind[*columns + i*2 + 1].buffer_type = MYSQL_TYPE_LONG;
+			bind[*columns + i*2 + 1].is_unsigned = 1;
+			if (register_values_deref.count((*it)->getId())) {
+				bind[*columns + i*2 + 1].buffer = &register_values_deref[(*it)->getId()];
+			} else {
+				bind[*columns + i*2 + 1].buffer = &width; // arbitrary
+				bind[*columns + i*2 + 1].is_null = &null;
+			}
+		}
+	}
+	if (mysql_stmt_bind_param(*stmt, bind)) {
+		LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(*stmt) << std::endl;
 		return false;
 	}
-	if (mysql_stmt_execute(stmt)) {
+	if (mysql_stmt_execute(*stmt)) {
-		LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(stmt) << std::endl;
+		LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(*stmt) << std::endl;
 		LOG << "IP: " << std::hex << end.ip << std::endl;
 		return false;
 	}

tools/import-trace/Importer.hpp

@@ -6,6 +6,7 @@
 #include "util/ProtoStream.hpp"
 #include "util/ElfReader.hpp"
 #include "sal/SALConfig.hpp"
+#include "sal/Architecture.hpp"
 #include "util/Database.hpp"
 #include "util/MemoryMap.hpp"
 #include "comm/TracePlugin.pb.h"
@@ -22,7 +23,10 @@ protected:
 	fail::MemoryMap *m_mm;
 	char m_faultspace_rightmargin;
 	bool m_sanitychecks;
+	bool m_extended_trace;
 	fail::Database *db;
+	fail::Architecture m_arch;
+	fail::UniformRegisterSet *m_extended_trace_regs;
 
 	/* How many rows were inserted into the database */
 	unsigned m_row_count;
@@ -61,7 +65,7 @@ protected:
 	fail::simtime_t m_last_time;
 
 public:
-	Importer() : m_sanitychecks(false), m_row_count(0), m_time_trace_start(0) {}
+	Importer() : m_sanitychecks(false), m_extended_trace(false), m_row_count(0), m_time_trace_start(0) {}
 	bool init(const std::string &variant, const std::string &benchmark, fail::Database *db);
 
 	/**
@@ -73,15 +77,25 @@ public:
 	virtual bool create_database();
 	virtual bool copy_to_database(fail::ProtoIStream &ps);
 	virtual bool clear_database();
+	/**
+	 * Use this variant if passing through the IP/MEM event does not make any
+	 * sense for your Importer implementation.
+	 */
 	virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end,
 								 access_info_t &event, bool is_fake = false);
+	/**
+	 * Use this variant for passing through the IP/MEM event your Importer
+	 * received.
+	 */
+	virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end,
+								 Trace_Event &event, bool is_fake = false);
 	virtual void open_unused_ec_intervals();
 	virtual bool close_ec_intervals();
 
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev) = 0;
+								 Trace_Event &ev) = 0;
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev) = 0;
+								 Trace_Event &ev) = 0;
 
 
 	void set_elf(fail::ElfReader *elf) { m_elf = elf; }
@@ -89,9 +103,7 @@ public:
 	void set_memorymap(fail::MemoryMap *mm) { m_mm = mm; }
 	void set_faultspace_rightmargin(char accesstype) { m_faultspace_rightmargin = accesstype; }
 	void set_sanitychecks(bool enabled) { m_sanitychecks = enabled; }
-
+	void set_extended_trace(bool enabled) { m_extended_trace = enabled; }
-
-
 };
 
 #endif

tools/import-trace/InstructionImporter.cc

@@ -11,7 +11,7 @@ using namespace fail;
 static Logger LOG("InstructionImporter");
 
 bool InstructionImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-										  const Trace_Event &ev) {
+										  Trace_Event &ev) {
 	if (!binary) {
 		/* Disassemble the binary if necessary */
 		llvm::InitializeAllTargetInfos();
@@ -60,11 +60,12 @@ bool InstructionImporter::handle_ip_event(fail::simtime_t curtime, instruction_c
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
+
-		access.access_type  = 'R'; // instruction fetch is always a read
+		// pass through potentially available extended trace information
-		access.data_address = data_address;
+		ev.set_accesstype(ev.READ); // instruction fetch is always a read
-		access.data_width    = 1; // exactly one byte
+		ev.set_memaddr(data_address);
-		if (!add_trace_event(left_margin, right_margin, access)) {
+		ev.set_width(1); // exactly one byte
+		if (!add_trace_event(left_margin, right_margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}

tools/import-trace/InstructionImporter.hpp

@@ -11,9 +11,9 @@ class InstructionImporter : public Importer {
 
 public:
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev) {
+								  Trace_Event &ev) {
 		/* ignore on purpose */
 		return true;
 	}

tools/import-trace/MemoryImporter.cc

@@ -5,12 +5,12 @@ using namespace fail;
 static fail::Logger LOG("MemoryImporter");
 
 
-bool MemoryImporter::handle_ip_event(simtime_t curtime, instruction_count_t instr, const Trace_Event &ev) {
+bool MemoryImporter::handle_ip_event(simtime_t curtime, instruction_count_t instr, Trace_Event &ev) {
 	return true;
 }
 
 bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t instr,
-									  const Trace_Event &ev) {
+									  Trace_Event &ev) {
 	address_t from = ev.memaddr(), to = ev.memaddr() + ev.width();
 	// Iterate over all accessed bytes
 	// FIXME Keep complete trace information (access width)?
@@ -39,11 +39,11 @@ bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t ins
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
+
-		access.access_type  = ev.accesstype() == ev.READ ? 'R' : 'W';
+		// pass through potentially available extended trace information
-		access.data_address = data_address;
+		ev.set_memaddr(data_address);
-		access.data_width    = 1; // exactly one byte
+		ev.set_width(1); // exactly one byte
-		if (!add_trace_event(left_margin, right_margin, access)) {
+		if (!add_trace_event(left_margin, right_margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}

tools/import-trace/MemoryImporter.hpp

@@ -9,9 +9,9 @@ class MemoryImporter : public Importer {
 public:
 
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev);
+								  Trace_Event &ev);
 };
 
 #endif

tools/import-trace/RandomJumpImporter.cc

@@ -25,7 +25,7 @@ bool RandomJumpImporter::cb_commandline_init() {
 }
 
 bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-									   const Trace_Event &ev) {
+									   Trace_Event &ev) {
 	if (!binary) {
 		// Parse command line again, for jump-from and jump-to
 		// operations
@@ -116,11 +116,12 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
+
-		access.access_type  = 'R'; // instruction fetch is always a read
+		// pass through potentially available extended trace information
-		access.data_address = to_addr;
+		ev.set_accesstype(ev.READ); // instruction fetch is always a read
-		access.data_width    = 4; // exactly one byte
+		ev.set_memaddr(to_addr);
-		if (!add_trace_event(margin, margin, access)) {
+		ev.set_width(4); // FIXME arbitrary?
+		if (!add_trace_event(margin, margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}

tools/import-trace/RandomJumpImporter.hpp

@@ -23,9 +23,9 @@ public:
 	virtual bool cb_commandline_init();
 
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev) {
+								  Trace_Event &ev) {
 		/* ignore on purpose */
 		return true;
 	}

tools/import-trace/RegisterImporter.cc

@@ -29,7 +29,7 @@ bool RegisterImporter::cb_commandline_init() {
 
 
 bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t instr,
-										const Trace_Event &ev,
+										Trace_Event &ev,
 										const LLVMtoFailTranslator::reginfo_t &info,
 										char access_type) {
 	address_t from = info.toDataAddress();
@@ -57,11 +57,12 @@ bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t i
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
+
-		access.access_type	= access_type; // instruction fetch is always a read
+		// pass through potentially available extended trace information
-		access.data_address = data_address;
+		ev.set_width(1); // exactly one byte
-		access.data_width	 = 1; // exactly one byte
+		ev.set_memaddr(data_address);
-		if (!add_trace_event(left_margin, right_margin, access)) {
+		ev.set_accesstype(access_type == 'R' ? ev.READ : ev.WRITE);
+		if (!add_trace_event(left_margin, right_margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}
@@ -75,7 +76,7 @@ bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t i
 
 
 bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-									   const Trace_Event &ev) {
+									   Trace_Event &ev) {
 	if (!binary) {
 		// Parse command line again, for jump-from and jump-to
 		// operations

tools/import-trace/RegisterImporter.hpp

@@ -13,7 +13,7 @@ class RegisterImporter : public Importer {
 	llvm::OwningPtr<fail::LLVMDisassembler> disas;
 
 	bool addRegisterTrace(fail::simtime_t curtime, instruction_count_t instr,
-						  const Trace_Event &ev,
+						  Trace_Event &ev,
 						  const fail::LLVMtoFailTranslator::reginfo_t &info,
 						  char access_type);
 
@@ -29,9 +29,9 @@ public:
 	virtual bool cb_commandline_init();
 
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev) {
+								  Trace_Event &ev) {
 		/* ignore on purpose */
 		return true;
 	}

tools/import-trace/main.cc

@@ -87,7 +87,9 @@ int main(int argc, char *argv[]) {
 	CommandLine::option_handle NO_DELETE =
 		cmd.addOption("", "no-delete", Arg::None,
 			"--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE");
-
+	CommandLine::option_handle EXTENDED_TRACE =
+		cmd.addOption("", "extended-trace", Arg::None,
+			"--extended-trace \tImport extended trace information if available");
 
 	// variant 1: care (synthetic Rs)
 	// variant 2: don't care (synthetic Ws)
@@ -208,6 +210,9 @@ int main(int argc, char *argv[]) {
 	if (cmd[ENABLE_SANITYCHECKS].count() > 0) {
 		importer->set_sanitychecks(true);
 	}
+	if (cmd[EXTENDED_TRACE].count() > 0) {
+		importer->set_extended_trace(true);
+	}
 
 	if (!importer->init(variant, benchmark, db)) {
 		LOG << "importer->init() failed" << endl;