import-trace: introduce AdvancedMemoryImporter

A MemoryImporter that additionally imports Relyzer-style conditional
branch history, instruction opcodes, and a virtual
duration=time2-time1+1 column (MariaDB 5.2+ only) for fault-space
pruning purposes.

Change-Id: I6764a26fa8aae21655be44134b88fdee85e67ff6

src/core/util/llvmdisassembler/LLVMDisassembler.cpp

@@ -105,6 +105,7 @@ void LLVMDisassembler::disassemble()
 					instr_info.opcode = Inst.getOpcode();
 					instr_info.length = Size;
 					instr_info.address = SectionAddr + Index;
+					instr_info.conditional_branch = desc.isConditionalBranch();
 
 					unsigned int  pos = 0;
 					for (MCInst::iterator it = Inst.begin(); it != Inst.end(); ++it) {

src/core/util/llvmdisassembler/LLVMDisassembler.hpp

@@ -42,6 +42,7 @@ public:
 		unsigned int opcode;
 		unsigned int address;
 		unsigned char length;
+		bool conditional_branch;
 		std::vector<register_t> reg_uses;
 		std::vector<register_t> reg_defs;
 	};

src/core/util/llvmdisassembler/testing/llvmDisTest.cc

@@ -62,6 +62,10 @@ int main(int argc, char* argv[]) {
 			 it != instr.reg_defs.end(); ++it) {
 			std::cout << reg_info.getName(*it) << "(" << *it << ") ";
 		}
+
+		if (instr.conditional_branch) {
+			std::cout << "(conditional branch)";
+		}
 		std::cout << std::endl;
 	}
 }

tools/import-trace/AdvancedMemoryImporter.cc

@@ -0,0 +1,141 @@
+#include <algorithm>
+#include "AdvancedMemoryImporter.hpp"
+
+using namespace llvm;
+using namespace llvm::object;
+using namespace fail;
+
+static fail::Logger LOG("AdvancedMemoryImporter");
+
+std::string AdvancedMemoryImporter::database_additional_columns()
+{
+	return MemoryImporter::database_additional_columns() +
+		"opcode INT UNSIGNED NULL, "
+		"duration BIGINT UNSIGNED AS (time2 - time1 + 1) PERSISTENT, "
+		"jumphistory INT UNSIGNED NULL, ";
+}
+
+bool AdvancedMemoryImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
+	Trace_Event &ev)
+{
+	// Previous instruction was a branch, check and remember whether it was taken
+	if (m_last_was_conditional_branch) {
+		m_last_was_conditional_branch = false;
+		branches_taken.push_back(ev.ip() != m_ip_jump_not_taken);
+	}
+
+	if (!binary) {
+		/* Disassemble the binary if necessary */
+		llvm::InitializeAllTargetInfos();
+		llvm::InitializeAllTargetMCs();
+		llvm::InitializeAllDisassemblers();
+
+		if (error_code ec = createBinary(m_elf->getFilename(), binary)) {
+			LOG << m_elf->getFilename() << "': " << ec.message() << ".\n";
+			return false;
+		}
+
+// necessary due to an AspectC++ bug triggered by LLVM 3.3's dyn_cast()
+#ifndef __puma
+		ObjectFile *obj = dyn_cast<ObjectFile>(binary.get());
+		disas.reset(new LLVMDisassembler(obj));
+#endif
+		disas->disassemble();
+		LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap();
+		LOG << "instructions disassembled: " << std::dec << instr_map.size() << " Triple: " << disas->GetTriple() << std::endl;
+#if 0
+		for (LLVMDisassembler::InstrMap::const_iterator it = instr_map.begin();
+			it != instr_map.end(); ++it) {
+			LOG << "DIS " << std::hex << it->second.address << " " << (int) it->second.length << std::endl;
+		}
+#endif
+	}
+
+	const LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap();
+	const LLVMDisassembler::InstrMap::const_iterator it = instr_map.find(ev.ip());
+	if (it == instr_map.end()) {
+		LOG << "WARNING: LLVMDisassembler hasn't disassembled instruction at 0x"
+		    << ev.ip() << " -- are you using LLVM < 3.3?" << std::endl;
+		return true; // probably weird things will happen now
+	}
+	const LLVMDisassembler::Instr &opcode = it->second;
+
+	/* Now we've got the opcode and know whether it's a conditional branch.  If
+	 * it is, the next IP event will tell us whether it was taken or not. */
+	if (opcode.conditional_branch) {
+		m_last_was_conditional_branch = true;
+		m_ip_jump_not_taken = opcode.address + opcode.length;
+	}
+
+	return MemoryImporter::handle_ip_event(curtime, instr, ev);
+}
+
+bool AdvancedMemoryImporter::handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
+	Trace_Event &ev)
+{
+	const LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap();
+	const LLVMDisassembler::Instr &opcode = instr_map.at(ev.ip());
+	TraceEntry entry = { instr, ev.memaddr(), ev.width(), opcode.opcode, branches_taken.size() };
+	update_entries.push_back(entry);
+
+	return MemoryImporter::handle_mem_event(curtime, instr, ev);
+}
+
+bool AdvancedMemoryImporter::finalize()
+{
+	LOG << "adding opcodes and jump history to trace events ..." << std::endl;
+
+	MYSQL_STMT *stmt = 0;
+	std::stringstream sql;
+	sql << "UPDATE trace SET opcode = ?, jumphistory = ? "
+	       "WHERE variant_id = " << m_variant_id << " AND data_address BETWEEN ? AND ? AND instr2 = ?";
+	stmt = mysql_stmt_init(db->getHandle());
+	if (mysql_stmt_prepare(stmt, sql.str().c_str(), sql.str().length())) {
+	LOG << "query '" << sql.str() << "' failed: " << mysql_error(db->getHandle()) << std::endl;
+		return false;
+	}
+	MYSQL_BIND bind[5];
+
+	unsigned rowcount = 0, rowcount_blocks = 0;
+	for (std::vector<TraceEntry>::iterator it = update_entries.begin();
+	     it != update_entries.end(); ++it) {
+		// determine branche decisions before / after this mem event
+		unsigned branchmask = 0;
+		int pos = std::max(-16, - (signed) it->branches_before);
+		int maxpos = std::min((unsigned) 16, branches_taken.size() - it->branches_before);
+		for (; pos < maxpos; ++pos) {
+			branchmask |=
+				((unsigned) branches_taken[it->branches_before + pos])
+				<< (16 - pos - 1);
+		}
+
+		memset(bind, 0, sizeof(bind));
+		for (unsigned i = 0; i < sizeof(bind)/sizeof(*bind); ++i) {
+			bind[i].buffer_type = MYSQL_TYPE_LONG;
+			bind[i].is_unsigned = 1;
+		}
+		bind[0].buffer = &it->opcode;
+		bind[1].buffer = &branchmask;
+		bind[2].buffer = &it->data_address;
+		unsigned rightmargin = it->data_address + it->data_width - 1;
+		bind[3].buffer = &rightmargin;
+		bind[4].buffer = &it->instr2;
+
+		if (mysql_stmt_bind_param(stmt, bind)) {
+			LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(stmt) << std::endl;
+			return false;
+		} else if (mysql_stmt_execute(stmt)) {
+			LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(stmt) << std::endl;
+			return false;
+		}
+		rowcount += mysql_stmt_affected_rows(stmt);
+
+		if (rowcount >= rowcount_blocks + 10000) {
+			LOG << "Updated " << rowcount << " trace events" << std::endl;
+			rowcount_blocks += 10000;
+		}
+	}
+	LOG << "Updated " << rowcount << " trace events.  Done." << std::endl;
+
+	return true;
+}

tools/import-trace/AdvancedMemoryImporter.hpp

@@ -0,0 +1,39 @@
+#ifndef __ADVANCED_MEMORY_IMPORTER_H__
+#define __ADVANCED_MEMORY_IMPORTER_H__
+
+#include <vector>
+#include "MemoryImporter.hpp"
+
+#include "util/llvmdisassembler/LLVMDisassembler.hpp"
+
+/**
+ * A MemoryImporter that additionally imports Relyzer-style conditional branch
+ * history, instruction opcodes, and a virtual duration = time2 - time1 + 1
+ * column (MariaDB 5.2+ only!) for fault-space pruning purposes.
+ */
+class AdvancedMemoryImporter : public MemoryImporter {
+	llvm::OwningPtr<llvm::object::Binary> binary;
+	llvm::OwningPtr<fail::LLVMDisassembler> disas;
+	bool m_last_was_conditional_branch;
+	fail::guest_address_t m_ip_jump_not_taken;
+	std::vector<bool> branches_taken;
+	struct TraceEntry {
+		unsigned instr2;
+		uint64_t data_address;
+		unsigned data_width;
+		unsigned opcode;
+		unsigned branches_before;
+	};
+	std::vector<TraceEntry> update_entries;
+
+public:
+	AdvancedMemoryImporter() : m_last_was_conditional_branch(false) {}
+	virtual std::string database_additional_columns();
+	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
+		Trace_Event &ev);
+	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
+		Trace_Event &ev);
+	virtual bool finalize();
+};
+
+#endif

tools/import-trace/CMakeLists.txt

@@ -8,6 +8,7 @@ if (BUILD_LLVM_DISASSEMBLER)
     InstructionImporter.cc
     RegisterImporter.cc
     RandomJumpImporter.cc
+    AdvancedMemoryImporter.cc
   )
 
   include(FindLLVM)

tools/import-trace/Importer.cc

@@ -41,6 +41,7 @@ bool Importer::create_database() {
 			create_statement << " r" << (*it)->getId() << "_deref int(10) unsigned NULL,";
 		}
 	}
+	create_statement << database_additional_columns();
 	create_statement <<
 		"	PRIMARY KEY (variant_id,data_address,instr2)"
 		") engine=MyISAM ";

tools/import-trace/Importer.hpp

@@ -75,6 +75,13 @@ public:
 	virtual bool cb_commandline_init() { return true; }
 
 	virtual bool create_database();
+	/**
+	 * Allows specialized importers to add more table columns instead of
+	 * completely overriding create_database().  The returned SQL CREATE TABLE
+	 * snippet should be terminated with a comma if non-empty.  Should call and
+	 * pass through their parent's implementation.
+	 */
+	virtual std::string database_additional_columns() { return ""; }
 	virtual bool copy_to_database(fail::ProtoIStream &ps);
 	virtual bool clear_database();
 	/**
@@ -96,7 +103,11 @@ public:
 								 Trace_Event &ev) = 0;
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
 								 Trace_Event &ev) = 0;
-
+	/**
+	 * May be overridden by importers that need to do stuff after the last
+	 * event was consumed.
+	 */
+	virtual bool finalize() { return true; }
 
 	void set_elf(fail::ElfReader *elf) { m_elf = elf; }
 

tools/import-trace/main.cc

@@ -12,6 +12,7 @@
 #include "InstructionImporter.hpp"
 #include "RegisterImporter.hpp"
 #include "RandomJumpImporter.hpp"
+#include "AdvancedMemoryImporter.hpp"
 #endif
 
 
@@ -133,6 +134,8 @@ int main(int argc, char *argv[]) {
 
 		} else if (imp == "RandomJumpImporter") {
 			importer = new RandomJumpImporter();
+		} else if (imp == "AdvancedMemoryImporter") {
+			importer = new AdvancedMemoryImporter();
 #endif
 		} else {
 			LOG << "Unkown import method: " << imp << endl;
@@ -235,4 +238,9 @@ int main(int argc, char *argv[]) {
 		LOG << "copy_to_database() failed" << endl;
 		exit(-1);
 	}
+
+	if (!importer->finalize()) {
+		LOG << "finalize() failed" << endl;
+		exit(-1);
+	}
 }