Merge commit 'dcd2c021a5ac91d38187d397914e5f51e2fc8819'

Conflicts:
	tools/import-trace/RegisterImporter.cc

Change-Id: I4f49c976bd60badba73c15746aa03c420cb9f77b

cmake/config_failbochs.sh.in

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-SOURCE_DIR=@CMAKE_SOURCE_DIR@
-BINARY_DIR=@CMAKE_BINARY_DIR@
-PREFIX_DIR=@BOCHS_PREFIX_DIR@
-
-if [ ! -d "$SOURCE_DIR" ]; then
-	echo Source directory does not exists! $SOURCE_DIR
-	exit 2
-fi
-
-if [ ! -d "$PREFIX_DIR" ]; then
-	echo Prefix directory does not exists! $BINARY_DIR 
-	exit 2
-fi
-
-
-./configure CXX="ag++ -p $SOURCE_DIR -I$SOURCE_DIR/src -I"$BINARY_DIR"/src --real-instances --Xcompiler" LIBTOOL="/bin/sh ./libtool --tag=CXX" --prefix=$PREFIX_DIR --enable-{a20-pin,x86-64,cpu-level=6,ne2000,acpi,pci,usb,repeat-speedups,trace-cache,fast-function-calls,host-specific-asms,disasm,all-optimizations,readline,clgd54xx,fpu,vmx=2,monitor-mwait,cdrom,sb16=linux,gdb-stub} --with-all-libs
-

doc/how-to-build.txt

@@ -9,16 +9,18 @@ Required for Fail*:
  - libpcl1-dev
  - libboost-thread-dev
  - protobuf-compiler
- - cmake
+ - cmake 2.8.2 (2.8.11 preferred)
  - cmake-curses-gui
  - binutils-dev
  - AspectC++ (ag++, ac++): AspectC++ 1.1 or newer is known to work and can be
-   obtained from http://www.aspectc.org; nightlies can be downloaded from
-   http://akut.aspectc.org
+   obtained from <http://www.aspectc.org>; nightlies can be downloaded from
+   <http://akut.aspectc.org>.  Make sure you use the 64-bit version if running
+   in a 64-bit environment.
  - optional:
    * LLVM 3.3 (needed for several importers in tools/import-trace)
      (compiles/links with 3.1 or 3.2, but fails to properly import information
      from ELF binaries not compiled with -ffunction-sections)
+   * a MySQL 5.0+ or MariaDB 5.1+ (MariaDB 5.5 recommended) server
 
 
 Required for the Bochs simulator backend:
@@ -66,11 +68,8 @@ For the first time:
       $ cd fail/
  2. (Optional) Cleanup previous CMake remnants:
       $ find -name CMakeCache.txt | xargs rm
- 3. Create out of source build directory (${BUILD_DIR}, see also "fail-structure.txt"):
+ 3. Create out-of-source build directory (${BUILD_DIR}, see also "fail-structure.txt"):
       $ mkdir build
-    Note that currently this build directory must be located somewhere below
-    the fail/ directory, as generated .ah files in there will not be included
-    in the compile process otherwise.
  4. Enter out-of-source build directory.  All generated files end up there.
       $ cd build
  5. Generate CMake environment.
@@ -168,6 +167,7 @@ Configure Bochs to use debugging-related compiler flags (expects to be in ${BUIL
   $ CFLAGS="-g -O0" CXXFLAGS="-g -O0" ./configure --prefix=... ... (see above)
 You might additionally want to configure the rest of Fail* into debug mode by
 setting CMAKE_BUILD_TYPE to "Debug" (ccmake, see above).
+FIXME: Does this still work?
 
 
 Profiling-based optimization build:
@@ -233,3 +233,27 @@ After changes to Fail*/gem5 code (incl. aspect headers):
       $ make   (or  nice make -jN)
     (as in the last step of the previous section).
 
+=========================================================================================
+Database backend setup: MySQL / MariaDB
+=========================================================================================
+ 1. Install MySQL or MariaDB via your favourite package manager.  (MariaDB
+    offers repositories for all major distributions.)
+ 2. Increase network timeouts in /etc/mysql/my.cnf to a reasonably high value,
+    campaigns tend to timeout on a regular basis with the default settings:
+      net_write_timeout = 3600
+      net_read_timeout = 3600
+    You may also want to tune other settings, especially increase memory usage.
+    There's lots of tuning knowledge on the web, and also automatic tuning
+    scripts such as the "Tuning Primer" <https://launchpad.net/mysql-tuning-primer>
+    or "MySQL Tuner" <http://mysqltuner.com>.
+ 3. Setup a non-root DB user.  <http://dev.mysql.com/doc/refman/5.5/en/adding-users.html>
+    Create a ~/.my.cnf (mode 0600) to avoid having to pass user name and
+    password to every program contacting the DB:
+      [client]
+      user=XXX
+      password=XXX
+ 4. Create a database for each FI campaign you want to conduct, and grant all
+    permissions to your non-root user.
+    <http://dev.mysql.com/doc/refman/5.5/en/create-database.html>
+    <http://dev.mysql.com/doc/refman/5.5/en/grant.html>
+    <http://dev.mysql.com/doc/refman/5.5/en/adding-users.html>

doc/how-to-use.txt

@@ -112,6 +112,63 @@ An example for separate campaign/experiment implementations.
  2. Run the FailBochs instance, in properly defined environment:
       $ fail-client -q
 
+Experiment "dciao-kernelstructs":
+**********************************************************************
+This is an example for a database-driven FI campaign, with a campaign server
+instantiating the generic DatabaseCampaign.  The general workflow is as follows:
+ - Create a new database for this campaign, e.g., "dciao".  (See
+   how-to-build.txt, "Database backend setup")
+ - Record a trace using the generic tracing tool: Build a fail-client with the
+   "generic-tracing" experiment, and parametrize it with -Wf,[option] (e.g.,
+   "fail-client -Wf,--help").  Be aware that the generic tracing "experiment"
+   needs a complete simulator environment along with the ELF binary that
+   supplies symbol addresses; for FailBochs this means the usual bochsrc and
+   boot image files need to be present.
+   FIXME: dciao-kernelstructs has not been committed to
+          danceos/devel/experiment_targets/, use a "complete" example here
+ - Import the trace to the database using the import-trace tool (enable
+   BUILD_IMPORT_TRACE in the CMake configuration to get this built).  The
+   --variant/--benchmark parameters are only significant if you intend to
+   evaluate multiple protection variants and/or benchmarks.  Currently the
+   following importers (--importer X) are implemented:
+     MemoryImporter: Imports fault locations for single-location single-event
+       RAM faults (e.g., single-bit flips, or burst faults within the same
+       byte).  Directly maps memory access events from the trace to the DB.
+     AdvancedMemoryImporter: A MemoryImporter that additionally imports
+       Relyzer-style conditional branch history, instruction opcodes, and a
+       virtual duration = time2 - time1 + 1 column.
+     RegisterImporter: Imports fault locations for single-location single-event
+       register-file faults (e.g., single-bit flips in general-purpose
+       registers).  Considers only instruction addresses from the trace,
+       disassembles corresponding instructions in the supplied ELF binary (using
+       LLVM's disassembler library), and extracts used/defined registers.
+       Registers are mapped to/from "addresses" with fail::LLVMtoFailTranslator.
+     InstructionImporter: Interprets every instruction fetch as a memory read,
+       and handles it the same way the MemoryImporter handles "normal" memory
+       accesses.  Implements a fault model with faults in CPU instructions.
+     RandomJumpImporter: Implements multi-bit faults in the instruction pointer.
+       As the IP is read before every instruction, the fault space explodes
+       rapidly.  Should therefore be limited to small memory areas to jump
+       from/to.
+   Note that specifying an importer with --importer adds more parameters to the
+   --help output in some cases.
+   DB detail: This tool creates and fills the variant and trace tables.
+ - Prune the fault space with the prune-trace tool (enable BUILD_PRUNE_TRACE in
+   the CMake configuration).  This prepares all information necessary for
+   running the FI campaign.  Currently only the "basic" pruning method is
+   available, applying usual def/use pruning.
+   DB detail: This tool creates and fills the fsppilot, fspgroup and fspmethod
+   tables.
+ - Run the campaign server as usual.  In this case it does not do much more than
+   to instantiate the generic DatabaseCampaign, and to parametrize it with the
+   experiment-specific protobuf message type (DCIAOKernelProtoMsg) which must
+   adhere to some structural constraints (a required DatabaseCampaignMessage
+   fsppilot member, and a repeated group Result that contains no further
+   subgroups or repeated fields).  The DatabaseCampaign takes care of
+   distributing unfinished jobs, and collecting the results in an automatically
+   created result table with columns corresponding to the fields in the Result
+   group of the protobuf message.
+
 =========================================================================================
 Parallelization
 =========================================================================================
@@ -165,16 +222,6 @@ Some useful things to note:
  - The runcampaign.sh script starts the coolchecksum-server. Note that the server
    instance will terminate immediately (without notice), if there is still an
    existing coolcampaign.csv file.
- - In order to make the performance gains (mentioned above) take effect, a "workload
-   balancing" between the server and the clients is mandatory. This means that
-   the communication overhead (client <-> server) and the time needed to execute
-   the experiment code on the client-side should be in due proportion. More
-   specifically, for each experiment there will be exactly 2 TCP connections
-   (send parameter set to client, send result to server) established. Therefore
-   you should ensure that the jobs you distribute take enough time not to
-   overflow the server with requests. You may need to bundle parameters for
-   more than one experiment if a single experiment only takes a few hundred
-   milliseconds.  (See existing experiments for examples.)
 
 =========================================================================================
 Steps to run an experiment with gem5:

src/core/sal/Architecture.hpp

@@ -0,0 +1,15 @@
+// Architecture.hpp: wraps architecture definition headers
+#ifndef __ARCHITECTURE_HPP__
+  #define __ARCHITECTURE_HPP__
+
+#include "config/FailConfig.hpp"
+
+#ifdef BUILD_X86
+#include "x86/X86Architecture.hpp"
+#endif
+
+#ifdef BUILD_ARM
+#include "arm/ARMArchitecture.hpp"
+#endif
+
+#endif

src/core/sal/CPU.cc

@@ -22,6 +22,7 @@ void CPUArchitecture::m_addRegister(Register* reg, RegisterType type)
 Register* CPUArchitecture::getRegister(size_t i) const
 {
 	assert(i < m_Registers.size() && "FATAL ERROR: Invalid index provided!");
+	assert(m_Registers[i]->getId() == i && "FATAL ERROR: Register index mismatch");
 	return m_Registers[i];
 }
 

src/core/sal/CPU.hpp

@@ -13,7 +13,7 @@ namespace fail {
 /**
  * \class CPUArchitecture
  * This is the base class for CPU architectures that can be used to merge information and
- * functionallity that every backend with the same target architecture will share. The classes
+ * functionality that every backend with the same target architecture will share.  The classes
  * directly derived from this are especially meant to be usable in campaigns, so they shouldn't
  * contain any backend specific code.
  */

src/core/sal/Register.hpp

@@ -102,7 +102,7 @@ public:
 	 */
 	iterator begin() { return m_Regs.begin(); }
 	/**
-	 * Returns an iterator to the end of the interal data structure.
+	 * Returns an iterator to the end of the internal data structure.
 	 * \code
 	 * [1|2| ... |n]X
 	 *              ^

src/core/sal/arm/ArmArchitecture.hpp

@@ -17,6 +17,10 @@ public:
 	~ArmArchitecture();
 };
 
+#ifdef BUILD_ARM
+typedef ArmArchitecture Architecture;
+#endif
+
 /**
  * \enum GPRegIndex
  * Defines the general purpose (GP) register identifier for the ARM

src/core/sal/x86/X86Architecture.hpp

@@ -18,6 +18,10 @@ public:
 	~X86Architecture();
 };
 
+#ifdef BUILD_X86
+typedef X86Architecture Architecture;
+#endif
+
 /**
  * \enum GPRegisterId
  * Symbolic identifier to access the x86 general purpose register

src/core/util/Database.cc

@@ -8,7 +8,15 @@ static fail::Logger LOG("Database", true);
 
 using namespace fail;
 
+#ifndef __puma
+boost::mutex Database::m_global_lock;
+#endif
+
 Database::Database(const std::string &username, const std::string &host, const std::string &database) {
+#ifndef __puma
+	boost::lock_guard<boost::mutex> guard(m_global_lock);
+#endif
+
 	handle = mysql_init(0);
 	last_result = 0;
 	mysql_options(handle, MYSQL_READ_DEFAULT_FILE, "~/.my.cnf");
@@ -26,6 +34,9 @@ Database::~Database()
 	// flush cached INSERTs if available
 	insert_multiple();
 
+#ifndef __puma
+	boost::lock_guard<boost::mutex> guard(m_global_lock);
+#endif
 	mysql_close(handle);
 }
 
@@ -217,7 +228,7 @@ void Database::cmdline_setup() {
 	HOSTNAME	  = cmd.addOption("H", "hostname", Arg::Required,
 								  "-h/--hostname \tMYSQL Hostname (default: taken from ~/.my.cnf)");
 	USERNAME	  = cmd.addOption("u", "username", Arg::Required,
-								  "-u/--username \tMYSQL Username (default: taken from ~/.my.cnf, or your current user)");
+								  "-u/--username \tMYSQL Username (default: taken from ~/.my.cnf, or your current user)\n");
 }
 
 Database * Database::cmdline_connect() {

src/core/util/Database.hpp

@@ -23,6 +23,7 @@ namespace fail {
 		MYSQL_RES *last_result; // !< Used for mysql_result_free
 #ifndef __puma
 		boost::mutex m_handle_lock;
+		static boost::mutex m_global_lock;
 #endif
 		std::string m_insertquery;
 		std::vector<std::string> m_insertquery_values;

src/core/util/llvmdisassembler/LLVMDisassembler.cpp

@@ -105,6 +105,7 @@ void LLVMDisassembler::disassemble()
 					instr_info.opcode = Inst.getOpcode();
 					instr_info.length = Size;
 					instr_info.address = SectionAddr + Index;
+					instr_info.conditional_branch = desc.isConditionalBranch();
 
 					unsigned int  pos = 0;
 					for (MCInst::iterator it = Inst.begin(); it != Inst.end(); ++it) {

src/core/util/llvmdisassembler/LLVMDisassembler.hpp

@@ -42,6 +42,7 @@ public:
 		unsigned int opcode;
 		unsigned int address;
 		unsigned char length;
+		bool conditional_branch;
 		std::vector<register_t> reg_uses;
 		std::vector<register_t> reg_defs;
 	};

src/core/util/llvmdisassembler/testing/llvmDisTest.cc

@@ -62,6 +62,10 @@ int main(int argc, char* argv[]) {
 			 it != instr.reg_defs.end(); ++it) {
 			std::cout << reg_info.getName(*it) << "(" << *it << ") ";
 		}
+
+		if (instr.conditional_branch) {
+			std::cout << "(conditional branch)";
+		}
 		std::cout << std::endl;
 	}
 }

src/experiments/generic-tracing/experiment.cc

@@ -41,7 +41,7 @@ void  GenericTracing::parseOptions() {
 
 	CommandLine::option_handle FULL_TRACE = cmd.addOption("", "full-trace", Arg::None, "--full-trace \tDo a full trace (more data, default: off)");
 	CommandLine::option_handle MEM_SYMBOL	= cmd.addOption("m", "memory-symbol", Arg::Required,
-															"-m,--memory-symbol \tELF symbol(s) to trace accesses (without specifiying all mem read/writes are traced)");
+															"-m,--memory-symbol \tELF symbol(s) to trace accesses (default: all mem read/writes are traced)");
 	CommandLine::option_handle MEM_REGION	= cmd.addOption("M", "memory-region", Arg::Required,
 															"-M,--memory-region \trestrict memory region which is traced"
 															"	Possible formats: 0x<address>, 0x<address>:0x<address>, 0x<address>:<length>");

src/experiments/instantiate-experiment-indirect.ah.in

@@ -5,10 +5,6 @@
 // includes generated headers (e.g., protobuf message definitions) that are not
 // guaranteed to exist when the aspect is woven.
 
-// FIXME: cmake does not remove these .ah files when the user configures
-// another experiment (or even makes "clean").  Currently, this needs to be
-// worked around by manually removing $BUILDDIR/core/experiments/*/*.ah .
-
 // You need to provide the implementation of this function in your experiment
 // directory:
 void instantiate@EXPERIMENT_TYPE@();

src/experiments/instantiate-experiment.ah.in

@@ -1,10 +1,6 @@
 #ifndef __INSTANTIATE_@EXPERIMENT_TYPE@_AH__
   #define __INSTANTIATE_@EXPERIMENT_TYPE@_AH__
 
-// FIXME: cmake does not remove these .ah files when the user configures
-// another experiment (or even makes "clean").  Currently, this needs to be
-// worked around by manually removing $BUILDDIR/core/experiments/*/*.ah .
-
 // Make sure your experiment declaration is in experiment.hpp:
 #include "../experiments/@EXPERIMENT_NAME@/experiment.hpp"
 #include "sal/SALInst.hpp"

src/experiments/nanojpeg/campaign.cc

@@ -52,14 +52,14 @@ bool NanoJPEGCampaign::run()
 	// list: latest accesses (instr offset | bit mask)
 	map<GPRegisterId, std::list<std::pair<unsigned, uint64_t> > > reg_cascade;
 	// open up an equivalence class for all bits in all GPRs
-	reg_cascade[RID_EAX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_EBX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_ECX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_EDX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_ESP].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_EBP].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_ESI].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
-	reg_cascade[RID_EDI].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CAX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CBX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CCX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CDX].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CSP].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CBP].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CSI].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
+	reg_cascade[RID_CDI].push_front(std::pair<unsigned, uint64_t>(0, 0xffffffffffffffffULL));
 
 	// load trace
 	ifstream tracef(NANOJPEG_TRACE);
@@ -145,7 +145,7 @@ bool NanoJPEGCampaign::run()
 				acc->second &= ~common_mask;
 
 				// new EC with experiments: acc->first -- instr, common_mask
-//				if (reg != RID_EBP && reg != RID_ESI && reg != RID_EDI) {
+//				if (reg != RID_CBP && reg != RID_CSI && reg != RID_CDI) {
 					count_exp += add_experiment_ec(acc->first, instr, absolute_instr, reg, common_mask);
 //				}
 
@@ -189,7 +189,7 @@ bool NanoJPEGCampaign::run()
 				// skip empty EC (because register was read within the same instruction)?
 				if (acc->first <= instr) {
 					// new EC with known result: acc->first -- instr, common_mask
-//					if (reg != RID_EBP && reg != RID_ESI && reg != RID_EDI) {
+//					if (reg != RID_CBP && reg != RID_CSI && reg != RID_CDI) {
 						count_known += add_known_ec(acc->first, instr, absolute_instr, reg, common_mask);
 //					}
 				}
@@ -234,7 +234,7 @@ bool NanoJPEGCampaign::run()
 			// skip empty EC (because register was read within the same instruction)?
 			if (acc->first <= instr) {
 				// new EC with known result: acc->first -- instr, common_mask
-//				if (reg != RID_EBP && reg != RID_ESI && reg != RID_EDI) {
+//				if (reg != RID_CBP && reg != RID_CSI && reg != RID_CDI) {
 					count_exp += add_experiment_ec(acc->first, instr, absolute_instr, reg, common_mask);
 //				}
 			}

tools/import-trace/AdvancedMemoryImporter.cc

@@ -0,0 +1,171 @@
+#include <algorithm>
+#include "AdvancedMemoryImporter.hpp"
+
+using namespace llvm;
+using namespace llvm::object;
+using namespace fail;
+
+static fail::Logger LOG("AdvancedMemoryImporter");
+
+std::string AdvancedMemoryImporter::database_additional_columns()
+{
+	return MemoryImporter::database_additional_columns() +
+		"opcode INT UNSIGNED NULL, "
+		"duration BIGINT UNSIGNED AS (time2 - time1 + 1) PERSISTENT, "
+		"jumphistory INT UNSIGNED NULL, ";
+}
+
+void AdvancedMemoryImporter::database_insert_columns(std::string& sql, unsigned& num_columns)
+{
+	// FIXME upcall?
+	sql = ", opcode, jumphistory";
+	num_columns = 2;
+}
+
+//#include <google/protobuf/text_format.h>
+
+bool AdvancedMemoryImporter::database_insert_data(Trace_Event &ev, MYSQL_BIND *bind, unsigned num_columns, bool is_fake)
+{
+	static my_bool null = true;
+	// FIXME upcall?
+	assert(num_columns == 2);
+#if 0
+	// sanity check
+	if (!is_fake && delayed_entries.size() > 0 && ev.ip() != delayed_entries.front().ev.ip()) {
+		std::string out;
+		google::protobuf::TextFormat::PrintToString(ev, &out);
+		std::cout << "ev: " << out << std::endl;
+		google::protobuf::TextFormat::PrintToString(delayed_entries.front().ev, &out);
+		std::cout << "delayed_entries.front.ev: " << out << std::endl;
+	}
+#endif
+	assert(is_fake || delayed_entries.size() == 0 || ev.ip() == delayed_entries.front().ev.ip());
+	bind[0].buffer_type = MYSQL_TYPE_LONG;
+	bind[0].is_unsigned = 1;
+	bind[0].buffer = &delayed_entries.front().opcode;
+	bind[1].buffer_type = MYSQL_TYPE_LONG;
+	bind[1].is_unsigned = 1;
+	bind[1].buffer = &m_cur_branchmask;
+	if (is_fake) {
+		bind[0].is_null = &null;
+		bind[1].is_null = &null;
+	}
+	return true;
+}
+
+void AdvancedMemoryImporter::insert_delayed_entries(bool finalizing)
+{
+	unsigned branchmask;
+	unsigned last_branches_before = UINT_MAX;
+	// If we don't know enough future, and there's a chance we'll learn more,
+	// delay further.
+	for (std::deque<DelayedTraceEntry>::iterator it = delayed_entries.begin();
+	     it != delayed_entries.end() &&
+	     (it->branches_before + BRANCH_WINDOW_SIZE <= branches_taken.size() ||
+	      finalizing);
+	     it = delayed_entries.erase(it)) {
+		// determine branche decisions before / after this mem event
+		if (it->branches_before != last_branches_before) {
+			branchmask = 0;
+			int pos = std::max(-(signed)BRANCH_WINDOW_SIZE, - (signed) it->branches_before);
+			int maxpos = std::min(BRANCH_WINDOW_SIZE, branches_taken.size() - it->branches_before);
+			for (; pos < maxpos; ++pos) {
+				branchmask |=
+					((unsigned) branches_taken[it->branches_before + pos])
+					<< (16 - pos - 1);
+			}
+			m_cur_branchmask = branchmask;
+		}
+
+		//LOG << "AdvancedMemoryImporter::insert_delayed_entries instr = " << it->instr << " data_address = " << it->ev.memaddr() << std::endl;
+
+		// trigger INSERT
+		// (will call back via database_insert_data() and ask for additional data)
+		MemoryImporter::handle_mem_event(it->curtime, it->instr, it->ev);
+	}
+
+	// FIXME branches_taken could be shrunk here to stay within a bounded
+	// memory footprint
+}
+
+bool AdvancedMemoryImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
+	Trace_Event &ev)
+{
+	// Previous instruction was a branch, check and remember whether it was taken
+	if (m_last_was_conditional_branch) {
+		m_last_was_conditional_branch = false;
+		branches_taken.push_back(ev.ip() != m_ip_jump_not_taken);
+	}
+
+	// Check whether we know enough branch-taken future to INSERT a few more
+	// (delayed) trace entries
+	insert_delayed_entries(false);
+
+	if (!binary) {
+		/* Disassemble the binary if necessary */
+		llvm::InitializeAllTargetInfos();
+		llvm::InitializeAllTargetMCs();
+		llvm::InitializeAllDisassemblers();
+
+		if (error_code ec = createBinary(m_elf->getFilename(), binary)) {
+			LOG << m_elf->getFilename() << "': " << ec.message() << ".\n";
+			return false;
+		}
+
+// necessary due to an AspectC++ bug triggered by LLVM 3.3's dyn_cast()
+#ifndef __puma
+		ObjectFile *obj = dyn_cast<ObjectFile>(binary.get());
+		disas.reset(new LLVMDisassembler(obj));
+#endif
+		disas->disassemble();
+		LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap();
+		LOG << "instructions disassembled: " << std::dec << instr_map.size() << " Triple: " << disas->GetTriple() << std::endl;
+#if 0
+		for (LLVMDisassembler::InstrMap::const_iterator it = instr_map.begin();
+			it != instr_map.end(); ++it) {
+			LOG << "DIS " << std::hex << it->second.address << " " << (int) it->second.length << std::endl;
+		}
+#endif
+	}
+
+	const LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap();
+	const LLVMDisassembler::InstrMap::const_iterator it = instr_map.find(ev.ip());
+	if (it == instr_map.end()) {
+		LOG << "WARNING: LLVMDisassembler hasn't disassembled instruction at 0x"
+		    << ev.ip() << " -- are you using LLVM < 3.3?" << std::endl;
+		return true; // probably weird things will happen now
+	}
+	const LLVMDisassembler::Instr &opcode = it->second;
+
+	/* Now we've got the opcode and know whether it's a conditional branch.  If
+	 * it is, the next IP event will tell us whether it was taken or not. */
+	if (opcode.conditional_branch) {
+		m_last_was_conditional_branch = true;
+		m_ip_jump_not_taken = opcode.address + opcode.length;
+	}
+
+	// IP events may need to be delayed, too, if the parent Importer draws any
+	// information from them.  MemoryImporter does not, though.
+	//return MemoryImporter::handle_ip_event(curtime, instr, ev);
+	return true;
+}
+
+bool AdvancedMemoryImporter::handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
+	Trace_Event &ev)
+{
+	const LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap();
+	const LLVMDisassembler::Instr &opcode = instr_map.at(ev.ip());
+	DelayedTraceEntry entry = { curtime, instr, ev, opcode.opcode, branches_taken.size() };
+	delayed_entries.push_back(entry);
+
+	// delay upcall to handle_mem_event until we know enough future branch decisions
+	return true;
+}
+
+bool AdvancedMemoryImporter::trace_end_reached()
+{
+	LOG << "inserting remaining trace events ..." << std::endl;
+	// INSERT the remaining entries (with incomplete branch future)
+	insert_delayed_entries(true);
+	return true;
+}

tools/import-trace/AdvancedMemoryImporter.hpp

@@ -0,0 +1,60 @@
+#ifndef __ADVANCED_MEMORY_IMPORTER_H__
+#define __ADVANCED_MEMORY_IMPORTER_H__
+
+#include <vector>
+#include <deque>
+#include "MemoryImporter.hpp"
+
+#include "util/llvmdisassembler/LLVMDisassembler.hpp"
+
+/**
+ * A MemoryImporter that additionally imports Relyzer-style conditional branch
+ * history, instruction opcodes, and a virtual duration = time2 - time1 + 1
+ * column (MariaDB 5.2+ only!) for fault-space pruning purposes.
+ *
+ * Initially this was implemented by directly passing through trace events to
+ * the MemoryImporter, keeping a record of conditional jumps and opcodes, and
+ * UPDATEing all inserted rows in a second pass when the MemoryImporter is
+ * finished.
+ *
+ * Unfortunately, UPDATE is very slow, and keeping all information in memory
+ * till the end doesn't scale indefinitely.  Therefore the implementation now
+ * delays passing memory access events upwards to the MemoryImporter only until
+ * enough branch history is aggregated, and taps into Importer's database
+ * operations with a set of new virtual functions that are called downwards.
+ */
+class AdvancedMemoryImporter : public MemoryImporter {
+	llvm::OwningPtr<llvm::object::Binary> binary;
+	llvm::OwningPtr<fail::LLVMDisassembler> disas;
+	bool m_last_was_conditional_branch;
+	fail::guest_address_t m_ip_jump_not_taken;
+	std::vector<bool> branches_taken;
+	struct DelayedTraceEntry {
+		fail::simtime_t curtime;
+		instruction_count_t instr;
+		Trace_Event ev;
+		unsigned opcode;
+		unsigned branches_before;
+	};
+	std::deque<DelayedTraceEntry> delayed_entries;
+	static const unsigned BRANCH_WINDOW_SIZE = 16; //!< increasing this requires changing the underlying data types
+
+	unsigned m_cur_branchmask;
+
+	void insert_delayed_entries(bool finalizing);
+
+public:
+	AdvancedMemoryImporter() : m_last_was_conditional_branch(false) {}
+
+protected:
+	virtual std::string database_additional_columns();
+	virtual void database_insert_columns(std::string& sql, unsigned& num_columns);
+	virtual bool database_insert_data(Trace_Event &ev, MYSQL_BIND *bind, unsigned num_columns, bool is_fake);
+	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
+		Trace_Event &ev);
+	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
+		Trace_Event &ev);
+	virtual bool trace_end_reached();
+};
+
+#endif

tools/import-trace/CMakeLists.txt

@@ -8,6 +8,7 @@ if (BUILD_LLVM_DISASSEMBLER)
     InstructionImporter.cc
     RegisterImporter.cc
     RandomJumpImporter.cc
+    AdvancedMemoryImporter.cc
   )
 
   include(FindLLVM)

tools/import-trace/Importer.cc

@@ -13,13 +13,16 @@ bool Importer::init(const std::string &variant, const std::string &benchmark, Da
 	if (!m_variant_id) {
 		return false;
 	}
+	m_extended_trace_regs =
+		m_arch.getRegisterSetOfType(RT_TRACE);
 	LOG << "Importing to variant " << variant << "/" << benchmark
 		<< " (ID: " << m_variant_id << ")" << std::endl;
 	return true;
 }
 
 bool Importer::create_database() {
-	std::string create_statement = "CREATE TABLE IF NOT EXISTS trace ("
+	std::stringstream create_statement;
+	create_statement << "CREATE TABLE IF NOT EXISTS trace ("
 		"	variant_id int(11) NOT NULL,"
 		"	instr1 int(10) unsigned NOT NULL,"
 		"	instr1_absolute int(10) unsigned DEFAULT NULL,"
@@ -29,10 +32,20 @@ bool Importer::create_database() {
 		"	time2 bigint(10) unsigned NOT NULL,"
 		"	data_address int(10) unsigned NOT NULL,"
 		"	width tinyint(3) unsigned NOT NULL,"
-		"	accesstype enum('R','W') NOT NULL,"
+		"	accesstype enum('R','W') NOT NULL,";
+	if (m_extended_trace) {
+		create_statement << "   data_value int(10) unsigned NULL,";
+		for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin();
+			it != m_extended_trace_regs->end(); ++it) {
+			create_statement << " r" << (*it)->getId() << " int(10) unsigned NULL,";
+			create_statement << " r" << (*it)->getId() << "_deref int(10) unsigned NULL,";
+		}
+	}
+	create_statement << database_additional_columns();
+	create_statement <<
 		"	PRIMARY KEY (variant_id,data_address,instr2)"
 		") engine=MyISAM ";
-	return db->query(create_statement.c_str());
+	return db->query(create_statement.str().c_str());
 }
 
 
@@ -82,17 +95,24 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) {
 			/* Another instruction was executed, handle it in the
 			   subclass */
 			if (!handle_ip_event(curtime, instr, ev)) {
+				LOG << "error: handle_ip_event() failed at instr=" << instr << std::endl;
 				return false;
 			}
 
 			instr++;
 		} else {
 			if (!handle_mem_event(curtime, instr, ev)) {
+				LOG << "error: handle_mem_event() failed at instr=" << instr << std::endl;
 				return false;
 			}
 		}
 	}
 
+	if (!trace_end_reached()) {
+		LOG << "trace_end_reached() failed" << std::endl;
+		return false;
+	}
+
 	// Why -1?	In most cases it does not make sense to inject before the
 	// very last instruction, as we won't execute it anymore.  This *only*
 	// makes sense if we also inject into parts of the result vector.  This
@@ -103,7 +123,7 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) {
 	m_last_ip = ev.ip();      // The last event in the log
 
 	/* Signal that the trace was completely imported */
-	LOG << "trace duration: " << (curtime - m_time_trace_start) << " ticks" << std::endl;
+	LOG << "trace duration: " << std::dec << (curtime - m_time_trace_start) << " ticks" << std::endl;
 	LOG << "Inserted " << m_row_count << " real trace events into the database" << std::endl;
 
 
@@ -219,60 +239,169 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) {
 
 bool Importer::add_trace_event(margin_info_t &begin, margin_info_t &end,
 							   access_info_t &access, bool is_fake) {
-	static MYSQL_STMT *stmt = 0;
-	if (!stmt) {
-		std::string sql("INSERT INTO trace (variant_id, instr1, instr1_absolute, instr2, instr2_absolute, time1, time2, data_address, width,"
-		                "					accesstype)"
-		                "VALUES (?,?,?,?,?,?,?,?,?,?)");
-		stmt = mysql_stmt_init(db->getHandle());
-		if (mysql_stmt_prepare(stmt, sql.c_str(), sql.length())) {
-			LOG << "query '" << sql << "' failed: " << mysql_error(db->getHandle()) << std::endl;
+	Trace_Event e;
+	e.set_ip(end.ip);
+	e.set_memaddr(access.data_address);
+	e.set_width(access.data_width);
+	e.set_accesstype(access.access_type == 'R' ? e.READ : e.WRITE);
+	return add_trace_event(begin, end, e, is_fake);
+}
+
+bool Importer::add_trace_event(margin_info_t &begin, margin_info_t &end,
+							   Trace_Event &event, bool is_fake) {
+	if (!m_import_write_ecs && event.accesstype() == event.WRITE) {
+		return true;
+	}
+
+	// insert extended trace info if configured and available
+	bool extended = m_extended_trace && event.has_trace_ext();
+
+	MYSQL_STMT **stmt;
+	unsigned *columns;
+	static MYSQL_STMT *stmt_basic = 0;
+	static unsigned columns_basic = 0;
+	static MYSQL_STMT *stmt_extended = 0;
+	static unsigned columns_extended = 0;
+	stmt = extended ? &stmt_extended : &stmt_basic;
+	columns = extended ? &columns_extended : &columns_basic;
+
+	static unsigned num_additional_columns = 0;
+
+	if (!*stmt) {
+		std::stringstream sql;
+		sql << "INSERT INTO trace (variant_id, instr1, instr1_absolute, instr2, instr2_absolute, time1, time2, "
+		       "data_address, width, accesstype";
+		*columns = 10;
+		if (extended) {
+			sql << ", data_value";
+			(*columns)++;
+			for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin();
+				it != m_extended_trace_regs->end(); ++it) {
+				sql << ", r" << (*it)->getId() << ", r" << (*it)->getId() << "_deref";
+			}
+		}
+
+		// Ask specialized importers whether they want to INSERT additional
+		// columns.
+		std::string additional_columns;
+		database_insert_columns(additional_columns, num_additional_columns);
+		sql << additional_columns;
+
+		sql << ") VALUES (?";
+		for (unsigned i = 1;
+		     i < *columns +
+		         (extended ? m_extended_trace_regs->count() * 2 : 0) +
+		         num_additional_columns;
+		     ++i) {
+			sql << ",?";
+		}
+		sql << ")";
+
+		*stmt = mysql_stmt_init(db->getHandle());
+		if (mysql_stmt_prepare(*stmt, sql.str().c_str(), sql.str().length())) {
+			LOG << "query '" << sql.str() << "' failed: " << mysql_error(db->getHandle()) << std::endl;
 			return false;
 		}
 	}
 
-	MYSQL_BIND bind[10];
-	my_bool is_null = is_fake;
-	my_bool null = true;
+	// extended trace:
+	// retrieve register / register-dereferenced values
+	std::map<int, uint32_t> register_values;
+	std::map<int, uint32_t> register_values_deref;
+	unsigned data_value = 0;
+	const Trace_Event_Extended& ev_ext = event.trace_ext();
+	if (extended) {
+		data_value = ev_ext.data();
+		for (int i = 0; i < ev_ext.registers_size(); i++) {
+			const Trace_Event_Extended_Registers& reg = ev_ext.registers(i);
+			register_values[reg.id()] = reg.value();
+			register_values_deref[reg.id()] = reg.value_deref();
+		}
+	}
+
+	// C99 / g++ extension VLA to the rescue:
+	MYSQL_BIND bind[*columns + m_extended_trace_regs->count() * 2 + num_additional_columns];
+	my_bool fake_null = is_fake;
+	my_bool null = true, not_null = false;
 	long unsigned accesstype_len = 1;
 
+	unsigned data_address = event.memaddr();
+	unsigned width = event.width();
+	char accesstype = event.accesstype() == event.READ ? 'R' : 'W';
+
 	// LOG << m_variant_id  << "-" << ":" << begin.dyninstr << ":" << end.dyninstr << "-" << data_address << " " << accesstype << std::endl;
 
-
+	// sizeof works fine for VLAs
 	memset(bind, 0, sizeof(bind));
-	for (unsigned i = 0; i < sizeof(bind)/sizeof(*bind); ++i) {
+	for (unsigned i = 0; i < *columns; ++i) {
 		bind[i].buffer_type = MYSQL_TYPE_LONG;
 		bind[i].is_unsigned = 1;
 		switch (i) {
 		case 0: bind[i].buffer = &m_variant_id; break;
 		case 1: bind[i].buffer = &begin.dyninstr; break;
 		case 2: bind[i].buffer = &begin.ip;
-			bind[i].is_null = begin.ip == 0 ? &null : &is_null;
+			bind[i].is_null = begin.ip == 0 ? &null : &fake_null;
 			break;
 		case 3: bind[i].buffer = &end.dyninstr; break;
 		case 4: bind[i].buffer = &end.ip;
-			bind[i].is_null = &is_null; break;
+			bind[i].is_null = &fake_null; break;
 		case 5: bind[i].buffer = &begin.time;
 			bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
 			break;
 		case 6: bind[i].buffer = &end.time;
 			bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
 			break;
-		case 7: bind[i].buffer = &access.data_address; break;
-		case 8: bind[i].buffer = &access.data_width; break;
-		case 9: bind[i].buffer = &access.access_type;
+		case 7: bind[i].buffer = &data_address; break;
+		case 8: bind[i].buffer = &width; break;
+		case 9: bind[i].buffer = &accesstype;
 			bind[i].buffer_type = MYSQL_TYPE_STRING;
 			bind[i].buffer_length = accesstype_len;
 			bind[i].length = &accesstype_len;
 			break;
+		// only visited in extended mode:
+		case 10: bind[i].buffer = &data_value;
+			bind[i].is_null = ev_ext.has_data() ? &not_null : &null; break;
 		}
 	}
-	if (mysql_stmt_bind_param(stmt, bind)) {
-		LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(stmt) << std::endl;
+	if (extended) {
+		unsigned i = 0;
+		for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin();
+			it != m_extended_trace_regs->end(); ++it, ++i) {
+			assert(*columns + i*2 + 1 < sizeof(bind)/sizeof(*bind));
+			bind[*columns + i*2    ].buffer_type = MYSQL_TYPE_LONG;
+			bind[*columns + i*2    ].is_unsigned = 1;
+			if (register_values.count((*it)->getId())) {
+				bind[*columns + i*2    ].buffer = &register_values[(*it)->getId()];
+			} else {
+				bind[*columns + i*2    ].buffer = &width; // arbitrary
+				bind[*columns + i*2    ].is_null = &null;
+			}
+
+			bind[*columns + i*2 + 1].buffer_type = MYSQL_TYPE_LONG;
+			bind[*columns + i*2 + 1].is_unsigned = 1;
+			if (register_values_deref.count((*it)->getId())) {
+				bind[*columns + i*2 + 1].buffer = &register_values_deref[(*it)->getId()];
+			} else {
+				bind[*columns + i*2 + 1].buffer = &width; // arbitrary
+				bind[*columns + i*2 + 1].is_null = &null;
+			}
+		}
+	}
+
+	// Ask specialized importers what concrete data they want to INSERT.
+	if (num_additional_columns) {
+		unsigned pos = *columns + (extended ? m_extended_trace_regs->count() * 2 : 0);
+		if (!database_insert_data(event, bind + pos, num_additional_columns, is_fake)) {
+			return false;
+		}
+	}
+
+	if (mysql_stmt_bind_param(*stmt, bind)) {
+		LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(*stmt) << std::endl;
 		return false;
 	}
-	if (mysql_stmt_execute(stmt)) {
-		LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(stmt) << std::endl;
+	if (mysql_stmt_execute(*stmt)) {
+		LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(*stmt) << std::endl;
 		LOG << "IP: " << std::hex << end.ip << std::endl;
 		return false;
 	}

tools/import-trace/Importer.hpp

@@ -6,6 +6,7 @@
 #include "util/ProtoStream.hpp"
 #include "util/ElfReader.hpp"
 #include "sal/SALConfig.hpp"
+#include "sal/Architecture.hpp"
 #include "util/Database.hpp"
 #include "util/MemoryMap.hpp"
 #include "comm/TracePlugin.pb.h"
@@ -22,7 +23,11 @@ protected:
 	fail::MemoryMap *m_mm;
 	char m_faultspace_rightmargin;
 	bool m_sanitychecks;
+	bool m_import_write_ecs;
+	bool m_extended_trace;
 	fail::Database *db;
+	fail::Architecture m_arch;
+	fail::UniformRegisterSet *m_extended_trace_regs;
 
 	/* How many rows were inserted into the database */
 	unsigned m_row_count;
@@ -60,8 +65,54 @@ protected:
 	instruction_count_t m_last_instr;
 	fail::simtime_t m_last_time;
 
+protected:
+	/**
+	 * Allows specialized importers to add more table columns instead of
+	 * completely overriding create_database().  The returned SQL CREATE TABLE
+	 * snippet should be terminated with a comma if non-empty.  Should call and
+	 * pass through their parent's implementation.
+	 */
+	virtual std::string database_additional_columns() { return ""; }
+	/**
+	 * Similar to database_additional_columns(), this allows specialized
+	 * importers to define which additional columns it wants to INSERT
+	 * alongside what add_trace_event() adds by itself.  This may be identical
+	 * to or a subset of what database_additional_columns() specifies.  The SQL
+	 * snippet should *begin* with a comma if non-empty.
+	 */
+	virtual void database_insert_columns(std::string& sql, unsigned& num_columns) { num_columns = 0; }
+	/**
+	 * Will be called back from add_trace_event() to fill in data for the
+	 * columns specified by database_insert_columns().
+	 */
+	virtual bool database_insert_data(Trace_Event &ev, MYSQL_BIND *bind, unsigned num_columns, bool is_fake) { return true; }
+	/**
+	 * Use this variant if passing through the IP/MEM event does not make any
+	 * sense for your Importer implementation.
+	 */
+	virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end,
+								 access_info_t &event, bool is_fake = false);
+	/**
+	 * Use this variant for passing through the IP/MEM event your Importer
+	 * received.
+	 */
+	virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end,
+								 Trace_Event &event, bool is_fake = false);
+	virtual void open_unused_ec_intervals();
+	virtual bool close_ec_intervals();
+
+	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
+								 Trace_Event &ev) = 0;
+	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
+								 Trace_Event &ev) = 0;
+	/**
+	 * May be overridden by importers that need to do stuff after the last
+	 * event was consumed.
+	 */
+	virtual bool trace_end_reached() { return true; }
+
 public:
-	Importer() : m_sanitychecks(false), m_row_count(0), m_time_trace_start(0) {}
+	Importer() : m_sanitychecks(false), m_import_write_ecs(true), m_extended_trace(false), m_row_count(0), m_time_trace_start(0) {}
 	bool init(const std::string &variant, const std::string &benchmark, fail::Database *db);
 
 	/**
@@ -73,25 +124,14 @@ public:
 	virtual bool create_database();
 	virtual bool copy_to_database(fail::ProtoIStream &ps);
 	virtual bool clear_database();
-	virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end,
-								 access_info_t &event, bool is_fake = false);
-	virtual void open_unused_ec_intervals();
-	virtual bool close_ec_intervals();
-
-	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev) = 0;
-	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev) = 0;
-
 
 	void set_elf(fail::ElfReader *elf) { m_elf = elf; }
 
 	void set_memorymap(fail::MemoryMap *mm) { m_mm = mm; }
 	void set_faultspace_rightmargin(char accesstype) { m_faultspace_rightmargin = accesstype; }
 	void set_sanitychecks(bool enabled) { m_sanitychecks = enabled; }
-
-
-
+	void set_import_write_ecs(bool enabled) { m_import_write_ecs = enabled; }
+	void set_extended_trace(bool enabled) { m_extended_trace = enabled; }
 };
 
 #endif

tools/import-trace/InstructionImporter.cc

@@ -11,7 +11,7 @@ using namespace fail;
 static Logger LOG("InstructionImporter");
 
 bool InstructionImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-										  const Trace_Event &ev) {
+										  Trace_Event &ev) {
 	if (!binary) {
 		/* Disassemble the binary if necessary */
 		llvm::InitializeAllTargetInfos();
@@ -60,11 +60,12 @@ bool InstructionImporter::handle_ip_event(fail::simtime_t curtime, instruction_c
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
-		access.access_type  = 'R'; // instruction fetch is always a read
-		access.data_address = data_address;
-		access.data_width    = 1; // exactly one byte
-		if (!add_trace_event(left_margin, right_margin, access)) {
+
+		// pass through potentially available extended trace information
+		ev.set_accesstype(ev.READ); // instruction fetch is always a read
+		ev.set_memaddr(data_address);
+		ev.set_width(1); // exactly one byte
+		if (!add_trace_event(left_margin, right_margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}

tools/import-trace/InstructionImporter.hpp

@@ -9,11 +9,11 @@ class InstructionImporter : public Importer {
 	llvm::OwningPtr<llvm::object::Binary> binary;
 	llvm::OwningPtr<fail::LLVMDisassembler> disas;
 
-public:
+protected:
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev) {
+								  Trace_Event &ev) {
 		/* ignore on purpose */
 		return true;
 	}

tools/import-trace/MemoryImporter.cc

@@ -5,12 +5,12 @@ using namespace fail;
 static fail::Logger LOG("MemoryImporter");
 
 
-bool MemoryImporter::handle_ip_event(simtime_t curtime, instruction_count_t instr, const Trace_Event &ev) {
+bool MemoryImporter::handle_ip_event(simtime_t curtime, instruction_count_t instr, Trace_Event &ev) {
 	return true;
 }
 
 bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t instr,
-									  const Trace_Event &ev) {
+									  Trace_Event &ev) {
 	address_t from = ev.memaddr(), to = ev.memaddr() + ev.width();
 	// Iterate over all accessed bytes
 	// FIXME Keep complete trace information (access width)?
@@ -39,11 +39,11 @@ bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t ins
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
-		access.access_type  = ev.accesstype() == ev.READ ? 'R' : 'W';
-		access.data_address = data_address;
-		access.data_width    = 1; // exactly one byte
-		if (!add_trace_event(left_margin, right_margin, access)) {
+
+		// pass through potentially available extended trace information
+		ev.set_memaddr(data_address);
+		ev.set_width(1); // exactly one byte
+		if (!add_trace_event(left_margin, right_margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}
@@ -54,4 +54,3 @@ bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t ins
 	}
 	return true;
 }
-

tools/import-trace/MemoryImporter.hpp

@@ -6,12 +6,11 @@
 
 class MemoryImporter : public Importer {
 
-public:
-
+protected:
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev);
+								  Trace_Event &ev);
 };
 
 #endif

tools/import-trace/RandomJumpImporter.cc

@@ -18,14 +18,14 @@ bool RandomJumpImporter::cb_commandline_init() {
 	CommandLine &cmd = CommandLine::Inst();
 
 	FROM = cmd.addOption("", "jump-from", Arg::Required,
-						 "--jump-from\t RandomJump: Which addresses should be jumped from\n");
+		"--jump-from \tRandomJump: Which addresses should be jumped from (a memory map; may be used more than once)");
 	TO   = cmd.addOption("", "jump-to", Arg::Required,
-						 "--jump-to\t RandomJump: Where to jump (a memory map>\n");
+		"--jump-to \tRandomJump: Where to jump (a memory map; may be used more than once)");
 	return true;
 }
 
 bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-									   const Trace_Event &ev) {
+									   Trace_Event &ev) {
 	if (!binary) {
 		// Parse command line again, for jump-from and jump-to
 		// operations
@@ -35,8 +35,8 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co
 			return false;
 		}
 
-		// Read FROM memory file
-		if (cmd[FROM].count() > 0) {
+		// Read FROM memory map(s)
+		if (cmd[FROM]) {
 			m_mm_from = new MemoryMap();
 			for (option::Option *o = cmd[FROM]; o; o = o->next()) {
 				if (!m_mm_from->readFromFile(o->arg)) {
@@ -46,7 +46,8 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co
 			}
 		}
 
-		if (cmd[TO].count() > 0) {
+		// Read TO memory map(s)
+		if (cmd[TO]) {
 			m_mm_to = new MemoryMap();
 			for (option::Option *o = cmd[TO]; o; o = o->next()) {
 				if (!m_mm_to->readFromFile(o->arg)) {
@@ -81,7 +82,7 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co
 		/* Collect all addresses we want to jump to */
 		for (LLVMDisassembler::InstrMap::const_iterator instr = instr_map.begin();
 			 instr != instr_map.end(); ++instr) {
-			if (m_mm_to->isMatching(instr->first)) {
+			if (m_mm_to && m_mm_to->isMatching(instr->first)) {
 				m_jump_to_addresses.push_back(instr->first);
 			}
 		}
@@ -91,12 +92,12 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co
 
 	// skip events that are outside the memory map. -m instruction map
 	if (m_mm && !m_mm->isMatching(ev.ip())) {
-		return  true;
+		return true;
 	}
 
 	// skip events that are outside the --jump-from memory map.
-	if (!m_mm_from->isMatching(ev.ip())) {
-		return  true;
+	if (m_mm_from && !m_mm_from->isMatching(ev.ip())) {
+		return true;
 	}
 
 
@@ -116,11 +117,12 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
-		access_info_t access;
-		access.access_type  = 'R'; // instruction fetch is always a read
-		access.data_address = to_addr;
-		access.data_width    = 4; // exactly one byte
-		if (!add_trace_event(margin, margin, access)) {
+
+		// pass through potentially available extended trace information
+		ev.set_accesstype(ev.READ); // instruction fetch is always a read
+		ev.set_memaddr(to_addr);
+		ev.set_width(4); // FIXME arbitrary, use Instr.length instead?
+		if (!add_trace_event(margin, margin, ev)) {
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}

tools/import-trace/RandomJumpImporter.hpp

@@ -16,16 +16,18 @@ class RandomJumpImporter : public Importer {
 	fail::MemoryMap *m_mm_from, *m_mm_to;
 	std::vector<fail::guest_address_t> m_jump_to_addresses;
 public:
+	RandomJumpImporter() : m_mm_from(0), m_mm_to(0) {}
 	/**
 	 * Callback function that can be used to add command line options
 	 * to the campaign
 	 */
 	virtual bool cb_commandline_init();
 
+protected:
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev) {
+								  Trace_Event &ev) {
 		/* ignore on purpose */
 		return true;
 	}

tools/import-trace/RegisterImporter.cc

@@ -17,22 +17,21 @@ static Logger LOG("RegisterImporter");
 bool RegisterImporter::cb_commandline_init() {
 	CommandLine &cmd = CommandLine::Inst();
 
-	NO_GP	= cmd.addOption("", "no-gp", Arg::None,
-							"--no-gp\t RegisterImporter: do not inject general purpose registers\n");
+	NO_GP = cmd.addOption("", "no-gp", Arg::None,
+		"--no-gp \tRegisterImporter: do not inject general purpose registers");
 	FLAGS = cmd.addOption("", "flags", Arg::None,
-						  "--flags: RegisterImporter: trace flags register\n");
-	IP	 = cmd.addOption("", "ip", Arg::None,
-						 "--ip: RegisterImporter: trace instruction pointer\n");
-	NO_SPLIT	 = cmd.addOption("", "do-not-split", Arg::None,
-						 "--do-not-split: RegisterImporter: Do not split the registers into one byte chunks\n");
-
+		"--flags \tRegisterImporter: inject flags register");
+	IP    = cmd.addOption("", "ip", Arg::None,
+		"--ip \tRegisterImporter: inject instruction pointer");
+	NO_SPLIT = cmd.addOption("", "do-not-split", Arg::None,
+		 "--do-not-split \tRegisterImporter: Do not split the registers into one byte chunks");
 
 	return true;
 }
 
 
 bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t instr,
-										const Trace_Event &ev,
+										Trace_Event &ev,
 										const LLVMtoFailTranslator::reginfo_t &info,
 										char access_type) {
 	address_t from, to;
@@ -75,11 +74,20 @@ bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t i
 		// we're currently looking at; the EC is defined by
 		// data_address, dynamic instruction start/end, the absolute PC at
 		// the end, and time start/end
+<<<<<<< HEAD
 		access_info_t access;
 		access.access_type	= access_type; // instruction fetch is always a read
 		access.data_address = data_address;
 		access.data_width	 = chunk_width;
 		if (!add_trace_event(left_margin, right_margin, access)) {
+=======
+
+		// pass through potentially available extended trace information
+		ev.set_width(1); // exactly one byte
+		ev.set_memaddr(data_address);
+		ev.set_accesstype(access_type == 'R' ? ev.READ : ev.WRITE);
+		if (!add_trace_event(left_margin, right_margin, ev)) {
+>>>>>>> dcd2c021a5ac91d38187d397914e5f51e2fc8819
 			LOG << "add_trace_event failed" << std::endl;
 			return false;
 		}
@@ -93,7 +101,7 @@ bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t i
 
 
 bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-									   const Trace_Event &ev) {
+									   Trace_Event &ev) {
 	if (!binary) {
 		// Parse command line again, for jump-from and jump-to
 		// operations
@@ -102,6 +110,7 @@ bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_coun
 			std::cerr << "Error parsing arguments." << std::endl;
 			return false;
 		}
+<<<<<<< HEAD
 
 		// Read FROM memory file
 		if (cmd[NO_GP].count() > 0) {
@@ -117,6 +126,11 @@ bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_coun
 			do_split_registers = false;
 		}
 
+=======
+		do_gp = !cmd[NO_GP];
+		do_flags = cmd[FLAGS];
+		do_ip = cmd[IP];
+>>>>>>> dcd2c021a5ac91d38187d397914e5f51e2fc8819
 
 		/* Disassemble the binary if necessary */
 		llvm::InitializeAllTargetInfos();
@@ -146,7 +160,7 @@ bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_coun
 	const LLVMDisassembler::Instr &opcode = instr_map.at(ev.ip());
 	//const MCRegisterInfo &reg_info = disas->getRegisterInfo();
 
-	fail::LLVMtoFailTranslator & ltof =	 disas->getTranslator() ;
+	fail::LLVMtoFailTranslator & ltof = disas->getTranslator() ;
 
 	for (std::vector<LLVMDisassembler::register_t>::const_iterator it = opcode.reg_uses.begin();
 		 it != opcode.reg_uses.end(); ++it) {

tools/import-trace/RegisterImporter.hpp

@@ -13,7 +13,7 @@ class RegisterImporter : public Importer {
 	llvm::OwningPtr<fail::LLVMDisassembler> disas;
 
 	bool addRegisterTrace(fail::simtime_t curtime, instruction_count_t instr,
-						  const Trace_Event &ev,
+						  Trace_Event &ev,
 						  const fail::LLVMtoFailTranslator::reginfo_t &info,
 						  char access_type);
 
@@ -29,10 +29,11 @@ public:
 	 */
 	virtual bool cb_commandline_init();
 
+protected:
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
-								 const Trace_Event &ev);
+								 Trace_Event &ev);
 	virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr,
-								  const Trace_Event &ev) {
+								  Trace_Event &ev) {
 		/* ignore on purpose */
 		return true;
 	}

tools/import-trace/main.cc

@@ -12,6 +12,7 @@
 #include "InstructionImporter.hpp"
 #include "RegisterImporter.hpp"
 #include "RandomJumpImporter.hpp"
+#include "AdvancedMemoryImporter.hpp"
 #endif
 
 
@@ -49,8 +50,7 @@ ProtoIStream openProtoStream(std::string input_file) {
 }
 
 int main(int argc, char *argv[]) {
-	std::string trace_file, username, hostname, database, benchmark;
-	std::string variant;
+	std::string trace_file, variant, benchmark;
 	ElfReader *elf_file = 0;
 	MemoryMap *memorymap = 0;
 
@@ -87,7 +87,13 @@ int main(int argc, char *argv[]) {
 	CommandLine::option_handle NO_DELETE =
 		cmd.addOption("", "no-delete", Arg::None,
 			"--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE");
-
+	CommandLine::option_handle NO_WRITE_ECS =
+		cmd.addOption("", "no-write-ecs", Arg::None,
+			"--no-write-ecs \tDo not import any write ECs into the database; "
+			"results in a perforated fault space and is OK if you only use absolute failure numbers");
+	CommandLine::option_handle EXTENDED_TRACE =
+		cmd.addOption("", "extended-trace", Arg::None,
+			"--extended-trace \tImport extended trace information if available");
 
 	// variant 1: care (synthetic Rs)
 	// variant 2: don't care (synthetic Ws)
@@ -106,7 +112,7 @@ int main(int argc, char *argv[]) {
 	CommandLine::option_handle ENABLE_SANITYCHECKS =
 		cmd.addOption("", "enable-sanitychecks", Arg::None,
 			"--enable-sanitychecks \tEnable sanity checks "
-			"(in case something looks fishy)"
+			"(in case something looks fishy) "
 			"(default: disabled)");
 
 	if (!cmd.parse()) {
@@ -116,28 +122,30 @@ int main(int argc, char *argv[]) {
 
 	Importer *importer;
 
-	if (cmd[IMPORTER].count() > 0) {
+	if (cmd[IMPORTER]) {
 		std::string imp(cmd[IMPORTER].first()->arg);
 		if (imp == "BasicImporter" || imp == "MemoryImporter" || imp == "memory" || imp == "mem") {
-			LOG << "Using MemoryImporter" << endl;
+			imp = "MemoryImporter";
 			importer = new MemoryImporter();
 #ifdef BUILD_LLVM_DISASSEMBLER
 		} else if (imp == "InstructionImporter" || imp == "code") {
-			LOG << "Using InstructionImporter" << endl;
+			imp = "InstructionImporter";
 			importer = new InstructionImporter();
 
 		} else if (imp == "RegisterImporter" || imp == "regs") {
-			LOG << "Using RegisterImporter" << endl;
+			imp = "RegisterImporter";
 			importer = new RegisterImporter();
 
 		} else if (imp == "RandomJumpImporter") {
-			LOG << "Using RandomJumpImporter" << endl;
 			importer = new RandomJumpImporter();
+		} else if (imp == "AdvancedMemoryImporter") {
+			importer = new AdvancedMemoryImporter();
 #endif
 		} else {
 			LOG << "Unkown import method: " << imp << endl;
 			exit(-1);
 		}
+		LOG << "Using " << imp << endl;
 
 	} else {
 		LOG << "Using MemoryImporter" << endl;
@@ -158,30 +166,33 @@ int main(int argc, char *argv[]) {
 		exit(0);
 	}
 
-	if (cmd[TRACE_FILE].count() > 0)
+	if (cmd[TRACE_FILE]) {
 		trace_file = std::string(cmd[TRACE_FILE].first()->arg);
-	else
+	} else {
 		trace_file = "trace.pb";
+	}
 
 	ProtoIStream ps = openProtoStream(trace_file);
 	Database *db = Database::cmdline_connect();
 
-	if (cmd[VARIANT].count() > 0)
+	if (cmd[VARIANT]) {
 		variant = std::string(cmd[VARIANT].first()->arg);
-	else
+	} else {
 		variant = "none";
+	}
 
-	if (cmd[BENCHMARK].count() > 0)
+	if (cmd[BENCHMARK]) {
 		benchmark = std::string(cmd[BENCHMARK].first()->arg);
-	else
+	} else {
 		benchmark = "none";
+	}
 
-	if (cmd[ELF_FILE].count() > 0) {
+	if (cmd[ELF_FILE]) {
 		elf_file = new ElfReader(cmd[ELF_FILE].first()->arg);
 	}
 	importer->set_elf(elf_file);
 
-	if (cmd[MEMORYMAP].count() > 0) {
+	if (cmd[MEMORYMAP]) {
 		memorymap = new MemoryMap();
 		for (option::Option *o = cmd[MEMORYMAP]; o; o = o->next()) {
 			if (!memorymap->readFromFile(o->arg)) {
@@ -191,7 +202,7 @@ int main(int argc, char *argv[]) {
 	}
 	importer->set_memorymap(memorymap);
 
-	if (cmd[FAULTSPACE_RIGHTMARGIN].count() > 0) {
+	if (cmd[FAULTSPACE_RIGHTMARGIN]) {
 		std::string rightmargin(cmd[FAULTSPACE_RIGHTMARGIN].first()->arg);
 		if (rightmargin == "W") {
 			importer->set_faultspace_rightmargin('W');
@@ -205,9 +216,9 @@ int main(int argc, char *argv[]) {
 		importer->set_faultspace_rightmargin('W');
 	}
 
-	if (cmd[ENABLE_SANITYCHECKS].count() > 0) {
-		importer->set_sanitychecks(true);
-	}
+	importer->set_sanitychecks(cmd[ENABLE_SANITYCHECKS]);
+	importer->set_extended_trace(cmd[EXTENDED_TRACE]);
+	importer->set_import_write_ecs(!cmd[NO_WRITE_ECS]);
 
 	if (!importer->init(variant, benchmark, db)) {
 		LOG << "importer->init() failed" << endl;
@@ -223,7 +234,7 @@ int main(int argc, char *argv[]) {
 		exit(-1);
 	}
 
-	if (cmd[NO_DELETE].count() == 0 && !importer->clear_database()) {
+	if (!cmd[NO_DELETE] && !importer->clear_database()) {
 		LOG << "clear_database() failed" << endl;
 		exit(-1);
 	}

tools/prune-trace/Pruner.cc

@@ -71,7 +71,7 @@ bool Pruner::create_database() {
 	    "  data_address    int(10) unsigned NOT NULL,"
 	    "  fspmethod_id    int(11) NOT NULL,"
 	    "  pilot_id        int(11) NOT NULL,"
-	    "  PRIMARY KEY (variant_id, data_address, instr2, fspmethod_id, pilot_id),"
+	    "  PRIMARY KEY (variant_id, data_address, instr2, fspmethod_id),"
 	    "  KEY joinresults (pilot_id,fspmethod_id)) engine=MyISAM";
 
 	return db->query(create_statement.c_str());