Merge changes If331ad7a,I6241ea2d,Iba4f73dc,I7a49e8e9,Ifae805ae

* changes:
  bochs: fix parsing ips values > 2^31
  tools/prune-trace: various improvements
  tools/import-trace: sanity checks
  tools/import-trace: import timing information + various additions
  cosmetics

scripts/distribute-experiment.sh

@@ -39,7 +39,7 @@ fi
 for h in $FAIL_DISTRIBUTE_HOSTS
 do
 	echo Distributing to $h ...
-	rsync -az --partial --delete-before --delete-excluded --exclude=core --exclude=trace.tc . $h:"$FAIL_EXPERIMENT_TARGETDIR" &
+	rsync -az --partial --delete-before --delete-excluded --exclude=core --exclude=*.tc . $h:"$FAIL_EXPERIMENT_TARGETDIR" &
 done
 
 wait

scripts/killall-bochs.sh

@@ -14,7 +14,7 @@ SCRIPTDIR=$(readlink -f $(dirname $0))
 # env variable defaults
 source $SCRIPTDIR/fail-env.sh
 
-CMD="killall -q client.sh"
+CMD="killall -q client.sh; killall -q fail-client"
 CONNECTION_ATTEMPTS=2
 SSH="ssh -o BatchMode=yes -o ConnectTimeout=60 -o ConnectionAttempts=$CONNECTION_ATTEMPTS"
 
@@ -31,3 +31,6 @@ do
 
 	$SSH $h "$CMD $NCLIENTS" &
 done
+
+wait
+echo "Done."

simulators/bochs/config.cc

@@ -2558,7 +2558,9 @@ static int parse_line_formatted(const char *context, int num_params, char *param
         SIM->get_param_num(BXPN_CPU_NCORES)->set(cores);
         SIM->get_param_num(BXPN_CPU_NTHREADS)->set(threads);
       } else if (!strncmp(params[i], "ips=", 4)) {
-        SIM->get_param_num(BXPN_IPS)->set(atol(&params[i][4]));
+        // DanceOS
+        //SIM->get_param_num(BXPN_IPS)->set(atol(&params[i][4]));
+        SIM->get_param_num(BXPN_IPS)->set(atoll(&params[i][4]));
 #if BX_SUPPORT_SMP
       } else if (!strncmp(params[i], "quantum=", 8)) {
         SIM->get_param_num(BXPN_SMP_QUANTUM)->set(atol(&params[i][8]));
@@ -3823,12 +3825,18 @@ int bx_write_configuration(const char *rc, int overwrite)
   fprintf(fp, "vga_update_interval: %u\n", SIM->get_param_num(BXPN_VGA_UPDATE_INTERVAL)->get());
   fprintf(fp, "vga: extension=%s\n", SIM->get_param_string(BXPN_VGA_EXTENSION)->getptr());
 #if BX_SUPPORT_SMP
-  fprintf(fp, "cpu: count=%u:%u:%u, ips=%u, quantum=%d, ",
+  // DanceOS
+  //fprintf(fp, "cpu: count=%u:%u:%u, ips=%u, quantum=%d, ",
+  fprintf(fp, "cpu: count=%u:%u:%u, ips=%llu, quantum=%d, ",
     SIM->get_param_num(BXPN_CPU_NPROCESSORS)->get(), SIM->get_param_num(BXPN_CPU_NCORES)->get(),
-    SIM->get_param_num(BXPN_CPU_NTHREADS)->get(), SIM->get_param_num(BXPN_IPS)->get(),
+    // DanceOS
+    //SIM->get_param_num(BXPN_CPU_NTHREADS)->get(), SIM->get_param_num(BXPN_IPS)->get(),
+    SIM->get_param_num(BXPN_CPU_NTHREADS)->get(), SIM->get_param_num(BXPN_IPS)->get64(),
     SIM->get_param_num(BXPN_SMP_QUANTUM)->get());
 #else
-  fprintf(fp, "cpu: count=1, ips=%u, ", SIM->get_param_num(BXPN_IPS)->get());
+  // DanceOS
+  //fprintf(fp, "cpu: count=1, ips=%u, ", SIM->get_param_num(BXPN_IPS)->get());
+  fprintf(fp, "cpu: count=1, ips=%u, ", SIM->get_param_num(BXPN_IPS)->get64());
 #endif
   fprintf(fp, "reset_on_triple_fault=%d",
     SIM->get_param_bool(BXPN_RESET_ON_TRIPLE_FAULT)->get());

simulators/bochs/iodev/virt_timer.cc

@@ -396,7 +396,9 @@ void bx_virt_timer_c::init(void)
   }
 
   // Local copy of IPS value to avoid reading it frequently in timer handler
-  ips = SIM->get_param_num(BXPN_IPS)->get();
+  // DanceOS
+  //ips = SIM->get_param_num(BXPN_IPS)->get();
+  ips = SIM->get_param_num(BXPN_IPS)->get64();
 
   register_timer(this, nullTimer, (Bit32u)NullTimerInterval, 1, 1, "Null Timer");
 

simulators/bochs/main.cc

@@ -1054,7 +1054,9 @@ void bx_init_hardware()
   }
 
 
-  bx_pc_system.initialize(SIM->get_param_num(BXPN_IPS)->get());
+  // DanceOS
+  //bx_pc_system.initialize(SIM->get_param_num(BXPN_IPS)->get());
+  bx_pc_system.initialize(SIM->get_param_num(BXPN_IPS)->get64());
 
 
   if (SIM->get_param_string(BXPN_LOG_FILENAME)->getptr()[0]!='-') {

simulators/bochs/pc_system.cc

@@ -64,7 +64,9 @@ bx_pc_system_c::bx_pc_system_c()
   numTimers = 1; // So far, only the nullTimer.
 }
 
-void bx_pc_system_c::initialize(Bit32u ips)
+// DanceOS
+//void bx_pc_system_c::initialize(Bit32u ips)
+void bx_pc_system_c::initialize(Bit64u ips)
 {
   ticksTotal = 0;
   timer[0].timeToFire = NullTimerInterval;
@@ -79,7 +81,9 @@ void bx_pc_system_c::initialize(Bit32u ips)
   // parameter 'ips' is the processor speed in Instructions-Per-Second
   m_ips = double(ips) / 1000000.0L;
 
-  BX_DEBUG(("ips = %u", (unsigned) ips));
+  // DanceOS
+  //BX_DEBUG(("ips = %u", (unsigned) ips));
+  BX_DEBUG(("ips = %llu", (unsigned long long) ips));
 }
 
 void bx_pc_system_c::set_HRQ(bx_bool val)

simulators/bochs/pc_system.h

@@ -87,7 +87,9 @@ public:
   // Timer oriented public features
   // ==============================
 
-  void   initialize(Bit32u ips);
+  // DanceOS
+  //void   initialize(Bit32u ips);
+  void   initialize(Bit64u ips);
   int    register_timer(void *this_ptr, bx_timer_handler_t, Bit32u useconds,
                          bx_bool continuous, bx_bool active, const char *id);
   bx_bool unregisterTimer(unsigned timerID);

src/core/cpn/JobServer.cc

@@ -40,7 +40,6 @@ boost::mutex CommThread::m_CommMutex;
 
 ExperimentData *JobServer::getDone()
 {
-
 #ifndef __puma
 	if (m_undoneJobs.Size() == 0
 	 && noMoreExperiments()
@@ -151,7 +150,7 @@ void JobServer::run()
 		// TODO: Log-level?
 		return;
 	}
-	cout << "JobServer listening...." << endl;
+	cout << "JobServer listening ..." << endl;
 	// TODO: Log-level?
 #ifndef __puma
 	boost::thread* th;
@@ -266,12 +265,8 @@ void CommThread::sendPendingExperimentData(Minion& minion)
 	if (exp.size() != 0) {
 		ctrlmsg.set_job_size(exp.size());
 
-			cout << " >>[";
+		cout << " >>[" << ctrlmsg.workloadid(0) << "+"
-			for ( i = 0; i < exp.size() ; i++) {
+			<< exp.size() << "]      \r" << flush;
-				cout << " "<<   ctrlmsg.workloadid(i) <<" ";
-			}
-			cout << "] " << flush;
-
 
 		if (SocketComm::sendMsg(minion.getSocketDescriptor(), ctrlmsg)) {
 			for (i = 0; i < ctrlmsg.job_size(); i++) {
@@ -310,7 +305,7 @@ void CommThread::sendPendingExperimentData(Minion& minion)
 		ctrlmsg.add_workloadid(workloadID); // set workload id
 		ctrlmsg.set_job_size(1); // In 2nd priority the jobserver send only one job
 		//cout << ">>[Server] Re-sending workload [" << workloadID << "]" << endl;
-		cout << ">>R[" << workloadID << "] " << flush;
+		cout << ">>R[" << workloadID << "]        \r" << flush;
 		if (SocketComm::sendMsg(minion.getSocketDescriptor(), ctrlmsg)) {
 			SocketComm::sendMsg(minion.getSocketDescriptor(), temp_exp->getMessage());
 		}
@@ -332,11 +327,10 @@ void CommThread::receiveExperimentResults(Minion& minion, FailControlMessage& ct
 {
 	int i;
 	ExperimentData* exp = NULL; // Get exp* from running jobs
-	cout << " <<[ ";
+	if (ctrlmsg.workloadid_size() > 0) {
-	for (i = 0; i < ctrlmsg.workloadid_size(); i++) {
+		cout << " <<[" << ctrlmsg.workloadid(0) << "+"
-		cout << ctrlmsg.workloadid(i) << " ";
+		     << ctrlmsg.workloadid_size() << "]        \r" << flush;
 	}
-	cout << "] " << flush;
 #ifndef __puma
 	// Prevent re-sending jobs in sendPendingExperimentData:
 	// a) sendPendingExperimentData needs an intact job to serialize and send it.

src/core/cpn/JobServer.hpp

@@ -112,21 +112,21 @@ public:
 	 */
 	ExperimentData* getDone();
 	/**
-	 * The Campaign controller must signalize, that there will be no
+	 * The Campaign controller must signal that there will be no more parameter
-	 * more parameter sets. We need this, as we allow concurrent parameter
+	 * sets.  We need this, as we allow concurrent parameter generation and
-	 * generation and distribution.
+	 * distribution.
 	 */
 	void setNoMoreExperiments() { m_noMoreExps = true; }
 	/**
-	 * Checks whether there are no more experiment paremeter sets.
+	 * Checks whether there are no more experiment parameter sets.
 	 * @return \c true if no more parameter sets available, \c false otherwise
 	 * @see setNoMoreExperiments
 	 */
 	bool noMoreExperiments() const { return m_noMoreExps; }
 
 	/**
-	 * The Campaign Controller can signalize, that the jobserver can
+	 * The Campaign Controller may signal that the jobserver can stop listening
-	 * stop listening for client connections.
+	 * for client connections.
 	 */
 	void done() { m_finish = true; }
 };

src/core/efw/JobClient.cc

@@ -188,9 +188,7 @@ bool JobClient::sendResult(ExperimentData& result)
 
 		if (m_job_throughput > CLIENT_JOB_LIMIT) {
 			m_job_throughput = CLIENT_JOB_LIMIT;
-		}
+		} else if (m_job_throughput < 1) {
-
-		if (m_job_throughput < 1) {
 			m_job_throughput = 1;
 		}
 

src/core/sal/bochs/BochsController.hpp

@@ -115,6 +115,8 @@ public:
 	 * Fire done: Callback from Simulator 
 	 */
 	void fireInterruptDone();
+	virtual simtime_t getTimerTicks() { return bx_pc_system.time_ticks(); }
+	virtual simtime_t getTimerTicksPerSecond() { return bx_pc_system.time_ticks() / bx_pc_system.time_usec(); /* imprecise hack */ }
 	/* ********************************************************************
 	 * BochsController-specific (not implemented in SimulatorController!):
 	 * ********************************************************************/
@@ -150,8 +152,6 @@ public:
 	 * @see The uses SimulatorController::getCPU().
 	 */
 	ConcreteCPU& detectCPU(BX_CPU_C* pCPU) const;
-	virtual simtime_t getTimerTicks() { return bx_pc_system.time_ticks(); }
-	virtual simtime_t getTimerTicksPerSecond() { return bx_pc_system.time_ticks() / bx_pc_system.time_usec(); /* imprecise hack */ }
 };
 
 } // end-of-namespace: fail

src/core/util/DatabaseProtobufAdapter.cc

@@ -251,7 +251,7 @@ int DatabaseProtobufAdapter::TypeBridge_message::gatherTypes(StringJoiner &inser
 		bool can_be_repeated = true; // default value
 
 		// For repeated messages
-		TypeBridge_message *top_level_msg;
+		TypeBridge_message *top_level_msg = 0;
 
 		const FieldOptions& field_options = field->options();
 		if (field_options.GetExtension(sql_ignore)) {
@@ -411,9 +411,7 @@ int DatabaseProtobufAdapter::field_size_at_pos(const Message *msg, std::vector<i
 }
 
 bool DatabaseProtobufAdapter::insert_row(const google::protobuf::Message *msg) {
-	const Reflection *ref = msg->GetReflection();
+	assert (msg->GetDescriptor() != 0 && msg->GetReflection() != 0);
-	const Descriptor *d = msg->GetDescriptor();
-	assert (d != 0 && ref != 0);
 
 	MYSQL_BIND *bind = new MYSQL_BIND[top_level_msg.field_count];
 

src/core/util/SynchronizedMap.hpp

@@ -28,7 +28,10 @@ private:
 
 	int nextpick;
 	// We need a window at least as wide as the number of clients we serve.
-	enum { pick_window_size = 2000 };
+	// FIXME better solution: when inbound queue is empty, *copy* in-flight map
+	// to a vector, iterate but don't delete; when at the end, copy in-flight
+	// map again and repeat
+	enum { pick_window_size = 50000 };
 
 public:
 	SynchronizedMap() : nextpick(0) { }

src/core/util/SynchronizedQueue.hpp

@@ -68,11 +68,13 @@ public:
 		// Lock is automatically released in the wait and obtained
 		// again after the wait
 #ifndef __puma
-		while (m_queue.size() == 0)
+		while (m_queue.size() == 0) {
 			m_cond.wait(lock);
+		}
 #endif
 		// Retrieve the data from the queue
-		T result=m_queue.front(); m_queue.pop();
+		T result = m_queue.front();
+		m_queue.pop();
 
 		// Notify others that we have free slots
 #ifndef __puma
@@ -101,7 +103,8 @@ public:
 		// again after the wait
 		if (m_queue.size() > 0) {
 			// Retrieve the data from the queue
-			d = m_queue.front(); m_queue.pop();
+			d = m_queue.front();
+			m_queue.pop();
 			// Notify others that we have free slots
 #ifndef __puma
 			if (m_queue.size() < capacity) {

tools/dump-trace/.gitignore

@@ -1 +1 @@
-TracePlugin_pb2.py{,c}
+TracePlugin_pb2.py

tools/dump-trace/DumpTrace.cc

@@ -14,13 +14,13 @@ using std::cerr;
 using std::hex;
 using std::dec;
 
-Logger log("dump-trace", true);
+Logger LOG("dump-trace", true);
 
 std::istream& openStream(const char *input_file,
 	std::ifstream& normal_stream, igzstream& gz_stream) {
 	normal_stream.open(input_file);
 	if (!normal_stream) {
-		log << "couldn't open " << input_file << endl;
+		LOG << "couldn't open " << input_file << endl;
 		exit(-1);
 	}
 	unsigned char b1, b2;
@@ -30,16 +30,16 @@ std::istream& openStream(const char *input_file,
 		normal_stream.close();
 		gz_stream.open(input_file);
 		if (!gz_stream) {
-			log << "couldn't open " << input_file << endl;
+			LOG << "couldn't open " << input_file << endl;
 			exit(-1);
 		}
-		//log << "opened file " << input_file << " in GZip mode" << endl;
+		//LOG << "opened file " << input_file << " in GZip mode" << endl;
 		return gz_stream;
 	}
 
 	normal_stream.seekg(0);
 
-	//log << "opened file " << input_file << " in normal mode" << endl;
+	//LOG << "opened file " << input_file << " in normal mode" << endl;
 	return normal_stream;
 }
 

tools/import-trace/BasicImporter.cc

@@ -1,4 +1,3 @@
-#include <iostream>
 #include "util/Logger.hpp"
 #include "BasicImporter.hpp"
 
@@ -10,6 +9,8 @@ bool BasicImporter::create_database() {
 		"	instr1 int(10) unsigned NOT NULL,"
 		"	instr2 int(10) unsigned NOT NULL,"
 		"	instr2_absolute int(10) unsigned DEFAULT NULL,"
+		"	time1 bigint(10) unsigned NOT NULL,"
+		"	time2 bigint(10) unsigned NOT NULL,"
 		"	data_address int(10) unsigned NOT NULL,"
 		"	width tinyint(3) unsigned NOT NULL,"
 		"	accesstype enum('R','W') NOT NULL,"
@@ -19,13 +20,14 @@ bool BasicImporter::create_database() {
 }
 
 bool BasicImporter::add_trace_event(instruction_count_t begin, instruction_count_t end,
+	fail::simtime_t time_begin, fail::simtime_t time_end,
 	const Trace_Event &event, bool is_fake) {
 
 	static MYSQL_STMT *stmt = 0;
 	if (!stmt) {
-		std::string sql("INSERT INTO trace (variant_id, instr1, instr2, instr2_absolute, data_address, width,"
+		std::string sql("INSERT INTO trace (variant_id, instr1, instr2, instr2_absolute, time1, time2, data_address, width,"
 		                "					accesstype)"
-		                "VALUES (?,?,?,?, ?,?,?)");
+		                "VALUES (?,?,?,?,?,?,?,?,?)");
 		stmt = mysql_stmt_init(db->getHandle());
 		if (mysql_stmt_prepare(stmt, sql.c_str(), sql.length())) {
 			LOG << "query '" << sql << "' failed: " << mysql_error(db->getHandle()) << std::endl;
@@ -33,7 +35,7 @@ bool BasicImporter::add_trace_event(instruction_count_t begin, instruction_count
 		}
 	}
 
-	MYSQL_BIND bind[7];
+	MYSQL_BIND bind[9];
 	my_bool is_null = is_fake;
 	unsigned long accesstype_len = 1;
 	unsigned ip = event.ip();
@@ -42,7 +44,7 @@ bool BasicImporter::add_trace_event(instruction_count_t begin, instruction_count
 	char accesstype = event.accesstype() == event.READ ? 'R' : 'W';
 
 	memset(bind, 0, sizeof(bind));
-	for (int i = 0; i < 7; ++i) {
+	for (unsigned i = 0; i < sizeof(bind)/sizeof(*bind); ++i) {
 		bind[i].buffer_type = MYSQL_TYPE_LONG;
 		bind[i].is_unsigned = 1;
 		switch (i) {
@@ -51,9 +53,15 @@ bool BasicImporter::add_trace_event(instruction_count_t begin, instruction_count
 		case 2: bind[i].buffer = &end; break;
 		case 3: bind[i].buffer = &ip;
 			bind[i].is_null = &is_null; break;
-		case 4: bind[i].buffer = &data_address; break;
+		case 4: bind[i].buffer = &time_begin;
-		case 5: bind[i].buffer = &width; break;
+			bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
-		case 6: bind[i].buffer = &accesstype;
+			break;
+		case 5: bind[i].buffer = &time_end;
+			bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
+			break;
+		case 6: bind[i].buffer = &data_address; break;
+		case 7: bind[i].buffer = &width; break;
+		case 8: bind[i].buffer = &accesstype;
 			bind[i].buffer_type = MYSQL_TYPE_STRING;
 			bind[i].buffer_length = accesstype_len;
 			bind[i].length = &accesstype_len;
@@ -71,5 +79,3 @@ bool BasicImporter::add_trace_event(instruction_count_t begin, instruction_count
 	}
 	return true;
 }
-
-

tools/import-trace/BasicImporter.hpp

@@ -2,12 +2,15 @@
 #define __BASIC_IMPORTER_H__
 
 #include "Importer.hpp"
+#include "util/CommandLine.hpp"
 
 class BasicImporter : public Importer {
 public:
 	virtual bool create_database();
 	bool add_trace_event(instruction_count_t begin,
 						 instruction_count_t end,
+						 fail::simtime_t time_begin,
+						 fail::simtime_t time_end,
 						 const Trace_Event &event,
 						 bool is_fake = false);
 };

tools/import-trace/CMakeLists.txt

@@ -4,7 +4,6 @@ set(SRCS
   DCiAOKernelImporter.cc
 )
 
-## This is the example's campaign server distributing experiment parameters
+add_executable(import-trace main.cc ${SRCS})
-add_executable(import-trace main.cc ${SRCS} ${PROTO_SRCS} ${PROTO_HDRS})
 target_link_libraries(import-trace ${PROTOBUF_LIBRARY} -lmysqlclient fail-util fail-sal fail-comm)
 install(TARGETS import-trace RUNTIME DESTINATION bin)

tools/import-trace/DCiAOKernelImporter.cc

@@ -92,7 +92,7 @@ bool DCiAOKernelImporter::copy_to_database(fail::ProtoIStream &ps) {
 					// we now have an interval-terminating R/W event to the memaddr
 					// we're currently looking at; the EC is defined by
 					// data_address [last_kernel_leave, read_instr] (instr_absolute)
-					if (!add_trace_event(instr1, instr2, ev)) {
+					if (!add_trace_event(instr1, instr2, 0, 0, ev)) { // FIXME use timing data
 						LOG << "add_trace_event failed" << std::endl;
 						return false;
 					}

tools/import-trace/Importer.cc

@@ -28,36 +28,74 @@ bool Importer::clear_database() {
 }
 
 bool Importer::copy_to_database(fail::ProtoIStream &ps) {
-	unsigned row_count = 0;
+	unsigned row_count = 0, row_count_fake = 0;
 	// map for keeping one "open" EC for every address
-	// (maps injection data address => equivalence class)
+	// (maps injection data address =>
+	// dyn. instruction count / time information for equivalence class left margin)
 	AddrLastaccessMap open_ecs;
 
-	// instruction counter within trace
+	// time the trace started/ended
-	unsigned instr = 0;
+	// For now we just use the min/max occuring timestamp; for "sparse" traces
+	// (e.g., only mem accesses, only a subset of the address space) it might
+	// be a good idea to store this explicitly in the trace file, though.
+	simtime_t time_trace_start = 0, curtime = 0;
 
-	// "rightmost" instr where we did a FI experiment
+	// instruction counter within trace
-	unsigned instr_rightmost = 0;
+	instruction_count_t instr = 0;
 
 	Trace_Event ev;
 
 	while (ps.getNext(&ev)) {
+		if (ev.has_time_delta()) {
+			// record trace start
+			// it suffices to do this once, the events come in sorted
+			if (time_trace_start == 0) {
+				time_trace_start = ev.time_delta();
+				LOG << "trace start time: " << time_trace_start << std::endl;
+			}
+			// curtime also always holds the max time, provided we only get
+			// nonnegative deltas
+			assert(ev.time_delta() >= 0);
+			curtime += ev.time_delta();
+		}
+
 		// instruction events just get counted
 		if (!ev.has_memaddr()) {
 			// new instruction
+			// sanity check for overflow
+			if (instr == (1LL << (sizeof(instr)*8)) - 1) {
+				LOG << "error: instruction_count_t overflow, aborting at instr=" << instr << std::endl;
+				return false;
+			}
 			instr++;
 			continue;
 		}
 
 		address_t from = ev.memaddr(), to = ev.memaddr() + ev.width();
 		// Iterate over all accessed bytes
+		// FIXME Keep complete trace information (access width)?
+		//   advantages: may be used for pruning strategies, complete value would be visible; less DB entries
+		//   disadvantages: may need splitting when width varies, lots of special case handling
+		//   Probably implement this in a separate importer when necessary.
 		for (address_t data_address = from; data_address < to; ++data_address) {
-
+			// skip events outside a possibly supplied memory map
-			int instr1 = open_ecs[data_address]; // defaults to 0 if nonexistent
+			if (m_mm && !m_mm->isMatching(data_address)) {
-			int instr2 = instr; // the current instruction
+				continue;
+			}
+			instruction_count_t instr1 =
+				open_ecs[data_address].dyninstr; // defaults to 0 if nonexistent
+			instruction_count_t instr2 = instr; // the current instruction
+			simtime_t time1 = open_ecs[data_address].time;
+			// defaulting to 0 is not such a good idea, memory reads at the
+			// beginning of the trace would get an unnaturally high weight:
+			if (time1 == 0) {
+				time1 = time_trace_start;
+			}
+			simtime_t time2 = curtime;
 
 			// skip zero-sized intervals: these can occur when an instruction
 			// accesses a memory location more than once (e.g., INC, CMPXCHG)
+			// FIXME: look at timing instead?
 			if (instr1 > instr2) {
 				continue;
 			}
@@ -67,62 +105,170 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) {
 
 			// we now have an interval-terminating R/W event to the memaddr
 			// we're currently looking at; the EC is defined by
-			// data_address [instr1, instr2] (instr_absolute)
+			// data_address, dynamic instruction start/end, the absolute PC at
-			if (!add_trace_event(instr1, instr2, ev)) {
+			// the end, and time start/end
+			if (!add_trace_event(instr1, instr2, time1, time2, ev)) {
 				LOG << "add_trace_event failed" << std::endl;
 				return false;
 			}
 			row_count ++;
-			if (row_count % 1000 == 0) {
+			if (row_count % 10000 == 0) {
-				LOG << "Imported " << row_count << " traces into the database" << std::endl;
+				LOG << "Inserted " << row_count << " trace events into the database" << std::endl;
-			}
-
-			if (ev.accesstype() == ev.READ) {
-				// FIXME this is broken: we must abort after the rightmost R
-				// and must not allow Ws to find their way into the known
-				// results
-				instr_rightmost = instr2;
 			}
 
 			// next interval must start at next instruction; the aforementioned
 			// skipping mechanism wouldn't work otherwise
 			//lastuse_it->second = instr2 + 1;
-			open_ecs[data_address] = instr2 + 1;
+			open_ecs[data_address].dyninstr = instr2 + 1;
+			open_ecs[data_address].time = time2 + 1;
 		}
 	}
 
-	LOG << "Inserted " << row_count << " traces into the database" << std::endl;
+	// Close all open intervals (right end of the fault-space) with fake trace
+	// event.  This ensures we have a rectangular fault space (important for,
+	// e.g., calculating the total SDC rate), and unknown memory accesses after
+	// the end of the trace are properly taken into account: Either with a
+	// "don't care" (a synthetic memory write at the right margin), or a "care"
+	// (synthetic read), the latter resulting in more experiments to be done.
+	for (AddrLastaccessMap::iterator lastuse_it = open_ecs.begin();
+		lastuse_it != open_ecs.end(); ++lastuse_it) {
 
-	// FIXME
+		Trace_Event fake_ev;
-	// close all open intervals (right end of the fault-space) with fake trace event
+		fake_ev.set_memaddr(lastuse_it->first);
-//		for (AddrLastaccessMap::iterator lastuse_it = open_ecs.begin();
+		fake_ev.set_width(1);
-//			 lastuse_it != open_ecs.end(); ++lastuse_it) {
+		fake_ev.set_accesstype(m_faultspace_rightmargin == 'R' ? fake_ev.READ : fake_ev.WRITE);
-//			address_t data_address = lastuse_it->first;
+
-//			int instr1 = lastuse_it->second;
+		instruction_count_t instr1 = lastuse_it->second.dyninstr;
+		simtime_t time1 = lastuse_it->second.time;
+
+		// Why -1?	In most cases it does not make sense to inject before the
+		// very last instruction, as we won't execute it anymore.  This *only*
+		// makes sense if we also inject into parts of the result vector.  This
+		// is not the case in this experiment, and with -1 we'll get a result
+		// comparable to the non-pruned campaign.
+		instruction_count_t instr2 = instr - 1;
+
+		simtime_t time2 = curtime; // -1?
 
-// #if 0
-//			// Why -1?	In most cases it does not make sense to inject before the
-//			// very last instruction, as we won't execute it anymore.  This *only*
-//			// makes sense if we also inject into parts of the result vector.  This
-//			// is not the case in this experiment, and with -1 we'll get a result
-//			// comparable to the non-pruned campaign.
-//			int instr2 = instr - 1;
 // #else
 //		// EcosKernelTestCampaign only variant: fault space ends with the last FI experiment
+//		FIXME probably implement this with cmdline parameter FAULTSPACE_CUTOFF
 //		int instr2 = instr_rightmost;
 // #endif
-//			// zero-sized?	skip.
-//			if (instr1 > instr2) {
-//				continue;
-//			}
 
-//			add_trace_event(variant_id, instr1, instr2, 
+		// zero-sized?	skip.
-//				data_address, 1, 'W', 0,
+		// FIXME: look at timing instead?
-//				0, 0, 0, 0, 0, true);
+		if (instr1 > instr2) {
-//		}
+			continue;
+		}
 
-//		fsp.fini();
+		if (!add_trace_event(instr1, instr2, time1, time2, fake_ev, true)) {
+			LOG << "add_trace_event failed" << std::endl;
+			return false;
+		}
+		++row_count_fake;
+	}
+
+	LOG << "trace duration: " << (curtime - time_trace_start) << " ticks" << std::endl;
+	LOG << "Inserted " << row_count << " trace events (+" << row_count_fake
+	    << " fake events) into the database" << std::endl;
+
+	// sanity checks
+	if (m_sanitychecks) {
+		std::stringstream ss;
+		bool all_ok = true;
+		MYSQL_RES *res;
+
+		// PC-based fault space rectangular, covered, and non-overlapping?
+		// (same for timing-based fault space?)
+
+		LOG << "Sanity check: EC overlaps ..." << std::flush;
+		ss <<
+			"SELECT t1.variant_id\n" //, v.variant, v.benchmark, t1.instr1, t1.instr2, t1.data_address, t2.instr1, t2.instr2
+			"FROM trace t1\n"
+			"JOIN variant v\n"
+			"  ON v.id = t1.variant_id\n"
+			"JOIN trace t2\n"
+			"  ON t1.variant_id = t2.variant_id AND t1.data_address = t2.data_address\n"
+			" AND (t1.instr1 != t2.instr1 OR t2.instr2 != t2.instr2)\n"
+			" AND ((t1.instr1 >= t2.instr1 AND t1.instr1 <= t2.instr2)\n"
+			"  OR (t1.instr2 >= t2.instr1 AND t1.instr2 <= t2.instr2)\n"
+			"  OR (t1.instr1 < t2.instr1 AND t1.instr2 > t2.instr2))\n"
+			"WHERE t1.variant_id = " << m_variant_id;
+
+		res = db->query(ss.str().c_str(), true);
+		ss.str("");
+		ss.clear();
+		if (res && mysql_num_rows(res) == 0) {
+			std::cout << " OK" << std::endl;
+		} else {
+			std::cout << " FAILED: not all ECs are disjoint" << std::endl;
+			// TODO: show a list of overlapping ECs?
+			all_ok = false;
+		}
+
+		LOG << "Sanity check: FS row sum = total width ..." << std::flush;
+		ss <<
+			"SELECT t.variant_id, t.data_address,\n"
+			" (SELECT (MAX(t2.instr2) - MIN(t2.instr1) + 1)\n"
+			"  FROM trace t2\n"
+			"  WHERE t2.variant_id = t.variant_id)\n"
+			" -\n"
+			" (SELECT SUM(t3.instr2 - t3.instr1 + 1)\n"
+			"  FROM trace t3\n"
+			"  WHERE t3.variant_id = t.variant_id AND t3.data_address = t.data_address)\n"
+			" AS diff\n"
+			"FROM trace t\n"
+			"WHERE t.variant_id = " << m_variant_id << "\n"
+			"GROUP BY t.variant_id, t.data_address\n"
+			"HAVING diff != 0\n"
+			"ORDER BY t.data_address\n";
+
+		res = db->query(ss.str().c_str(), true);
+		ss.str("");
+		ss.clear();
+		if (res && mysql_num_rows(res) == 0) {
+			std::cout << " OK" << std::endl;
+		} else {
+			std::cout << " FAILED: MAX(instr2)-MIN(instr1)+1 == SUM(instr2-instr1+1) not true for all fault-space rows" << std::endl;
+			// TODO: show a list of failing data_addresses?
+			all_ok = false;
+		}
+
+		LOG << "Sanity check: Global min/max = FS row local min/max ..." << std::flush;
+		ss <<
+			"SELECT t.variant_id, local.data_address, global.min_instr, global.max_instr, local.min_instr, local.max_instr\n"
+			"FROM trace t\n"
+			"JOIN\n"
+			" (SELECT t2.variant_id, MIN(instr1) AS min_instr, MAX(instr2) AS max_instr\n"
+			"  FROM trace t2\n"
+			"  GROUP BY t2.variant_id) AS global\n"
+			" ON global.variant_id = t.variant_id\n"
+			"JOIN\n"
+			" (SELECT variant_id, data_address, MIN(instr1) AS min_instr, MAX(instr2) AS max_instr\n"
+			"  FROM trace t3\n"
+			"  GROUP BY t3.variant_id, t3.data_address) AS local\n"
+			" ON local.variant_id = t.variant_id\n"
+			"AND (local.min_instr != global.min_instr\n"
+			"  OR local.max_instr != global.max_instr)\n"
+			"WHERE t.variant_id = " << m_variant_id << "\n"
+			"GROUP BY t.variant_id, local.data_address\n";
+
+		res = db->query(ss.str().c_str(), true);
+		ss.str("");
+		ss.clear();
+		if (res && mysql_num_rows(res) == 0) {
+			std::cout << " OK" << std::endl;
+		} else {
+			std::cout << " FAILED: global MIN(instr1)/MAX(instr2) != row-local MIN(instr1)/MAX(instr2)" << std::endl;
+			// TODO: show a list of failing data_addresses and global min/max?
+			all_ok = false;
+		}
+
+		if (!all_ok) {
+			return false;
+		}
+	}
 
 	return true;
 }
-

tools/import-trace/Importer.hpp

@@ -7,30 +7,39 @@
 #include "util/ElfReader.hpp"
 #include "sal/SALConfig.hpp"
 #include "util/Database.hpp"
+#include "util/MemoryMap.hpp"
 #include "comm/TracePlugin.pb.h"
 
-
 class Importer {
 protected:
 	int m_variant_id;
 	fail::ElfReader *m_elf;
+	fail::MemoryMap *m_mm;
+	char m_faultspace_rightmargin;
+	bool m_sanitychecks;
 	fail::Database *db;
 
 public:
-	typedef unsigned instruction_count_t;
+	typedef unsigned instruction_count_t; //!< not big enough for some benchmarks
 
+	Importer() : m_sanitychecks(false) {}
 	bool init(const std::string &variant, const std::string &benchmark, fail::Database *db);
 
 	virtual bool create_database() = 0;
 	virtual bool copy_to_database(fail::ProtoIStream &ps);
 	virtual bool clear_database();
 	virtual bool add_trace_event(instruction_count_t begin, instruction_count_t end,
+								 fail::simtime_t time_begin, fail::simtime_t time_end,
 								 const Trace_Event &event, bool is_fake = false) = 0;
 
-	void set_elf_file(fail::ElfReader *elf) {m_elf = elf;};
+	void set_elf_file(fail::ElfReader *elf) { m_elf = elf; }
+	void set_memorymap(fail::MemoryMap *mm) { m_mm = mm; }
+	void set_faultspace_rightmargin(char accesstype) { m_faultspace_rightmargin = accesstype; }
+	void set_sanitychecks(bool enabled) { m_sanitychecks = enabled; }
 protected:
 private:
-	typedef std::map<fail::address_t, instruction_count_t> AddrLastaccessMap;
+	struct leftmargin_info_t { instruction_count_t dyninstr; fail::simtime_t time; };
+	typedef std::map<fail::address_t, leftmargin_info_t> AddrLastaccessMap;
 };
 
 #endif

tools/import-trace/main.cc

@@ -3,6 +3,7 @@
 #include "util/CommandLine.hpp"
 #include "util/Database.hpp"
 #include "util/ElfReader.hpp"
+#include "util/MemoryMap.hpp"
 #include "util/gzstream/gzstream.h"
 #include "util/Logger.hpp"
 #include <fstream>
@@ -49,31 +50,61 @@ int main(int argc, char *argv[]) {
 	std::string trace_file, username, hostname, database, benchmark;
 	std::string variant, importer_args;
 	ElfReader *elf_file = 0;
+	MemoryMap *memorymap = 0;
 
 	// Manually fill the command line option parser
 	CommandLine &cmd = CommandLine::Inst();
 	for (int i = 1; i < argc; ++i)
 		cmd.add_args(argv[i]);
 
-	CommandLine::option_handle IGNORE = cmd.addOption("", "", Arg::None, "USAGE: import-trace [options]");
+	cmd.addOption("", "", Arg::None, "USAGE: import-trace [options]");
-	CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h/--help\t Print usage and exit");
+	CommandLine::option_handle HELP =
-	CommandLine::option_handle TRACE_FILE	= cmd.addOption("t", "trace-file", Arg::Required,
+		cmd.addOption("h", "help", Arg::None, "-h/--help \tPrint usage and exit");
+	CommandLine::option_handle TRACE_FILE =
+		cmd.addOption("t", "trace-file", Arg::Required,
 			"-t/--trace-file \tFile to load the execution trace from\n");
 
 	// setup the database command line options
 	Database::cmdline_setup();
 
-	CommandLine::option_handle VARIANT	 = cmd.addOption("v", "variant", Arg::Required,
+	CommandLine::option_handle VARIANT =
+		cmd.addOption("v", "variant", Arg::Required,
 			"-v/--variant \tVariant label (default: \"none\")");
-	CommandLine::option_handle BENCHMARK = cmd.addOption("b", "benchmark", Arg::Required,
+	CommandLine::option_handle BENCHMARK =
+		cmd.addOption("b", "benchmark", Arg::Required,
 			"-b/--benchmark \tBenchmark label (default: \"none\")\n");
-	CommandLine::option_handle IMPORTER	  = cmd.addOption("i", "importer", Arg::Required,
+	CommandLine::option_handle IMPORTER =
+		cmd.addOption("i", "importer", Arg::Required,
 			"-i/--importer \tWhich import method to use (default: BasicImporter)");
-	CommandLine::option_handle IMPORTER_ARGS	  = cmd.addOption("I", "importer-args", Arg::Required,
+	CommandLine::option_handle IMPORTER_ARGS =
+		cmd.addOption("I", "importer-args", Arg::Required,
 			"-I/--importer-args \tWhich import method to use (default: "")");
-	CommandLine::option_handle ELF_FILE  = cmd.addOption("e", "elf-file", Arg::Required,
+	CommandLine::option_handle ELF_FILE =
+		cmd.addOption("e", "elf-file", Arg::Required,
 			"-e/--elf-file \tELF File (default: UNSET)");
+	CommandLine::option_handle MEMORYMAP =
+		cmd.addOption("m", "memorymap", Arg::Required,
+			"-m/--memorymap \tMemory map to intersect with trace (may be used more than once; default: UNSET)");
 
+	// variant 1: care (synthetic Rs)
+	// variant 2: don't care (synthetic Ws)
+	CommandLine::option_handle FAULTSPACE_RIGHTMARGIN =
+		cmd.addOption("", "faultspace-rightmargin", Arg::Required,
+			"--faultspace-rightmargin \tMemory access type (R or W) to "
+			"complete fault space at the right margin "
+			"(default: W -- don't care)");
+	// (don't) cutoff at first R
+	// (don't) cutoff at last R
+	//CommandLine::option_handle FAULTSPACE_CUTOFF =
+	//	cmd.addOption("", "faultspace-cutoff-end", Arg::Required,
+	//		"--faultspace-cutoff-end \tCut off fault space end (no, lastr) "
+	//		"(default: no)");
+
+	CommandLine::option_handle ENABLE_SANITYCHECKS =
+		cmd.addOption("", "enable-sanitychecks", Arg::None,
+			"--enable-sanitychecks \tEnable sanity checks "
+			"(in case something looks fishy)"
+			"(default: disabled)");
 
 	if (!cmd.parse()) {
 		std::cerr << "Error parsing arguments." << std::endl;
@@ -129,14 +160,40 @@ int main(int argc, char *argv[]) {
 	if (cmd[ELF_FILE].count() > 0) {
 		elf_file = new ElfReader(cmd[ELF_FILE].first()->arg);
 	}
+	importer->set_elf_file(elf_file);
 
+	if (cmd[MEMORYMAP].count() > 0) {
+		memorymap = new MemoryMap();
+		for (option::Option *o = cmd[MEMORYMAP]; o; o = o->next()) {
+			if (!memorymap->readFromFile(o->arg)) {
+				LOG << "failed to load memorymap " << o->arg << endl;
+			}
+		}
+	}
+	importer->set_memorymap(memorymap);
 
+	if (cmd[FAULTSPACE_RIGHTMARGIN].count() > 0) {
+		std::string rightmargin(cmd[FAULTSPACE_RIGHTMARGIN].first()->arg);
+		if (rightmargin == "W") {
+			importer->set_faultspace_rightmargin('W');
+		} else if (rightmargin == "R") {
+			importer->set_faultspace_rightmargin('R');
+		} else {
+			LOG << "unknown memory access type '" << rightmargin << "', using default" << endl;
+			importer->set_faultspace_rightmargin('W');
+		}
+	} else {
+		importer->set_faultspace_rightmargin('W');
+	}
+
+	if (cmd[ENABLE_SANITYCHECKS].count() > 0) {
+		importer->set_sanitychecks(true);
+	}
 
 	if (!importer->init(variant, benchmark, db)) {
 		LOG << "importer->init() failed" << endl;
 		exit(-1);
 	}
-	importer->set_elf_file(elf_file);
 
 	////////////////////////////////////////////////////////////////
 	// Do the actual import

tools/prune-trace/BasicPruner.cc

@@ -7,50 +7,60 @@ static fail::Logger LOG ("BasicPruner");
 bool BasicPruner::prune_all() {
 	std::stringstream ss;
 
-	ss << "INSERT INTO fsppilot (known_outcome, variant_id, instr1, instr2, data_address, fspmethod_id) "
+	/* When we are in basic-left mode we use the left boundary of the
-		  "SELECT 0, variant_id, instr1, instr2, data_address, " << m_method_id << " "
+	   equivalence interval. Since the current database scheme has no
+	   instr1_absolute, we set this to NULL in the basic-left mode. */
+	// FIXME "basic-left mode" doesn't make any sense; injections at instr1 and
+	// at instr2 are completely equivalent.
+	std::string injection_instr = this->use_instr1 ? "instr1" : "instr2";
+	std::string injection_instr_absolute = this->use_instr1 ? "NULL" : "instr2_absolute";
+
+	ss << "INSERT INTO fsppilot (known_outcome, variant_id, instr2, injection_instr, injection_instr_absolute, data_address, fspmethod_id) "
+		  "SELECT 0, variant_id, instr2, " << injection_instr << ", " << injection_instr_absolute << ", "
+		  "  data_address, " << m_method_id << " "
 		  "FROM trace "
-		  "WHERE variant_id = " << m_variant_id << " AND accesstype = 'R'";
+		  "WHERE variant_id IN (" << m_variant_id_query << ") AND accesstype = 'R'";
 	if (!db->query(ss.str().c_str())) return false; 
 	ss.str("");
 
 	int rows = db->affected_rows();
+
+	// for each variant:
+	MYSQL_RES *res = db->query(m_variant_id_query.c_str(), true);
+	MYSQL_ROW row;
+	while ((row = mysql_fetch_row(res))) {
 		// single entry for known outcome (write access)
-	ss << "INSERT INTO fsppilot (known_outcome, variant_id, instr1, instr2, data_address, fspmethod_id) "
+		ss << "INSERT INTO fsppilot (known_outcome, variant_id, instr2, injection_instr, injection_instr_absolute, data_address, fspmethod_id) "
-		  "SELECT 1, variant_id, instr1, instr2, data_address, " << m_method_id << " "
+			  "SELECT 1, variant_id, instr2, " << injection_instr << ", " << injection_instr_absolute << ", "
+			  "  data_address, " << m_method_id << " "
 			  "FROM trace "
-		  "WHERE variant_id = " << m_variant_id << " AND accesstype = 'W' "
+			  "WHERE variant_id = " << row[0] << " AND accesstype = 'W' "
 			  "LIMIT 1";
 		if (!db->query(ss.str().c_str())) return false;
 		ss.str("");
 		rows += db->affected_rows();
+	}
 
 	LOG << "created " << rows << " fsppilot entries" << std::endl;
 
-	/* When we are in basic-left mode we use the left boundary of the
+	ss << "INSERT INTO fspgroup (variant_id, instr2, data_address, fspmethod_id, pilot_id) "
-	   equivalence interval. Sine the current database scheme has no
+	   << "SELECT STRAIGHT_JOIN p.variant_id, p.instr2, p.data_address, p.fspmethod_id, p.id "
-	   instr2_absolute, we set this to NULL in the basic-left mode. */
-	std::string injection_instr = this->use_instr1 ? "t.instr1" : "t.instr2";
-	std::string injection_instr_absolute = this->use_instr1 ? "NULL" : "t.instr2_absolute";
-
-	ss << "INSERT INTO fspgroup (variant_id, injection_instr, injection_instr_absolute, "
-	   << "                      data_address, fspmethod_id, pilot_id) "
-	   << "SELECT p.variant_id, " << injection_instr << ", " << injection_instr_absolute << ", p.data_address, p.fspmethod_id, p.id "
 	   << "FROM fsppilot p "
 	   << "JOIN trace t ON t.variant_id = p.variant_id AND t.instr2 = p.instr2"
 	   << "            AND t.data_address = p.data_address "
-	   << "WHERE known_outcome = 0 AND p.fspmethod_id = " << m_method_id << " AND p.variant_id = " << m_variant_id;
+	   << "WHERE known_outcome = 0 AND p.fspmethod_id = " << m_method_id << " "
+	   << "AND p.variant_id IN (" << m_variant_id_query << ")";
 
 	if (!db->query(ss.str().c_str())) return false;
 	ss.str("");
 
 	rows = db->affected_rows();
-	ss << "INSERT INTO fspgroup (variant_id, injection_instr, injection_instr_absolute, data_address, fspmethod_id, pilot_id) "
+	ss << "INSERT INTO fspgroup (variant_id, instr2, data_address, fspmethod_id, pilot_id) "
-		"SELECT t.variant_id, "<< injection_instr << ", " << injection_instr_absolute <<", t.data_address, p.fspmethod_id, p.id "
+		"SELECT STRAIGHT_JOIN t.variant_id, t.instr2, t.data_address, p.fspmethod_id, p.id "
-		"FROM trace t "
+		"FROM fsppilot p "
-		"JOIN fsppilot p "
+		"JOIN trace t "
 		"ON t.variant_id = p.variant_id AND p.fspmethod_id = " << m_method_id << " AND p.known_outcome = 1 "
-		"WHERE t.variant_id = " << m_variant_id << " AND t.accesstype = 'W'";
+		"WHERE t.variant_id IN (" << m_variant_id_query << ") AND t.accesstype = 'W'";
 	if (!db->query(ss.str().c_str())) return false;
 	ss.str("");
 	rows += db->affected_rows();

tools/prune-trace/Pruner.cc

@@ -8,17 +8,42 @@ static Logger LOG ("Pruner");
 #include "Pruner.hpp"
 
 
-bool Pruner::init(const std::string &variant, const std::string &benchmark, Database *db) {
+bool Pruner::init(fail::Database *db,
+		const std::vector<std::string>& variants,
+		const std::vector<std::string>& variants_exclude,
+		const std::vector<std::string>& benchmarks,
+		const std::vector<std::string>& benchmarks_exclude)
+{
 	this->db = db;
-	if (!(m_variant_id = db->get_variant_id(variant, benchmark))) {
+	std::stringstream ss;
-		return false;
+
+	// FIXME string escaping
+	ss << "SELECT id FROM variant WHERE ";
+	for (std::vector<std::string>::const_iterator it = variants.begin();
+	     it != variants.end(); ++it) {
+		ss << "variant LIKE '" << *it << "' AND ";
 	}
+	for (std::vector<std::string>::const_iterator it = variants_exclude.begin();
+	     it != variants_exclude.end(); ++it) {
+		ss << "variant NOT LIKE '" << *it << "' AND ";
+	}
+	for (std::vector<std::string>::const_iterator it = benchmarks.begin();
+	     it != benchmarks.end(); ++it) {
+		ss << "benchmark LIKE '" << *it << "' AND ";
+	}
+	for (std::vector<std::string>::const_iterator it = benchmarks_exclude.begin();
+	     it != benchmarks_exclude.end(); ++it) {
+		ss << "benchmark NOT LIKE '" << *it << "' AND ";
+	}
+	// dummy terminator to avoid special cases in query construction above
+	ss << "1";
+	m_variant_id_query = ss.str();
+
 	if (!(m_method_id = db->get_fspmethod_id(method_name()))) {
 		return false;
 	}
-	LOG << "Pruning variant "
+
-	    << variant << "/" << benchmark      << " (ID: " << m_variant_id << ")"
+	LOG << "Pruning with method " << method_name() << " (ID: " << m_method_id << ")"
-	    << " with method " << method_name() << " (ID: " << m_method_id << ")"
 	    << std::endl;
 	return true;
 }
@@ -28,37 +53,37 @@ bool Pruner::create_database() {
 	    "  id int(11) NOT NULL AUTO_INCREMENT,"
 	    "  known_outcome tinyint(4) NOT NULL,"
 	    "  variant_id int(11) NOT NULL,"
-	    "  instr1 int(10) unsigned NOT NULL,"
 	    "  instr2 int(10) unsigned NOT NULL,"
+	    "  injection_instr int(10) unsigned NOT NULL,"
+	    "  injection_instr_absolute int(10) unsigned,"
 	    "  data_address int(10) unsigned NOT NULL,"
 	    "  fspmethod_id int(11) NOT NULL,"
 	    "  PRIMARY KEY (id),"
-	    "  KEY fspmethod_id (fspmethod_id,variant_id,instr1,instr2,data_address)"
+	    "  KEY fspmethod_id (fspmethod_id,variant_id,instr2,data_address)"
 	    ") engine=MyISAM ";
 	bool success = (bool) db->query(create_statement.c_str());
 	if (!success) return false;
 
 	create_statement = "CREATE TABLE IF NOT EXISTS fspgroup ("
 	    "  variant_id      int(11) NOT NULL,"
-	    "  injection_instr int(10) unsigned NOT NULL,"
+	    "  instr2          int(11) unsigned NOT NULL,"
-	    "  injection_instr_absolute int(10) unsigned,"
 	    "  data_address    int(10) unsigned NOT NULL,"
 	    "  fspmethod_id    int(11) NOT NULL,"
 	    "  pilot_id        int(11) NOT NULL,"
-	    "  PRIMARY KEY (variant_id, injection_instr, data_address, fspmethod_id, pilot_id),"
+	    "  PRIMARY KEY (variant_id, instr2, data_address, fspmethod_id, pilot_id),"
-	    "  KEY joinresults (pilot_id,fspmethod_id))";
+	    "  KEY joinresults (pilot_id,fspmethod_id)) engine=MyISAM";
 
 	return db->query(create_statement.c_str());
 }
 
 bool Pruner::clear_database() {
 	std::stringstream ss;
-	ss << "DELETE FROM fsppilot WHERE variant_id = " << m_variant_id << " AND fspmethod_id = " << m_method_id;
+	ss << "DELETE FROM fsppilot WHERE variant_id IN (" << m_variant_id_query << ")";
 	bool ret = (bool) db->query(ss.str().c_str());
 	LOG << "deleted " << db->affected_rows() << " rows from fsppilot table" << std::endl;
 	ss.str("");
 
-	ss << "DELETE FROM fspgroup WHERE variant_id = " << m_variant_id << " AND fspmethod_id = " << m_method_id;
+	ss << "DELETE FROM fspgroup WHERE variant_id IN (" << m_variant_id_query << ")";
 	ret = ret && (bool) db->query(ss.str().c_str());
 	LOG << "deleted " << db->affected_rows() << " rows from fspgroup table" << std::endl;
 

tools/prune-trace/Pruner.hpp

@@ -1,17 +1,22 @@
 #ifndef __PRUNER_H__
 #define __PRUNER_H__
 
+#include <vector>
+#include <string>
 #include "util/Database.hpp"
 
 class Pruner {
 protected:
-	int m_variant_id;
 	int m_method_id;
+	std::string m_variant_id_query;
 	fail::Database *db;
 
 public:
-	bool init(const std::string &variant, const std::string &benchmark,
+	bool init(fail::Database *db,
-              fail::Database *sql);
+		const std::vector<std::string>& variants,
+		const std::vector<std::string>& variants_exclude,
+		const std::vector<std::string>& benchmarks,
+		const std::vector<std::string>& benchmarks_exclude);
 
 	virtual std::string method_name() = 0;
 

tools/prune-trace/main.cc

@@ -13,24 +13,36 @@ using std::endl;
 #include "BasicPruner.hpp"
 
 int main(int argc, char *argv[]) {
-	std::string username, hostname, database, benchmark, variant;
+	std::string username, hostname, database;
 
 	// Manually fill the command line option parser
 	CommandLine &cmd = CommandLine::Inst();
 	for (int i = 1; i < argc; ++i)
 		cmd.add_args(argv[i]);
 
-	CommandLine::option_handle IGNORE = cmd.addOption("", "", Arg::None, "USAGE: import-trace [options]");
+	cmd.addOption("", "", Arg::None, "USAGE: import-trace [options]");
 	CommandLine::option_handle HELP = cmd.addOption("h", "help", Arg::None, "-h,--help \tPrint usage and exit");
 
 	Database::cmdline_setup();
 
-	CommandLine::option_handle VARIANT	 = cmd.addOption("v", "variant", Arg::Required,
+	CommandLine::option_handle VARIANT =
-	                                                     "-v/--variant\t Variant label (default: \"none\")");
+		cmd.addOption("v", "variant", Arg::Required,
-	CommandLine::option_handle BENCHMARK = cmd.addOption("b", "benchmark", Arg::Required,
+			"-v/--variant \tVariant label (default: \"none\"; use % and _ as wildcard characters; may be used more than once)");
-														 "-b/--benchmark\t Benchmark label (default: \"none\")\n");
+	CommandLine::option_handle VARIANT_EXCLUDE =
-	CommandLine::option_handle PRUNER	 = cmd.addOption("p", "prune-method", Arg::Required,
+		cmd.addOption("", "variant-exclude", Arg::Required,
+			"--variant-exclude \tVariant to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)");
+	CommandLine::option_handle BENCHMARK =
+		cmd.addOption("b", "benchmark", Arg::Required,
+			"-b/--benchmark \tBenchmark label (default: \"none\"; use % and _ as wildcard characters; may be used more than once)");
+	CommandLine::option_handle BENCHMARK_EXCLUDE =
+		cmd.addOption("", "benchmark-exclude", Arg::Required,
+			"--benchmark-exclude \tBenchmark to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)");
+	CommandLine::option_handle PRUNER =
+		cmd.addOption("p", "prune-method", Arg::Required,
 			"-p/--prune-method \tWhich import method to use (default: basic)");
+	CommandLine::option_handle NO_DELETE =
+		cmd.addOption("", "no-delete", Arg::None,
+			"--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE");
 
 	if(!cmd.parse()) {
 		std::cerr << "Error parsing arguments." << std::endl;
@@ -64,17 +76,36 @@ int main(int argc, char *argv[]) {
 
 	Database *db = Database::cmdline_connect();
 
-	if (cmd[VARIANT].count() > 0)
+	std::vector<std::string> variants, benchmarks, variants_exclude, benchmarks_exclude;
-		variant = std::string(cmd[VARIANT].first()->arg);
+	if (cmd[VARIANT].count() > 0) {
-	else
+		for (option::Option *o = cmd[VARIANT]; o; o = o->next()) {
-		variant = "none";
+			variants.push_back(std::string(o->arg));
+		}
+	} else {
+		variants.push_back(std::string("none"));
+	}
 
-	if (cmd[BENCHMARK].count() > 0)
+	if (cmd[VARIANT_EXCLUDE].count() > 0) {
-		benchmark = std::string(cmd[BENCHMARK].first()->arg);
+		for (option::Option *o = cmd[VARIANT_EXCLUDE]; o; o = o->next()) {
-	else
+			variants_exclude.push_back(std::string(o->arg));
-		benchmark = "none";
+		}
+	}
 
-	if (!pruner->init(variant, benchmark, db)) {
+	if (cmd[BENCHMARK].count() > 0) {
+		for (option::Option *o = cmd[BENCHMARK]; o; o = o->next()) {
+			benchmarks.push_back(std::string(o->arg));
+		}
+	} else {
+		benchmarks.push_back(std::string("none"));
+	}
+
+	if (cmd[BENCHMARK_EXCLUDE].count() > 0) {
+		for (option::Option *o = cmd[BENCHMARK_EXCLUDE]; o; o = o->next()) {
+			benchmarks_exclude.push_back(std::string(o->arg));
+		}
+	}
+
+	if (!pruner->init(db, variants, variants_exclude, benchmarks, benchmarks_exclude)) {
 		LOG << "pruner->init() failed" << endl;
 		exit(-1);
 	}
@@ -87,7 +118,7 @@ int main(int argc, char *argv[]) {
 		exit(-1);
 	}
 
-	if (!pruner->clear_database()) {
+	if (cmd[NO_DELETE].count() == 0 && !pruner->clear_database()) {
 		LOG << "clear_database() failed" << endl;
 		exit(-1);
 	}