Merge branch 'sampling'

Conflicts:
	src/core/cpn/DatabaseCampaign.cc

Change-Id: Ic11d9ce26546bccba11768383a8fda6a3458530f

src/core/cpn/DatabaseCampaign.cc

@@ -1,3 +1,5 @@
+#include <vector>
+
 #include "DatabaseCampaign.hpp"
 #include "cpn/CampaignManager.hpp"
 #include "util/CommandLine.hpp"
@@ -28,12 +30,21 @@ bool DatabaseCampaign::run() {
 	   line interface */
 	if (!cb_commandline_init()) return false;
 
-	CommandLine::option_handle VARIANT	 = cmd.addOption("v", "variant", Arg::Required,
+	CommandLine::option_handle VARIANT =
-														 "-v/--variant \tVariant label (default: \"none\"; use % and _ as wildcard characters)");
+		cmd.addOption("v", "variant", Arg::Required,
-	CommandLine::option_handle BENCHMARK = cmd.addOption("b", "benchmark", Arg::Required,
+			"-v/--variant \tVariant label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)");
-														 "-b/--benchmark \tBenchmark label (default: \"none\"; use % and _ as wildcard characters)\n");
+	CommandLine::option_handle VARIANT_EXCLUDE =
-	CommandLine::option_handle PRUNER	 = cmd.addOption("p", "prune-method", Arg::Required,
+		cmd.addOption("", "variant-exclude", Arg::Required,
-														 "-p/--prune-method \tWhich import method to use (default: basic)");
+			"--variant-exclude \tVariant to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)");
+	CommandLine::option_handle BENCHMARK =
+		cmd.addOption("b", "benchmark", Arg::Required,
+			"-b/--benchmark \tBenchmark label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)");
+	CommandLine::option_handle BENCHMARK_EXCLUDE =
+		cmd.addOption("", "benchmark-exclude", Arg::Required,
+			"--benchmark-exclude \tBenchmark to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)");
+	CommandLine::option_handle PRUNER =
+		cmd.addOption("p", "prune-method", Arg::Required,
+			"-p/--prune-method \tWhich import method to use (default: basic)");
 
 	if (!cmd.parse()) {
 		log_send << "Error parsing arguments." << std::endl;
@@ -45,25 +56,49 @@ bool DatabaseCampaign::run() {
 		exit(0);
 	}
 
-	std::string variant, benchmark, pruner;
+	std::vector<std::string> variants, benchmarks, variants_exclude, benchmarks_exclude;
+	if (cmd[VARIANT]) {
+		for (option::Option *o = cmd[VARIANT]; o; o = o->next()) {
+			variants.push_back(std::string(o->arg));
+		}
+	}
 
-	if (cmd[VARIANT].count() > 0)
+	if (cmd[VARIANT_EXCLUDE]) {
-		variant = std::string(cmd[VARIANT].first()->arg);
+		for (option::Option *o = cmd[VARIANT_EXCLUDE]; o; o = o->next()) {
-	else
+			variants_exclude.push_back(std::string(o->arg));
-		variant = "none";
+		}
+	}
 
-	if (cmd[BENCHMARK].count() > 0)
+	// fallback
-		benchmark = std::string(cmd[BENCHMARK].first()->arg);
+	if (variants.size() == 0) {
-	else
+		variants.push_back("%");
-		benchmark = "none";
+	}
 
-	if (cmd[PRUNER].count() > 0)
+	if (cmd[BENCHMARK]) {
+		for (option::Option *o = cmd[BENCHMARK]; o; o = o->next()) {
+			benchmarks.push_back(std::string(o->arg));
+		}
+	}
+
+	if (cmd[BENCHMARK_EXCLUDE]) {
+		for (option::Option *o = cmd[BENCHMARK_EXCLUDE]; o; o = o->next()) {
+			benchmarks_exclude.push_back(std::string(o->arg));
+		}
+	}
+
+	// fallback
+	if (benchmarks.size() == 0) {
+		benchmarks.push_back("%");
+	}
+
+	std::string pruner;
+	if (cmd[PRUNER]) {
 		pruner = std::string(cmd[PRUNER].first()->arg);
-	else
+	} else {
 		pruner = "basic";
+	}
 
 	db = Database::cmdline_connect();
-	log_send << "Variant to use " << variant << "/" << benchmark << std::endl;
 	fspmethod_id = db->get_fspmethod_id(pruner);
 	log_send << "Pruner to use " << pruner << " (ID: " << fspmethod_id << ")" << std::endl;
 
@@ -79,15 +114,14 @@ bool DatabaseCampaign::run() {
 	boost::thread collect_thread(&DatabaseCampaign::collect_result_thread, this);
 #endif
 
-	std::vector<Database::Variant> variants = db->get_variants(variant, benchmark);
+	std::vector<Database::Variant> variantlist =
-
+		db->get_variants(variants, variants_exclude, benchmarks, benchmarks_exclude);
 
 	// Which Pilots were already processed?
-	load_completed_pilots(variants);
+	load_completed_pilots(variantlist);
-
-	for (std::vector<Database::Variant>::const_iterator it = variants.begin();
-		 it != variants.end(); ++it) {
 
+	for (std::vector<Database::Variant>::const_iterator it = variantlist.begin();
+		 it != variantlist.end(); ++it) {
 		// Push all other variants to the queue
 		if (!run_variant(*it)) {
 			log_send << "run_variant failed for " << it->variant << "/" << it->benchmark <<std::endl;

tools/prune-trace/BasicPruner.hpp

@@ -9,6 +9,20 @@ public:
 	BasicPruner(bool use_instr1 = false) : use_instr1(use_instr1) {}
 	virtual std::string method_name() { return std::string("basic") + (use_instr1 ? "-left" : ""); }
 	virtual bool prune_all();
+
+	void getAliases(std::deque<std::string> *aliases) {
+		aliases->push_back("BasicPruner");
+		aliases->push_back("basic");
+	}
+};
+
+class BasicPrunerLeft : public BasicPruner {
+public:
+	BasicPrunerLeft() : BasicPruner(true) {}
+	void getAliases(std::deque<std::string> *aliases) {
+		aliases->push_back("BasicPrunerLeft");
+		aliases->push_back("basic-left");
+	}
 };
 
 #endif

tools/prune-trace/FESamplingPruner.hpp

@@ -24,6 +24,11 @@ public:
 	virtual bool commandline_init();
 	virtual bool prune_all();
 
+	void getAliases(std::deque<std::string> *aliases) {
+		aliases->push_back("FESamplingPruner");
+		aliases->push_back("sampling");
+	}
+
 private:
 	bool sampling_prune(const fail::Database::Variant& variant);
 };

tools/prune-trace/Pruner.cc

@@ -8,38 +8,72 @@ static Logger LOG ("Pruner");
 #include "Pruner.hpp"
 
 
-bool Pruner::init(fail::Database *db,
+bool Pruner::init(
 		const std::vector<std::string>& variants,
 		const std::vector<std::string>& variants_exclude,
 		const std::vector<std::string>& benchmarks,
-		const std::vector<std::string>& benchmarks_exclude)
+		const std::vector<std::string>& benchmarks_exclude,
+		bool overwrite)
 {
-	this->db = db;
-
 	m_variants = db->get_variants(
 		variants, variants_exclude,
 		benchmarks, benchmarks_exclude);
+
+	if (!(m_method_id = db->get_fspmethod_id(method_name()))) {
+		return false;
+	}
+	LOG << "Pruning with method " << method_name() << " (ID: " << m_method_id << ")"
+	    << std::endl;
+
+	// make sure we only prune variants that haven't been pruned previously
+	// (unless we run with --overwrite)
+	if (!overwrite) {
+		for (std::vector<fail::Database::Variant>::iterator it = m_variants.begin();
+			it != m_variants.end(); ) {
+			std::stringstream ss;
+			MYSQL_RES *res;
+			ss << "(SELECT variant_id FROM fsppilot WHERE "
+			   << " variant_id = " << it->id << " AND "
+			   << " fspmethod_id = " << m_method_id
+			   << " LIMIT 1)"
+			   << " UNION ALL "
+			   << "(SELECT variant_id FROM fspgroup WHERE "
+			   << " variant_id = " << it->id << " AND "
+			   << " fspmethod_id = " << m_method_id
+			   << " LIMIT 1)";
+			if (!(res = db->query(ss.str().c_str(), true))) {
+				return false;
+			}
+			if (mysql_num_rows(res) > 0) {
+				// skip this variant
+				LOG << "skipping " << it->variant << "/" << it->benchmark
+				    << " due to existing pruning data (use --overwrite to skip this check)"
+				    << std::endl;
+				it = m_variants.erase(it);
+			} else {
+				++it;
+			}
+		}
+	}
+
+	// any variants left?
 	if (m_variants.size() == 0) {
 		LOG << "no variants found, nothing to do" << std::endl;
 		return false;
 	}
 
-	std::stringstream ss;
+	// construct comma-separated list usable in SQL "IN (...)"
+	std::stringstream commalist;
 	for (std::vector<fail::Database::Variant>::const_iterator it = m_variants.begin();
 		it != m_variants.end(); ++it) {
+
 		if (it != m_variants.begin()) {
-			ss << ",";
+			commalist << ",";
 		}
-		ss << it->id;
+		commalist << it->id;
 	}
-	m_variants_sql = ss.str();
+	m_variants_sql = commalist.str();
 
-	if (!(m_method_id = db->get_fspmethod_id(method_name()))) {
-		return false;
-	}
-
-	LOG << "Pruning with method " << method_name() << " (ID: " << m_method_id << ")"
-	    << std::endl;
 	return true;
 }
 

tools/prune-trace/Pruner.hpp

@@ -4,8 +4,9 @@
 #include <vector>
 #include <string>
 #include "util/Database.hpp"
+#include "util/AliasedRegisterable.hpp"
 
-class Pruner {
+class Pruner : public fail::AliasedRegisterable {
 protected:
 	int m_method_id;
 	fail::Database *db;
@@ -13,11 +14,14 @@ protected:
 	std::string m_variants_sql;
 
 public:
-	bool init(fail::Database *db,
+	void set_db(fail::Database *db) { this->db = db; }
+
+	bool init(
 		const std::vector<std::string>& variants,
 		const std::vector<std::string>& variants_exclude,
 		const std::vector<std::string>& benchmarks,
-		const std::vector<std::string>& benchmarks_exclude);
+		const std::vector<std::string>& benchmarks_exclude,
+		bool overwrite);
 
 	/**
 	 * Callback function that can be used to add command line options

tools/prune-trace/main.cc

@@ -4,6 +4,8 @@
 
 #include "util/CommandLine.hpp"
 #include "util/Logger.hpp"
+#include "util/AliasedRegistry.hpp"
+
 static fail::Logger LOG("prune-trace", true);
 
 using namespace fail;
@@ -16,6 +18,17 @@ using std::endl;
 int main(int argc, char *argv[]) {
 	std::string username, hostname, database;
 
+	// register possible Pruners
+	AliasedRegistry registry;
+	BasicPruner basicpruner;
+	registry.add(&basicpruner);
+	BasicPrunerLeft basicprunerleft;
+	registry.add(&basicprunerleft);
+	FESamplingPruner fesamplingpruner;
+	registry.add(&fesamplingpruner);
+
+	std::string pruners = registry.getPrimeAliasesCSV();
+
 	// Manually fill the command line option parser
 	CommandLine &cmd = CommandLine::Inst();
 	for (int i = 1; i < argc; ++i) {
@@ -29,22 +42,25 @@ int main(int argc, char *argv[]) {
 
 	CommandLine::option_handle VARIANT =
 		cmd.addOption("v", "variant", Arg::Required,
-			"-v/--variant \tVariant label (default: \"none\"; use % and _ as wildcard characters; may be used more than once)");
+			"-v/--variant \tVariant label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)");
 	CommandLine::option_handle VARIANT_EXCLUDE =
 		cmd.addOption("", "variant-exclude", Arg::Required,
 			"--variant-exclude \tVariant to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)");
 	CommandLine::option_handle BENCHMARK =
 		cmd.addOption("b", "benchmark", Arg::Required,
-			"-b/--benchmark \tBenchmark label (default: \"none\"; use % and _ as wildcard characters; may be used more than once)");
+			"-b/--benchmark \tBenchmark label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)");
 	CommandLine::option_handle BENCHMARK_EXCLUDE =
 		cmd.addOption("", "benchmark-exclude", Arg::Required,
 			"--benchmark-exclude \tBenchmark to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)");
+	std::string pruner_help = "-p/--prune-method \tWhich pruning method to use (default: basic); available pruning methods: " + pruners;
 	CommandLine::option_handle PRUNER =
-		cmd.addOption("p", "prune-method", Arg::Required,
+		cmd.addOption("p", "prune-method", Arg::Required, pruner_help);
-			"-p/--prune-method \tWhich import method to use (default: basic)");
 	CommandLine::option_handle NO_DELETE =
 		cmd.addOption("", "no-delete", Arg::None,
 			"--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE");
+	CommandLine::option_handle OVERWRITE =
+		cmd.addOption("", "overwrite", Arg::None,
+			"--overwrite \tOverwrite already existing pruning data (the default is to skip variants with existing entries)");
 
 	if (!cmd.parse()) {
 		std::cerr << "Error parsing arguments." << std::endl;
@@ -52,6 +68,20 @@ int main(int argc, char *argv[]) {
 	}
 
 	Pruner *pruner;
+	std::string pruner_name = "BasicPruner";
+	if (cmd[PRUNER]) {
+		pruner_name = cmd[PRUNER].first()->arg;
+	}
+
+	// try and get the according pruner object; die on failure
+	if ((pruner = (Pruner *)registry.get(pruner_name)) == 0) {
+		if (pruner_name != "?" ) {
+			std::cerr << "Unknown import method: " << pruner_name << std::endl;
+		}
+		std::cerr << "Available import methods: " << pruners << std::endl;
+		exit(-1);
+	}
+
 	if (cmd[PRUNER]) {
 		std::string imp(cmd[PRUNER].first()->arg);
 		if (imp == "BasicPruner" || imp == "basic") {
@@ -88,6 +118,7 @@ int main(int argc, char *argv[]) {
 	}
 
 	Database *db = Database::cmdline_connect();
+	pruner->set_db(db);
 
 	std::vector<std::string> variants, benchmarks, variants_exclude, benchmarks_exclude;
 	if (cmd[VARIANT]) {
@@ -103,8 +134,8 @@ int main(int argc, char *argv[]) {
 	}
 
 	// fallback
-	if (variants.size() == 0 && variants_exclude.size() == 0) {
+	if (variants.size() == 0) {
-		variants.push_back(std::string("none"));
+		variants.push_back("%");
 	}
 
 	if (cmd[BENCHMARK]) {
@@ -120,11 +151,16 @@ int main(int argc, char *argv[]) {
 	}
 
 	// fallback
-	if (benchmarks.size() == 0 && benchmarks_exclude.size() == 0) {
+	if (benchmarks.size() == 0) {
-		benchmarks.push_back(std::string("none"));
+		benchmarks.push_back("%");
 	}
 
-	if (!pruner->init(db, variants, variants_exclude, benchmarks, benchmarks_exclude)) {
+	if (!pruner->create_database()) {
+		LOG << "pruner->create_database() failed" << endl;
+		exit(-1);
+	}
+
+	if (!pruner->init(variants, variants_exclude, benchmarks, benchmarks_exclude, cmd[OVERWRITE])) {
 		LOG << "pruner->init() failed" << endl;
 		exit(-1);
 	}
@@ -132,12 +168,7 @@ int main(int argc, char *argv[]) {
 	////////////////////////////////////////////////////////////////
 	// Do the actual import
 	////////////////////////////////////////////////////////////////
-	if (!pruner->create_database()) {
+	if (!cmd[NO_DELETE] && cmd[OVERWRITE] && !pruner->clear_database()) {
-		LOG << "create_database() failed" << endl;
-		exit(-1);
-	}
-
-	if (!cmd[NO_DELETE] && !pruner->clear_database()) {
 		LOG << "clear_database() failed" << endl;
 		exit(-1);
 	}