From 0e1ed1feab5fbd04b6e9683962aa7f8a030d57e2 Mon Sep 17 00:00:00 2001 From: Horst Schirmeier Date: Fri, 9 May 2014 00:46:30 +0200 Subject: [PATCH 1/4] prune-trace+DBCampaign: default to variant/benchmark % If no --variant / --benchmark is specified, it's more reasonable to prune or run *all* variants/benchmarks (using the wildcard "%") instead of defaulting to "none"/"none". The trivial case with only one single variant/benchmark (which may still be "none"/"none" if import-trace's default is used) is still covered by this new default behavior. Change-Id: I0e9001137d5e052183dd74211e2edbcfab749528 --- src/core/cpn/DatabaseCampaign.cc | 8 ++++---- tools/prune-trace/main.cc | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/cpn/DatabaseCampaign.cc b/src/core/cpn/DatabaseCampaign.cc index 274fd0a9..a4d01f60 100644 --- a/src/core/cpn/DatabaseCampaign.cc +++ b/src/core/cpn/DatabaseCampaign.cc @@ -29,9 +29,9 @@ bool DatabaseCampaign::run() { if (!cb_commandline_init()) return false; CommandLine::option_handle VARIANT = cmd.addOption("v", "variant", Arg::Required, - "-v/--variant \tVariant label (default: \"none\"; use % and _ as wildcard characters)"); + "-v/--variant \tVariant label (default: \"%\"; use % and _ as wildcard characters)"); CommandLine::option_handle BENCHMARK = cmd.addOption("b", "benchmark", Arg::Required, - "-b/--benchmark \tBenchmark label (default: \"none\"; use % and _ as wildcard characters)\n"); + "-b/--benchmark \tBenchmark label (default: \"%\"; use % and _ as wildcard characters)\n"); CommandLine::option_handle PRUNER = cmd.addOption("p", "prune-method", Arg::Required, "-p/--prune-method \tWhich import method to use (default: basic)"); @@ -50,12 +50,12 @@ bool DatabaseCampaign::run() { if (cmd[VARIANT].count() > 0) variant = std::string(cmd[VARIANT].first()->arg); else - variant = "none"; + variant = "%"; if (cmd[BENCHMARK].count() > 0) benchmark = std::string(cmd[BENCHMARK].first()->arg); else - benchmark = "none"; + benchmark = "%"; if (cmd[PRUNER].count() > 0) pruner = std::string(cmd[PRUNER].first()->arg); diff --git a/tools/prune-trace/main.cc b/tools/prune-trace/main.cc index 55d4c47a..d276bb5f 100644 --- a/tools/prune-trace/main.cc +++ b/tools/prune-trace/main.cc @@ -29,13 +29,13 @@ int main(int argc, char *argv[]) { CommandLine::option_handle VARIANT = cmd.addOption("v", "variant", Arg::Required, - "-v/--variant \tVariant label (default: \"none\"; use % and _ as wildcard characters; may be used more than once)"); + "-v/--variant \tVariant label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)"); CommandLine::option_handle VARIANT_EXCLUDE = cmd.addOption("", "variant-exclude", Arg::Required, "--variant-exclude \tVariant to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)"); CommandLine::option_handle BENCHMARK = cmd.addOption("b", "benchmark", Arg::Required, - "-b/--benchmark \tBenchmark label (default: \"none\"; use % and _ as wildcard characters; may be used more than once)"); + "-b/--benchmark \tBenchmark label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)"); CommandLine::option_handle BENCHMARK_EXCLUDE = cmd.addOption("", "benchmark-exclude", Arg::Required, "--benchmark-exclude \tBenchmark to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)"); @@ -103,8 +103,8 @@ int main(int argc, char *argv[]) { } // fallback - if (variants.size() == 0 && variants_exclude.size() == 0) { - variants.push_back(std::string("none")); + if (variants.size() == 0) { + variants.push_back("%"); } if (cmd[BENCHMARK]) { @@ -120,8 +120,8 @@ int main(int argc, char *argv[]) { } // fallback - if (benchmarks.size() == 0 && benchmarks_exclude.size() == 0) { - benchmarks.push_back(std::string("none")); + if (benchmarks.size() == 0) { + benchmarks.push_back("%"); } if (!pruner->init(db, variants, variants_exclude, benchmarks, benchmarks_exclude)) { From 2100001497bafe14ef24544e4a7398ce9b950f5a Mon Sep 17 00:00:00 2001 From: Horst Schirmeier Date: Fri, 9 May 2014 16:53:29 +0200 Subject: [PATCH 2/4] DatabaseCampaign: use more flexible get_variants() This change allows DatabaseCampaign users to take advantage of the improved variant selection methods in the Database class (multiple uses of --variant/--benchmark possible, plus --exclude-variant/--exclude-benchmark switches). Change-Id: Idb1ca04538ff7601b3648cd9ba766aa8690fff6b --- src/core/cpn/DatabaseCampaign.cc | 78 +++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/src/core/cpn/DatabaseCampaign.cc b/src/core/cpn/DatabaseCampaign.cc index a4d01f60..07992b11 100644 --- a/src/core/cpn/DatabaseCampaign.cc +++ b/src/core/cpn/DatabaseCampaign.cc @@ -1,3 +1,5 @@ +#include + #include "DatabaseCampaign.hpp" #include "cpn/CampaignManager.hpp" #include "util/CommandLine.hpp" @@ -28,12 +30,21 @@ bool DatabaseCampaign::run() { line interface */ if (!cb_commandline_init()) return false; - CommandLine::option_handle VARIANT = cmd.addOption("v", "variant", Arg::Required, - "-v/--variant \tVariant label (default: \"%\"; use % and _ as wildcard characters)"); - CommandLine::option_handle BENCHMARK = cmd.addOption("b", "benchmark", Arg::Required, - "-b/--benchmark \tBenchmark label (default: \"%\"; use % and _ as wildcard characters)\n"); - CommandLine::option_handle PRUNER = cmd.addOption("p", "prune-method", Arg::Required, - "-p/--prune-method \tWhich import method to use (default: basic)"); + CommandLine::option_handle VARIANT = + cmd.addOption("v", "variant", Arg::Required, + "-v/--variant \tVariant label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)"); + CommandLine::option_handle VARIANT_EXCLUDE = + cmd.addOption("", "variant-exclude", Arg::Required, + "--variant-exclude \tVariant to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)"); + CommandLine::option_handle BENCHMARK = + cmd.addOption("b", "benchmark", Arg::Required, + "-b/--benchmark \tBenchmark label (default: \"%\"; use % and _ as wildcard characters; may be used more than once)"); + CommandLine::option_handle BENCHMARK_EXCLUDE = + cmd.addOption("", "benchmark-exclude", Arg::Required, + "--benchmark-exclude \tBenchmark to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)"); + CommandLine::option_handle PRUNER = + cmd.addOption("p", "prune-method", Arg::Required, + "-p/--prune-method \tWhich import method to use (default: basic)"); if (!cmd.parse()) { log_send << "Error parsing arguments." << std::endl; @@ -45,25 +56,49 @@ bool DatabaseCampaign::run() { exit(0); } - std::string variant, benchmark, pruner; + std::vector variants, benchmarks, variants_exclude, benchmarks_exclude; + if (cmd[VARIANT]) { + for (option::Option *o = cmd[VARIANT]; o; o = o->next()) { + variants.push_back(std::string(o->arg)); + } + } - if (cmd[VARIANT].count() > 0) - variant = std::string(cmd[VARIANT].first()->arg); - else - variant = "%"; + if (cmd[VARIANT_EXCLUDE]) { + for (option::Option *o = cmd[VARIANT_EXCLUDE]; o; o = o->next()) { + variants_exclude.push_back(std::string(o->arg)); + } + } - if (cmd[BENCHMARK].count() > 0) - benchmark = std::string(cmd[BENCHMARK].first()->arg); - else - benchmark = "%"; + // fallback + if (variants.size() == 0) { + variants.push_back("%"); + } - if (cmd[PRUNER].count() > 0) + if (cmd[BENCHMARK]) { + for (option::Option *o = cmd[BENCHMARK]; o; o = o->next()) { + benchmarks.push_back(std::string(o->arg)); + } + } + + if (cmd[BENCHMARK_EXCLUDE]) { + for (option::Option *o = cmd[BENCHMARK_EXCLUDE]; o; o = o->next()) { + benchmarks_exclude.push_back(std::string(o->arg)); + } + } + + // fallback + if (benchmarks.size() == 0) { + benchmarks.push_back("%"); + } + + std::string pruner; + if (cmd[PRUNER]) { pruner = std::string(cmd[PRUNER].first()->arg); - else + } else { pruner = "basic"; + } db = Database::cmdline_connect(); - log_send << "Variant to use " << variant << "/" << benchmark << std::endl; fspmethod_id = db->get_fspmethod_id(pruner); log_send << "Pruner to use " << pruner << " (ID: " << fspmethod_id << ")" << std::endl; @@ -81,9 +116,10 @@ bool DatabaseCampaign::run() { load_completed_pilots(); - std::vector variants = db->get_variants(variant, benchmark); - for (std::vector::const_iterator it = variants.begin(); - it != variants.end(); ++it) { + std::vector variantlist = + db->get_variants(variants, variants_exclude, benchmarks, benchmarks_exclude); + for (std::vector::const_iterator it = variantlist.begin(); + it != variantlist.end(); ++it) { if (!run_variant(*it)) { log_send << "run_variant failed for " << it->variant << "/" << it->benchmark < Date: Thu, 28 Aug 2014 11:32:40 +0200 Subject: [PATCH 3/4] prune-trace: preserve existing data With this change, prune-trace checks for existing fsppilot/fspgroup entries for each variant to be pruned, and skips the variant in this case. This safety measure can be switched off with --overwrite. Change-Id: I7e758a9853a25685ca176cf1a1810523753cdd4a --- tools/prune-trace/Pruner.cc | 62 ++++++++++++++++++++++++++++-------- tools/prune-trace/Pruner.hpp | 7 ++-- tools/prune-trace/main.cc | 18 +++++++---- 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/tools/prune-trace/Pruner.cc b/tools/prune-trace/Pruner.cc index 644a9d0b..3e8c7e10 100644 --- a/tools/prune-trace/Pruner.cc +++ b/tools/prune-trace/Pruner.cc @@ -8,38 +8,72 @@ static Logger LOG ("Pruner"); #include "Pruner.hpp" -bool Pruner::init(fail::Database *db, +bool Pruner::init( const std::vector& variants, const std::vector& variants_exclude, const std::vector& benchmarks, - const std::vector& benchmarks_exclude) + const std::vector& benchmarks_exclude, + bool overwrite) { - this->db = db; - m_variants = db->get_variants( variants, variants_exclude, benchmarks, benchmarks_exclude); + + if (!(m_method_id = db->get_fspmethod_id(method_name()))) { + return false; + } + LOG << "Pruning with method " << method_name() << " (ID: " << m_method_id << ")" + << std::endl; + + // make sure we only prune variants that haven't been pruned previously + // (unless we run with --overwrite) + if (!overwrite) { + for (std::vector::iterator it = m_variants.begin(); + it != m_variants.end(); ) { + std::stringstream ss; + MYSQL_RES *res; + ss << "(SELECT variant_id FROM fsppilot WHERE " + << " variant_id = " << it->id << " AND " + << " fspmethod_id = " << m_method_id + << " LIMIT 1)" + << " UNION ALL " + << "(SELECT variant_id FROM fspgroup WHERE " + << " variant_id = " << it->id << " AND " + << " fspmethod_id = " << m_method_id + << " LIMIT 1)"; + if (!(res = db->query(ss.str().c_str(), true))) { + return false; + } + if (mysql_num_rows(res) > 0) { + // skip this variant + LOG << "skipping " << it->variant << "/" << it->benchmark + << " due to existing pruning data (use --overwrite to skip this check)" + << std::endl; + it = m_variants.erase(it); + } else { + ++it; + } + } + } + + // any variants left? if (m_variants.size() == 0) { LOG << "no variants found, nothing to do" << std::endl; return false; } - std::stringstream ss; + // construct comma-separated list usable in SQL "IN (...)" + std::stringstream commalist; for (std::vector::const_iterator it = m_variants.begin(); it != m_variants.end(); ++it) { + if (it != m_variants.begin()) { - ss << ","; + commalist << ","; } - ss << it->id; + commalist << it->id; } - m_variants_sql = ss.str(); + m_variants_sql = commalist.str(); - if (!(m_method_id = db->get_fspmethod_id(method_name()))) { - return false; - } - - LOG << "Pruning with method " << method_name() << " (ID: " << m_method_id << ")" - << std::endl; return true; } diff --git a/tools/prune-trace/Pruner.hpp b/tools/prune-trace/Pruner.hpp index e6e1f5dc..fcc02a61 100644 --- a/tools/prune-trace/Pruner.hpp +++ b/tools/prune-trace/Pruner.hpp @@ -13,11 +13,14 @@ protected: std::string m_variants_sql; public: - bool init(fail::Database *db, + void set_db(fail::Database *db) { this->db = db; } + + bool init( const std::vector& variants, const std::vector& variants_exclude, const std::vector& benchmarks, - const std::vector& benchmarks_exclude); + const std::vector& benchmarks_exclude, + bool overwrite); /** * Callback function that can be used to add command line options diff --git a/tools/prune-trace/main.cc b/tools/prune-trace/main.cc index d276bb5f..fe78cd32 100644 --- a/tools/prune-trace/main.cc +++ b/tools/prune-trace/main.cc @@ -45,6 +45,9 @@ int main(int argc, char *argv[]) { CommandLine::option_handle NO_DELETE = cmd.addOption("", "no-delete", Arg::None, "--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE"); + CommandLine::option_handle OVERWRITE = + cmd.addOption("", "overwrite", Arg::None, + "--overwrite \tOverwrite already existing pruning data (the default is to skip variants with existing entries)"); if (!cmd.parse()) { std::cerr << "Error parsing arguments." << std::endl; @@ -88,6 +91,7 @@ int main(int argc, char *argv[]) { } Database *db = Database::cmdline_connect(); + pruner->set_db(db); std::vector variants, benchmarks, variants_exclude, benchmarks_exclude; if (cmd[VARIANT]) { @@ -124,7 +128,12 @@ int main(int argc, char *argv[]) { benchmarks.push_back("%"); } - if (!pruner->init(db, variants, variants_exclude, benchmarks, benchmarks_exclude)) { + if (!pruner->create_database()) { + LOG << "pruner->create_database() failed" << endl; + exit(-1); + } + + if (!pruner->init(variants, variants_exclude, benchmarks, benchmarks_exclude, cmd[OVERWRITE])) { LOG << "pruner->init() failed" << endl; exit(-1); } @@ -132,12 +141,7 @@ int main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////// // Do the actual import //////////////////////////////////////////////////////////////// - if (!pruner->create_database()) { - LOG << "create_database() failed" << endl; - exit(-1); - } - - if (!cmd[NO_DELETE] && !pruner->clear_database()) { + if (!cmd[NO_DELETE] && cmd[OVERWRITE] && !pruner->clear_database()) { LOG << "clear_database() failed" << endl; exit(-1); } From 89817cf13fad6ee1c71d42179784fe1c0d5d15a2 Mon Sep 17 00:00:00 2001 From: Horst Schirmeier Date: Thu, 28 Aug 2014 13:28:44 +0200 Subject: [PATCH 4/4] prune-trace: list available pruners By using the AliasedRegistry, "prune-trace --help" (and "prune-trace -p '?'") now lists all available Pruners to the user. Change-Id: Ib5e3d00aabc37e6d48d804d2d709812af3f7efb2 --- tools/prune-trace/BasicPruner.hpp | 14 ++++++++++++ tools/prune-trace/FESamplingPruner.hpp | 5 +++++ tools/prune-trace/Pruner.hpp | 3 ++- tools/prune-trace/main.cc | 31 ++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tools/prune-trace/BasicPruner.hpp b/tools/prune-trace/BasicPruner.hpp index 67167ee5..a0f0e59d 100644 --- a/tools/prune-trace/BasicPruner.hpp +++ b/tools/prune-trace/BasicPruner.hpp @@ -9,6 +9,20 @@ public: BasicPruner(bool use_instr1 = false) : use_instr1(use_instr1) {} virtual std::string method_name() { return std::string("basic") + (use_instr1 ? "-left" : ""); } virtual bool prune_all(); + + void getAliases(std::deque *aliases) { + aliases->push_back("BasicPruner"); + aliases->push_back("basic"); + } +}; + +class BasicPrunerLeft : public BasicPruner { +public: + BasicPrunerLeft() : BasicPruner(true) {} + void getAliases(std::deque *aliases) { + aliases->push_back("BasicPrunerLeft"); + aliases->push_back("basic-left"); + } }; #endif diff --git a/tools/prune-trace/FESamplingPruner.hpp b/tools/prune-trace/FESamplingPruner.hpp index 6d4dc0cf..a9538622 100644 --- a/tools/prune-trace/FESamplingPruner.hpp +++ b/tools/prune-trace/FESamplingPruner.hpp @@ -24,6 +24,11 @@ public: virtual bool commandline_init(); virtual bool prune_all(); + void getAliases(std::deque *aliases) { + aliases->push_back("FESamplingPruner"); + aliases->push_back("sampling"); + } + private: bool sampling_prune(const fail::Database::Variant& variant); }; diff --git a/tools/prune-trace/Pruner.hpp b/tools/prune-trace/Pruner.hpp index fcc02a61..a0ec044d 100644 --- a/tools/prune-trace/Pruner.hpp +++ b/tools/prune-trace/Pruner.hpp @@ -4,8 +4,9 @@ #include #include #include "util/Database.hpp" +#include "util/AliasedRegisterable.hpp" -class Pruner { +class Pruner : public fail::AliasedRegisterable { protected: int m_method_id; fail::Database *db; diff --git a/tools/prune-trace/main.cc b/tools/prune-trace/main.cc index fe78cd32..22b2b6bf 100644 --- a/tools/prune-trace/main.cc +++ b/tools/prune-trace/main.cc @@ -4,6 +4,8 @@ #include "util/CommandLine.hpp" #include "util/Logger.hpp" +#include "util/AliasedRegistry.hpp" + static fail::Logger LOG("prune-trace", true); using namespace fail; @@ -16,6 +18,17 @@ using std::endl; int main(int argc, char *argv[]) { std::string username, hostname, database; + // register possible Pruners + AliasedRegistry registry; + BasicPruner basicpruner; + registry.add(&basicpruner); + BasicPrunerLeft basicprunerleft; + registry.add(&basicprunerleft); + FESamplingPruner fesamplingpruner; + registry.add(&fesamplingpruner); + + std::string pruners = registry.getPrimeAliasesCSV(); + // Manually fill the command line option parser CommandLine &cmd = CommandLine::Inst(); for (int i = 1; i < argc; ++i) { @@ -39,9 +52,9 @@ int main(int argc, char *argv[]) { CommandLine::option_handle BENCHMARK_EXCLUDE = cmd.addOption("", "benchmark-exclude", Arg::Required, "--benchmark-exclude \tBenchmark to exclude (default: UNSET; use % and _ as wildcard characters; may be used more than once)"); + std::string pruner_help = "-p/--prune-method \tWhich pruning method to use (default: basic); available pruning methods: " + pruners; CommandLine::option_handle PRUNER = - cmd.addOption("p", "prune-method", Arg::Required, - "-p/--prune-method \tWhich import method to use (default: basic)"); + cmd.addOption("p", "prune-method", Arg::Required, pruner_help); CommandLine::option_handle NO_DELETE = cmd.addOption("", "no-delete", Arg::None, "--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE"); @@ -55,6 +68,20 @@ int main(int argc, char *argv[]) { } Pruner *pruner; + std::string pruner_name = "BasicPruner"; + if (cmd[PRUNER]) { + pruner_name = cmd[PRUNER].first()->arg; + } + + // try and get the according pruner object; die on failure + if ((pruner = (Pruner *)registry.get(pruner_name)) == 0) { + if (pruner_name != "?" ) { + std::cerr << "Unknown import method: " << pruner_name << std::endl; + } + std::cerr << "Available import methods: " << pruners << std::endl; + exit(-1); + } + if (cmd[PRUNER]) { std::string imp(cmd[PRUNER].first()->arg); if (imp == "BasicPruner" || imp == "basic") {