christoph/fail
1
Fork 0
Code Releases Activity

db-campaign: load completed pilots in one query

Browse Source 
Instead of issuing a query for every variant, we assemble a set of
variant ids and query `WHERE variant_id in (1,...)'. This has not only
the effect of higher optimization potential for the database, but also
the query is issued before any result can come back. This will avoid an
overfull receive queue within the job server.

Change-Id: I5b1c60f92b97741ce26d9e50760b601929cef44f
This commit is contained in:
Christian Dietrich
2014-08-25 13:30:41 +02:00
parent 0fb6653fa8
commit 61050b3760
2 changed files with 22 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/core/cpn/DatabaseCampaign.cc
View File

@ -79,12 +79,14 @@ bool DatabaseCampaign::run() {
boost::thread collect_thread(&DatabaseCampaign::collect_result_thread, this);
#endif
std::vector<Database::Variant> variants = db->get_variants(variant, benchmark);
// Which Pilots were already processed?
load_completed_pilots(variants);
for (std::vector<Database::Variant>::const_iterator it = variants.begin();
it != variants.end(); ++it) {
// Which Pilots were already processed?
load_completed_pilots(*it);
// Push all other variants to the queue
if (!run_variant(*it)) {
@ -219,15 +221,27 @@ bool DatabaseCampaign::run_variant(Database::Variant variant) {
}
void DatabaseCampaign::load_completed_pilots(Database::Variant variant)
void DatabaseCampaign::load_completed_pilots(std::vector<Database::Variant> &variants)
{
// If no variants were given, do nothing
if (variants.size() == 0)
return;
// load list of partially or completely finished pilots
log_send << "loading completed pilot IDs ... (" << variant.variant << "/"
<< variant.benchmark << ")" << std::endl;
std::stringstream variant_str;
bool comma = false;
for (std::vector<Database::Variant>::const_iterator it = variants.begin();
it != variants.end(); ++it) {
if (comma) variant_str << ", ";
variant_str << it->id;
comma = true; // Next time we need a comma
}
log_send << "loading completed pilot IDs ..." << std::endl;
std::stringstream sql;
sql << "SELECT pilot_id, COUNT(*) FROM fsppilot p"
<< " JOIN " << db_connect.result_table() << " r ON r.pilot_id = p.id"
<< " WHERE variant_id = " << variant.id
<< " WHERE variant_id in (" << variant_str.str() << ")"
<< " GROUP BY pilot_id ";
MYSQL_RES *ids = db->query_stream(sql.str().c_str());
MYSQL_ROW row;

2
src/core/cpn/DatabaseCampaign.hpp
View File

@ -29,7 +29,7 @@ class DatabaseCampaign : public Campaign {
int fspmethod_id; // !< Which fspmethod should be put out to the clients
void collect_result_thread();
void load_completed_pilots(fail::Database::Variant);
void load_completed_pilots(std::vector<fail::Database::Variant> &);
unsigned existing_results_for_pilot(unsigned pilot_id);
#ifndef __puma