From 61050b3760b1f8f4e8f90eadd25ab021ed7d8c45 Mon Sep 17 00:00:00 2001 From: Christian Dietrich Date: Mon, 25 Aug 2014 13:30:41 +0200 Subject: [PATCH] db-campaign: load completed pilots in one query Instead of issuing a query for every variant, we assemble a set of variant ids and query `WHERE variant_id in (1,...)'. This has not only the effect of higher optimization potential for the database, but also the query is issued before any result can come back. This will avoid an overfull receive queue within the job server. Change-Id: I5b1c60f92b97741ce26d9e50760b601929cef44f --- src/core/cpn/DatabaseCampaign.cc | 28 +++++++++++++++++++++------- src/core/cpn/DatabaseCampaign.hpp | 2 +- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/core/cpn/DatabaseCampaign.cc b/src/core/cpn/DatabaseCampaign.cc index 07f3f598..a789fd3a 100644 --- a/src/core/cpn/DatabaseCampaign.cc +++ b/src/core/cpn/DatabaseCampaign.cc @@ -79,12 +79,14 @@ bool DatabaseCampaign::run() { boost::thread collect_thread(&DatabaseCampaign::collect_result_thread, this); #endif - std::vector variants = db->get_variants(variant, benchmark); + + + // Which Pilots were already processed? + load_completed_pilots(variants); + for (std::vector::const_iterator it = variants.begin(); it != variants.end(); ++it) { - // Which Pilots were already processed? - load_completed_pilots(*it); // Push all other variants to the queue if (!run_variant(*it)) { @@ -219,15 +221,27 @@ bool DatabaseCampaign::run_variant(Database::Variant variant) { } -void DatabaseCampaign::load_completed_pilots(Database::Variant variant) +void DatabaseCampaign::load_completed_pilots(std::vector &variants) { + // If no variants were given, do nothing + if (variants.size() == 0) + return; + // load list of partially or completely finished pilots - log_send << "loading completed pilot IDs ... (" << variant.variant << "/" - << variant.benchmark << ")" << std::endl; + std::stringstream variant_str; + bool comma = false; + for (std::vector::const_iterator it = variants.begin(); + it != variants.end(); ++it) { + if (comma) variant_str << ", "; + variant_str << it->id; + comma = true; // Next time we need a comma + } + log_send << "loading completed pilot IDs ..." << std::endl; + std::stringstream sql; sql << "SELECT pilot_id, COUNT(*) FROM fsppilot p" << " JOIN " << db_connect.result_table() << " r ON r.pilot_id = p.id" - << " WHERE variant_id = " << variant.id + << " WHERE variant_id in (" << variant_str.str() << ")" << " GROUP BY pilot_id "; MYSQL_RES *ids = db->query_stream(sql.str().c_str()); MYSQL_ROW row; diff --git a/src/core/cpn/DatabaseCampaign.hpp b/src/core/cpn/DatabaseCampaign.hpp index cc8c593c..977c120b 100644 --- a/src/core/cpn/DatabaseCampaign.hpp +++ b/src/core/cpn/DatabaseCampaign.hpp @@ -29,7 +29,7 @@ class DatabaseCampaign : public Campaign { int fspmethod_id; // !< Which fspmethod should be put out to the clients void collect_result_thread(); - void load_completed_pilots(fail::Database::Variant); + void load_completed_pilots(std::vector &); unsigned existing_results_for_pilot(unsigned pilot_id); #ifndef __puma