diff --git a/tools/analysis/data-aggregator/CMakeLists.txt b/tools/analysis/data-aggregator/CMakeLists.txt index 8df89f35..1b0c9315 100644 --- a/tools/analysis/data-aggregator/CMakeLists.txt +++ b/tools/analysis/data-aggregator/CMakeLists.txt @@ -3,8 +3,10 @@ fail-analysis-common.inc.sh function-occurrences-onwrite.sh function-occurrences.sh global-coverage-onwrite.sh +global-coverage-sampling.sh global-coverage.sh global-occurrences-onwrite.sh +global-occurrences-sampling.sh global-occurrences.sh symbol-coverage-onwrite.sh symbol-coverage.sh diff --git a/tools/analysis/data-aggregator/global-coverage-sampling.sh b/tools/analysis/data-aggregator/global-coverage-sampling.sh new file mode 100755 index 00000000..098a6b1f --- /dev/null +++ b/tools/analysis/data-aggregator/global-coverage-sampling.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +source $(dirname $0)/fail-analysis-common.inc.sh + +$MYSQL << EOT +SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; +SELECT benchmark, variant, method, resulttype, occurrences / n AS coverage, + + SQRT( + occurrences / n * (1 - occurrences / n) / n) AS standard_error, + 1.96 * SQRT( + occurrences / n * (1 - occurrences / n) / n) AS confidence95, + 2.326347874041 * SQRT( + occurrences / n * (1 - occurrences / n) / n) AS confidence98, + 2.576 * SQRT( + occurrences / n * (1 - occurrences / n) / n) AS confidence99, + + n + +FROM + (SELECT t.variant_id, v.benchmark, v.variant, g.fspmethod_id, m.method, CONCAT(r.resulttype) AS resulttype, SUM(g.weight) AS occurrences, + (SELECT SUM(g2.weight) + FROM fspgroup g2 + WHERE g2.variant_id = t.variant_id + AND g2.fspmethod_id = m.id -- repeat subquery instead? + ) AS n + FROM variant v + JOIN trace t ON v.id = t.variant_id + JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 + JOIN fspmethod m ON g.fspmethod_id = m.id + JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id + + WHERE m.method = 'sampling' + AND $FILTER + + GROUP BY t.variant_id, g.fspmethod_id, r.resulttype + ) sub +GROUP BY variant_id, fspmethod_id, resulttype + +UNION ALL + +SELECT v.benchmark, v.variant, method, r.resulttype, + SUM((t.time2-t.time1+1) * t.width) + / + (SELECT SUM(t.time2-t.time1+1)*t.width + FROM result_GenericExperimentMessage r + JOIN fspgroup g + ON g.pilot_id = r.pilot_id + AND g.fspmethod_id = (SELECT id FROM fspmethod WHERE method = 'basic') + JOIN trace t + ON g.instr2 = t.instr2 + AND g.data_address = t.data_address + AND g.variant_id = t.variant_id + WHERE t.variant_id = v.id -- refers to parent query + ) AS coverage, + 0,0,0,0,0 +FROM result_GenericExperimentMessage r +JOIN fspgroup g + ON g.pilot_id = r.pilot_id +JOIN fspmethod m + ON m.id = g.fspmethod_id + AND m.method = 'basic' +JOIN trace t + ON g.instr2 = t.instr2 + AND g.data_address = t.data_address + AND g.variant_id = t.variant_id +JOIN variant v + ON t.variant_id = v.id +WHERE $FILTER +GROUP BY v.id, fspmethod_id, resulttype + +ORDER BY benchmark, variant, method, resulttype +; +EOT diff --git a/tools/analysis/data-aggregator/global-occurrences-sampling.sh b/tools/analysis/data-aggregator/global-occurrences-sampling.sh new file mode 100755 index 00000000..a51f41a2 --- /dev/null +++ b/tools/analysis/data-aggregator/global-occurrences-sampling.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +source $(dirname $0)/fail-analysis-common.inc.sh + +$MYSQL << EOT +-- quantile function: http://en.wikipedia.org/wiki/Normal_distribution#Quantile_function +SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; +SELECT benchmark, variant, method, resulttype, occurrences, + + SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS standard_error, + 1.96 * SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS confidence95, + 2.326347874041 * SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS confidence98, + 2.576 * SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS confidence99, +/* 2.807 * SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS confidence995, + 3.090 * SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS confidence998, + 3.291 * SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea AS confidence999, + + SQRT( + occurrences_r / fullarea * (1 - occurrences_r / fullarea) / n + ) * fullarea / occurrences AS RSE, + +occurrences_r, fullarea, */ n + +FROM +( +SELECT variant_id, benchmark, variant, method, resulttype, + SUM(occurrences * + -- extrapolation factor + (SELECT SUM(t2.time2-t2.time1+1) -- complete area + FROM trace t2 + WHERE t2.accesstype = sub.accesstype AND t2.variant_id = sub.variant_id) + / + (SELECT SUM(g.weight) FROM trace t2 -- sampled area for this resulttype + accesstype + JOIN fspgroup g ON t2.variant_id = g.variant_id AND t2.data_address = g.data_address AND t2.instr2 = g.instr2 + WHERE g.fspmethod_id = sub.fspmethod_id AND t2.accesstype = sub.accesstype AND t2.variant_id = sub.variant_id) + ) AS occurrences, + + SUM(occurrences * + IF(sub.accesstype = 'R', + + -- extrapolation factor (same as above) + (SELECT SUM(t2.time2-t2.time1+1) + FROM trace t2 + WHERE t2.accesstype = sub.accesstype AND t2.variant_id = sub.variant_id) + / + (SELECT SUM(g.weight) FROM trace t2 + JOIN fspgroup g ON t2.variant_id = g.variant_id AND t2.data_address = g.data_address AND t2.instr2 = g.instr2 + WHERE g.fspmethod_id = sub.fspmethod_id AND t2.accesstype = sub.accesstype AND t2.variant_id = sub.variant_id) + + , 0) + ) AS occurrences_r, + + (SELECT SUM(g.weight) -- * 8 + FROM fspgroup g + JOIN trace t + ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 + WHERE g.variant_id = sub.variant_id AND g.fspmethod_id = sub.fspmethod_id AND t.accesstype = 'R' + ) AS n, -- #experiments for Reads + + (SELECT SUM(t2.time2-t2.time1+1) -- * 8 -- 8 experiments per byte, must be removed for burst experiments + FROM trace t2 + WHERE t2.accesstype = 'R' AND t2.variant_id = sub.variant_id + ) AS fullarea -- R area to extrapolate to + +FROM + (SELECT t.variant_id, v.benchmark, v.variant, g.fspmethod_id, m.method, r.resulttype, t.accesstype, SUM(g.weight) AS occurrences + FROM variant v + JOIN trace t ON v.id = t.variant_id + JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 + JOIN fspmethod m ON g.fspmethod_id = m.id + JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id + + WHERE m.method = 'sampling' + AND $FILTER + + GROUP BY t.variant_id, g.fspmethod_id, r.resulttype, t.accesstype + ) sub +-- WHERE resulttype IN ('TRAP', 'SDC', 'TIMEOUT') +-- GROUP BY variant_id, fspmethod_id -- , resulttype +GROUP BY variant_id, fspmethod_id, resulttype +) sub2 + +UNION ALL + +-- ------------------------------------ basic ------------------------------------ + +SELECT benchmark, variant, method, resulttype, SUM(t.time2-t.time1+1) AS occurrences, + 0 AS standard_error, + 0 AS confidence95, + 0 AS confidence98, + 0 AS confidence99, +/* + 0 AS confidence995, + 0 AS confidence998, + 0 AS confidence999, + 0 AS RSE, + 0 AS occurrences_r, +(SELECT SUM(t2.time2-t2.time1+1) *8 FROM trace t2 WHERE t2.variant_id = t.variant_id) AS fullarea, +*/ +(SELECT COUNT(*) /* *8 */ FROM fsppilot p WHERE p.variant_id = t.variant_id AND p.fspmethod_id = g.fspmethod_id) AS n +FROM trace t +JOIN variant v + ON t.variant_id = v.id +JOIN fspgroup g + ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 + AND g.fspmethod_id = (SELECT id FROM fspmethod WHERE method = 'basic') +JOIN fspmethod m + ON m.id = g.fspmethod_id +JOIN result_GenericExperimentMessage r + ON r.pilot_id = g.pilot_id +WHERE $FILTER +GROUP BY t.variant_id, g.fspmethod_id, r.resulttype + +ORDER BY benchmark, variant, method, CONCAT(resulttype) +; +EOT