From bbe60745e1d047a870dfa8bfba87a29b177b33ff Mon Sep 17 00:00:00 2001 From: Horst Schirmeier Date: Wed, 9 May 2018 15:20:03 +0200 Subject: [PATCH] data-aggregator: script overhaul + modularization Change-Id: I4353db1475f00956d19d91c8c558c34506ec836b --- tools/analysis/data-aggregator/CMakeLists.txt | 11 ++- .../fail-analysis-common.inc.sh | 36 ++++++++ .../data-aggregator/function-occurrences.sh | 25 +----- .../data-aggregator/resulttype-occurrences.sh | 60 ++++--------- .../resulttype-occurrences_coverage.sh | 90 ++++++------------- .../data-aggregator/symbol-occurrences.sh | 38 +++----- .../symbol-occurrences_coverage.sh | 58 ++++-------- .../translation-unit-occurrences.sh | 36 +++----- .../data-aggregator/variant-durations.sh | 45 ++-------- 9 files changed, 139 insertions(+), 260 deletions(-) create mode 100644 tools/analysis/data-aggregator/fail-analysis-common.inc.sh diff --git a/tools/analysis/data-aggregator/CMakeLists.txt b/tools/analysis/data-aggregator/CMakeLists.txt index 2b91f10b..06dec286 100644 --- a/tools/analysis/data-aggregator/CMakeLists.txt +++ b/tools/analysis/data-aggregator/CMakeLists.txt @@ -1 +1,10 @@ -install(PROGRAMS function-occurrences.sh resulttype-occurrences.sh resulttype-occurrences_coverage.sh symbol-occurrences.sh symbol-occurrences_coverage.sh translation-unit-occurrences.sh variant-durations.sh DESTINATION bin) +install(PROGRAMS +fail-analysis-common.inc.sh +function-occurrences.sh +resulttype-occurrences_coverage.sh +resulttype-occurrences.sh +symbol-occurrences_coverage.sh +symbol-occurrences.sh +translation-unit-occurrences.sh +variant-durations.sh +DESTINATION bin) diff --git a/tools/analysis/data-aggregator/fail-analysis-common.inc.sh b/tools/analysis/data-aggregator/fail-analysis-common.inc.sh new file mode 100644 index 00000000..e2b0ee9b --- /dev/null +++ b/tools/analysis/data-aggregator/fail-analysis-common.inc.sh @@ -0,0 +1,36 @@ +set -e + +if [ "$1" = -t ]; then + FORMAT=-t + shift +else + FORMAT=-B +fi + +if [ \( ! -z "${MUST_FILTER+x}" -a $# -ne 3 \) -o \( -z "${MUST_FILTER+x}" -a \( $# -lt 1 -o $# -gt 3 \) \) ]; then + if [ "$(type -t show_description)" = function ]; then + show_description + echo '' >&2 + fi + if [ ! -z "${MUST_FILTER+x}" ]; then + echo "usage: $0 [ -t ] DATABASE BENCHMARK VARIANT" >&2 + else + echo "usage: $0 [ -t ] DATABASE [ BENCHMARK [ VARIANT ] ]" >&2 + fi + echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 + exit 1 +fi + +DATABASE=$1 +BENCHMARK=$2 +VARIANT=$3 +MYSQL="mysql $FORMAT $DATABASE" + +# don't filter anything by default +FILTER=1 +if [ -n "$BENCHMARK" ]; then + FILTER="v.benchmark = '$BENCHMARK'" + if [ -n "$VARIANT" ]; then + FILTER="$FILTER AND v.variant = '$VARIANT'" + fi +fi diff --git a/tools/analysis/data-aggregator/function-occurrences.sh b/tools/analysis/data-aggregator/function-occurrences.sh index 65e2367b..ecad66c0 100755 --- a/tools/analysis/data-aggregator/function-occurrences.sh +++ b/tools/analysis/data-aggregator/function-occurrences.sh @@ -1,26 +1,10 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi - -if [ ! $# -eq 3 ]; then - echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" +MUST_FILTER=1 +source $(dirname $0)/fail-analysis-common.inc.sh $MYSQL << EOT -SELECT s.name, s.address, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences +SELECT v.benchmark, v.variant, s.name, s.address, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences FROM variant v JOIN symbol s ON s.variant_id = v.id @@ -30,8 +14,7 @@ JOIN fspgroup g ON g.variant_id = t.variant_id AND g.data_address = t.data_address AND g.instr2 = t.instr2 JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id -WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" +WHERE $FILTER GROUP BY s.variant_id, s.address, r.resulttype ORDER BY s.address, r.resulttype ; diff --git a/tools/analysis/data-aggregator/resulttype-occurrences.sh b/tools/analysis/data-aggregator/resulttype-occurrences.sh index 1c93900c..f458aeb4 100755 --- a/tools/analysis/data-aggregator/resulttype-occurrences.sh +++ b/tools/analysis/data-aggregator/resulttype-occurrences.sh @@ -1,50 +1,20 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi +source $(dirname $0)/fail-analysis-common.inc.sh -if [ $# -ne 3 -a $# -ne 1 ]; then - echo "usage: $0 [ -t ] DATABASE [ VARIANT BENCHMARK ]" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" - -if [ -z "$VARIANT" ]; then $MYSQL << EOT - SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; - SELECT v.benchmark, v.variant, r.resulttype, SUM((t.time2-t.time1+1) * t.width) AS occurrences - FROM result_GenericExperimentMessage r - INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id - INNER JOIN trace t ON g.instr2=t.instr2 - AND g.data_address=t.data_address - AND g.variant_id=t.variant_id - INNER JOIN variant v ON t.variant_id=v.id - GROUP BY v.id, r.resulttype - ORDER BY v.benchmark, v.variant, r.resulttype; +SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; +SELECT v.benchmark, v.variant, r.resulttype, SUM((t.time2-t.time1+1) * t.width) AS occurrences +FROM result_GenericExperimentMessage r +INNER JOIN fspgroup g + ON g.pilot_id = r.pilot_id +INNER JOIN trace t + ON g.instr2 = t.instr2 + AND g.data_address = t.data_address + AND g.variant_id = t.variant_id +INNER JOIN variant v + ON t.variant_id=v.id +WHERE $FILTER +GROUP BY v.id, r.resulttype +ORDER BY v.benchmark, v.variant, r.resulttype ASC; EOT -else -$MYSQL << EOT - SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; - SELECT r.resulttype, SUM((t.time2-t.time1+1) * t.width) AS occurrences - FROM result_GenericExperimentMessage r - INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id - INNER JOIN trace t ON g.instr2=t.instr2 - AND g.data_address=t.data_address - AND g.variant_id=t.variant_id - INNER JOIN variant v ON t.variant_id=v.id - WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" - GROUP BY r.resulttype - ORDER BY r.resulttype ASC; -EOT -fi diff --git a/tools/analysis/data-aggregator/resulttype-occurrences_coverage.sh b/tools/analysis/data-aggregator/resulttype-occurrences_coverage.sh index 5fa7c2c3..e3620f85 100755 --- a/tools/analysis/data-aggregator/resulttype-occurrences_coverage.sh +++ b/tools/analysis/data-aggregator/resulttype-occurrences_coverage.sh @@ -1,70 +1,32 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi +source $(dirname $0)/fail-analysis-common.inc.sh -if [ $# -ne 3 -a $# -ne 1 ]; then - echo "usage: $0 [ -t ] DATABASE [ VARIANT BENCHMARK ]" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" - -if [ -z "$VARIANT" ]; then $MYSQL << EOT - SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; - SELECT v.benchmark, v.variant, r.resulttype, - SUM((t.time2-t.time1+1) * t.width) - / - (SELECT SUM(t.time2-t.time1+1)*t.width - FROM result_GenericExperimentMessage r - INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id - INNER JOIN trace t ON g.instr2=t.instr2 - AND g.data_address=t.data_address - AND g.variant_id=t.variant_id - WHERE t.variant_id = v.id -- refers to parent query - ) AS coverage +SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; +SELECT v.benchmark, v.variant, r.resulttype, + SUM((t.time2-t.time1+1) * t.width) + / + (SELECT SUM(t.time2-t.time1+1)*t.width FROM result_GenericExperimentMessage r - INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id - INNER JOIN trace t ON g.instr2=t.instr2 - AND g.data_address=t.data_address - AND g.variant_id=t.variant_id - INNER JOIN variant v ON t.variant_id=v.id - GROUP BY v.id, r.resulttype - ORDER BY v.benchmark, v.variant, r.resulttype; + INNER JOIN fspgroup g + ON g.pilot_id = r.pilot_id + INNER JOIN trace t + ON g.instr2 = t.instr2 + AND g.data_address = t.data_address + AND g.variant_id = t.variant_id + WHERE t.variant_id = v.id -- refers to parent query + ) AS coverage +FROM result_GenericExperimentMessage r +INNER JOIN fspgroup g + ON g.pilot_id = r.pilot_id +INNER JOIN trace t + ON g.instr2 = t.instr2 + AND g.data_address = t.data_address + AND g.variant_id = t.variant_id +INNER JOIN variant v + ON t.variant_id = v.id +WHERE $FILTER +GROUP BY v.id, r.resulttype +ORDER BY v.benchmark, v.variant, r.resulttype; EOT -else -$MYSQL << EOT - SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; - SELECT r.resulttype, - SUM((t.time2-t.time1+1) * t.width) - / - (SELECT SUM(t.time2-t.time1+1)*t.width - FROM result_GenericExperimentMessage r - INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id - INNER JOIN trace t ON g.instr2=t.instr2 - AND g.data_address=t.data_address - AND g.variant_id=t.variant_id - WHERE t.variant_id = v.id -- refers to parent query - ) AS coverage - FROM result_GenericExperimentMessage r - INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id - INNER JOIN trace t ON g.instr2=t.instr2 - AND g.data_address=t.data_address - AND g.variant_id=t.variant_id - INNER JOIN variant v ON t.variant_id=v.id - WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" - GROUP BY r.resulttype - ORDER BY r.resulttype; -EOT -fi diff --git a/tools/analysis/data-aggregator/symbol-occurrences.sh b/tools/analysis/data-aggregator/symbol-occurrences.sh index f430fd0d..77bd5590 100755 --- a/tools/analysis/data-aggregator/symbol-occurrences.sh +++ b/tools/analysis/data-aggregator/symbol-occurrences.sh @@ -1,33 +1,15 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi - -if [ ! $# -eq 3 ]; then - echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" +source $(dirname $0)/fail-analysis-common.inc.sh $MYSQL << EOT - SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences - FROM variant v - INNER JOIN symbol s ON v.id = s.variant_id - INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 - INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 - INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id - WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" - GROUP BY v.benchmark, v.variant, s.name, r.resulttype - ORDER BY v.benchmark, v.variant, s.name, r.resulttype; +SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences +FROM variant v +INNER JOIN symbol s ON v.id = s.variant_id +INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 +INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 +INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id +WHERE $FILTER +GROUP BY v.id, s.name, r.resulttype +ORDER BY v.benchmark, v.variant, s.name, r.resulttype; EOT diff --git a/tools/analysis/data-aggregator/symbol-occurrences_coverage.sh b/tools/analysis/data-aggregator/symbol-occurrences_coverage.sh index 8335482c..c0cd881a 100755 --- a/tools/analysis/data-aggregator/symbol-occurrences_coverage.sh +++ b/tools/analysis/data-aggregator/symbol-occurrences_coverage.sh @@ -1,44 +1,24 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi - -if [ ! $# -eq 3 ]; then - echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" +source $(dirname $0)/fail-analysis-common.inc.sh $MYSQL << EOT - SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, - SUM(t.time2-t.time1+1) - / - (SELECT SUM(t.time2-t.time1+1) - FROM variant v - INNER JOIN symbol s ON v.id = s.variant_id - INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 - INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 - INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id - WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" - ) AS coverage - FROM variant v - INNER JOIN symbol s ON v.id = s.variant_id - INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 - INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 - INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id - WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" - GROUP BY v.benchmark, v.variant, s.name, r.resulttype - ORDER BY v.benchmark, v.variant, s.name, r.resulttype; +SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, + SUM(t.time2-t.time1+1) + / + (SELECT SUM(t.time2-t.time1+1) + FROM symbol s + INNER JOIN trace t ON t.variant_id = s.variant_id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 + INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 + INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id + WHERE t.variant_id = v.id -- refers to parent query + ) AS coverage +FROM variant v +INNER JOIN symbol s ON v.id = s.variant_id +INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 +INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 +INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id +WHERE $FILTER +GROUP BY v.id, s.name, r.resulttype +ORDER BY v.benchmark, v.variant, s.name, r.resulttype; EOT diff --git a/tools/analysis/data-aggregator/translation-unit-occurrences.sh b/tools/analysis/data-aggregator/translation-unit-occurrences.sh index 2427706b..2b6432d7 100755 --- a/tools/analysis/data-aggregator/translation-unit-occurrences.sh +++ b/tools/analysis/data-aggregator/translation-unit-occurrences.sh @@ -1,39 +1,25 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi - -if [ ! $# -eq 3 ]; then - echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" +source $(dirname $0)/fail-analysis-common.inc.sh $MYSQL << EOT SELECT v.benchmark, v.variant, f.path, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences FROM variant v JOIN dbg_filename f - ON f.variant_id = v.id + ON f.variant_id = v.id JOIN dbg_mapping m - ON f.variant_id = m.variant_id AND f.file_id = m.file_id + ON f.variant_id = m.variant_id + AND f.file_id = m.file_id JOIN trace t - ON t.variant_id = m.variant_id AND t.instr2_absolute BETWEEN m.instr_absolute AND m.instr_absolute + m.line_range_size - 1 + ON t.variant_id = m.variant_id + AND t.instr2_absolute BETWEEN m.instr_absolute AND m.instr_absolute + m.line_range_size - 1 JOIN fspgroup g - ON g.variant_id = t.variant_id AND g.data_address = t.data_address AND g.instr2 = t.instr2 + ON g.variant_id = t.variant_id + AND g.data_address = t.data_address + AND g.instr2 = t.instr2 JOIN result_GenericExperimentMessage r - ON r.pilot_id = g.pilot_id -WHERE v.variant="$VARIANT" - AND v.benchmark="$BENCHMARK" + ON r.pilot_id = g.pilot_id +WHERE $FILTER GROUP BY v.id, f.file_id, r.resulttype ORDER BY v.benchmark, v.variant, f.path, r.resulttype ; diff --git a/tools/analysis/data-aggregator/variant-durations.sh b/tools/analysis/data-aggregator/variant-durations.sh index 818c7516..9c8a13b6 100755 --- a/tools/analysis/data-aggregator/variant-durations.sh +++ b/tools/analysis/data-aggregator/variant-durations.sh @@ -1,44 +1,15 @@ #!/bin/bash -set -e -if [ "$1" = -t ]; then - FORMAT=-t - shift -else - FORMAT=-B -fi +MUST_FILTER=1 +source $(dirname $0)/fail-analysis-common.inc.sh -if [ $# -ne 3 -a $# -ne 1 ]; then - echo "usage: $0 [ -t ] DATABASE [ VARIANT BENCHMARK ]" >&2 - echo " -t Display output in table format (tab-separated CSV otherwise)" >&2 - exit 1 -fi - -DATABASE=$1 -VARIANT=$2 -BENCHMARK=$3 -MYSQL="mysql $FORMAT $DATABASE" - -if [ -z "$VARIANT" ]; then $MYSQL << EOT - SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; - SELECT v.benchmark, v.variant, +SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; +SELECT v.benchmark, v.variant, MAX(t.time2)-MIN(t.time1)+1 AS duration, MAX(t.instr2)-MIN(t.instr1)+1 AS dyn_instr - FROM trace t - JOIN variant v ON t.variant_id = v.id - GROUP BY v.id - ORDER BY v.benchmark, v.variant; +FROM trace t +JOIN variant v + ON t.variant_id = v.id +WHERE $FILTER EOT -else -$MYSQL << EOT - SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; - SELECT - MAX(t.time2)-MIN(t.time1)+1 AS duration, - MAX(t.instr2)-MIN(t.instr1)+1 AS dyn_instr - FROM trace t - JOIN variant v ON t.variant_id = v.id - WHERE v.variant = "$VARIANT" - AND v.benchmark = "$BENCHMARK" -EOT -fi