christoph/fail
1
Fork 0
Code Releases Activity

data-aggregator: script overhaul + modularization

Browse Source 
Change-Id: I4353db1475f00956d19d91c8c558c34506ec836b
This commit is contained in:
Horst Schirmeier
2018-05-09 15:20:03 +02:00
parent 9ae8123433
commit bbe60745e1
9 changed files with 139 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tools/analysis/data-aggregator/CMakeLists.txt
View File

@ -1 +1,10 @@
install(PROGRAMS function-occurrences.sh resulttype-occurrences.sh resulttype-occurrences_coverage.sh symbol-occurrences.sh symbol-occurrences_coverage.sh translation-unit-occurrences.sh variant-durations.sh DESTINATION bin) install(PROGRAMS
fail-analysis-common.inc.sh
function-occurrences.sh
resulttype-occurrences_coverage.sh
resulttype-occurrences.sh
symbol-occurrences_coverage.sh
symbol-occurrences.sh
translation-unit-occurrences.sh
variant-durations.sh
DESTINATION bin)

36
tools/analysis/data-aggregator/fail-analysis-common.inc.sh Normal file
View File

@ -0,0 +1,36 @@
set -e
if [ "$1" = -t ]; then
FORMAT=-t
shift
else
FORMAT=-B
fi
if [ \( ! -z "${MUST_FILTER+x}" -a $# -ne 3 \) -o \( -z "${MUST_FILTER+x}" -a \( $# -lt 1 -o $# -gt 3 \) \) ]; then
if [ "$(type -t show_description)" = function ]; then
show_description
echo '' >&2
fi
if [ ! -z "${MUST_FILTER+x}" ]; then
echo "usage: $0 [ -t ] DATABASE BENCHMARK VARIANT" >&2
else
echo "usage: $0 [ -t ] DATABASE [ BENCHMARK [ VARIANT ] ]" >&2
fi
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
BENCHMARK=$2
VARIANT=$3
MYSQL="mysql $FORMAT $DATABASE"
# don't filter anything by default
FILTER=1
if [ -n "$BENCHMARK" ]; then
FILTER="v.benchmark = '$BENCHMARK'"
if [ -n "$VARIANT" ]; then
FILTER="$FILTER AND v.variant = '$VARIANT'"
fi
fi

25
tools/analysis/data-aggregator/function-occurrences.sh
View File

@ -1,26 +1,10 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then MUST_FILTER=1
FORMAT=-t source $(dirname $0)/fail-analysis-common.inc.sh
shift
else
FORMAT=-B
fi
if [ ! $# -eq 3 ]; then
echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
$MYSQL << EOT $MYSQL << EOT
SELECT s.name, s.address, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences SELECT v.benchmark, v.variant, s.name, s.address, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences
FROM variant v FROM variant v
JOIN symbol s JOIN symbol s
ON s.variant_id = v.id ON s.variant_id = v.id
@ -30,8 +14,7 @@ JOIN fspgroup g
ON g.variant_id = t.variant_id AND g.data_address = t.data_address AND g.instr2 = t.instr2 ON g.variant_id = t.variant_id AND g.data_address = t.data_address AND g.instr2 = t.instr2
JOIN result_GenericExperimentMessage r JOIN result_GenericExperimentMessage r
ON r.pilot_id = g.pilot_id ON r.pilot_id = g.pilot_id
WHERE v.variant="$VARIANT" WHERE $FILTER
AND v.benchmark="$BENCHMARK"
GROUP BY s.variant_id, s.address, r.resulttype GROUP BY s.variant_id, s.address, r.resulttype
ORDER BY s.address, r.resulttype ORDER BY s.address, r.resulttype
; ;

60
tools/analysis/data-aggregator/resulttype-occurrences.sh
View File

@ -1,50 +1,20 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then source $(dirname $0)/fail-analysis-common.inc.sh
FORMAT=-t
shift
else
FORMAT=-B
fi
if [ $# -ne 3 -a $# -ne 1 ]; then
echo "usage: $0 [ -t ] DATABASE [ VARIANT BENCHMARK ]" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
if [ -z "$VARIANT" ]; then
$MYSQL << EOT $MYSQL << EOT
SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; SET sql_mode = 'NO_UNSIGNED_SUBTRACTION';
SELECT v.benchmark, v.variant, r.resulttype, SUM((t.time2-t.time1+1) * t.width) AS occurrences SELECT v.benchmark, v.variant, r.resulttype, SUM((t.time2-t.time1+1) * t.width) AS occurrences
FROM result_GenericExperimentMessage r FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id INNER JOIN fspgroup g
INNER JOIN trace t ON g.instr2=t.instr2 ON g.pilot_id = r.pilot_id
AND g.data_address=t.data_address INNER JOIN trace t
AND g.variant_id=t.variant_id ON g.instr2 = t.instr2
INNER JOIN variant v ON t.variant_id=v.id AND g.data_address = t.data_address
GROUP BY v.id, r.resulttype AND g.variant_id = t.variant_id
ORDER BY v.benchmark, v.variant, r.resulttype; INNER JOIN variant v
ON t.variant_id=v.id
WHERE $FILTER
GROUP BY v.id, r.resulttype
ORDER BY v.benchmark, v.variant, r.resulttype ASC;
EOT EOT
else
$MYSQL << EOT
SET sql_mode = 'NO_UNSIGNED_SUBTRACTION';
SELECT r.resulttype, SUM((t.time2-t.time1+1) * t.width) AS occurrences
FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id
INNER JOIN trace t ON g.instr2=t.instr2
AND g.data_address=t.data_address
AND g.variant_id=t.variant_id
INNER JOIN variant v ON t.variant_id=v.id
WHERE v.variant="$VARIANT"
AND v.benchmark="$BENCHMARK"
GROUP BY r.resulttype
ORDER BY r.resulttype ASC;
EOT
fi

90
tools/analysis/data-aggregator/resulttype-occurrences_coverage.sh
View File

@ -1,70 +1,32 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then source $(dirname $0)/fail-analysis-common.inc.sh
FORMAT=-t
shift
else
FORMAT=-B
fi
if [ $# -ne 3 -a $# -ne 1 ]; then
echo "usage: $0 [ -t ] DATABASE [ VARIANT BENCHMARK ]" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
if [ -z "$VARIANT" ]; then
$MYSQL << EOT $MYSQL << EOT
SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; SET sql_mode = 'NO_UNSIGNED_SUBTRACTION';
SELECT v.benchmark, v.variant, r.resulttype, SELECT v.benchmark, v.variant, r.resulttype,
SUM((t.time2-t.time1+1) * t.width) SUM((t.time2-t.time1+1) * t.width)
/ /
(SELECT SUM(t.time2-t.time1+1)*t.width (SELECT SUM(t.time2-t.time1+1)*t.width
FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id
INNER JOIN trace t ON g.instr2=t.instr2
AND g.data_address=t.data_address
AND g.variant_id=t.variant_id
WHERE t.variant_id = v.id -- refers to parent query
) AS coverage
FROM result_GenericExperimentMessage r FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id INNER JOIN fspgroup g
INNER JOIN trace t ON g.instr2=t.instr2 ON g.pilot_id = r.pilot_id
AND g.data_address=t.data_address INNER JOIN trace t
AND g.variant_id=t.variant_id ON g.instr2 = t.instr2
INNER JOIN variant v ON t.variant_id=v.id AND g.data_address = t.data_address
GROUP BY v.id, r.resulttype AND g.variant_id = t.variant_id
ORDER BY v.benchmark, v.variant, r.resulttype; WHERE t.variant_id = v.id -- refers to parent query
) AS coverage
FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g
ON g.pilot_id = r.pilot_id
INNER JOIN trace t
ON g.instr2 = t.instr2
AND g.data_address = t.data_address
AND g.variant_id = t.variant_id
INNER JOIN variant v
ON t.variant_id = v.id
WHERE $FILTER
GROUP BY v.id, r.resulttype
ORDER BY v.benchmark, v.variant, r.resulttype;
EOT EOT
else
$MYSQL << EOT
SET sql_mode = 'NO_UNSIGNED_SUBTRACTION';
SELECT r.resulttype,
SUM((t.time2-t.time1+1) * t.width)
/
(SELECT SUM(t.time2-t.time1+1)*t.width
FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id
INNER JOIN trace t ON g.instr2=t.instr2
AND g.data_address=t.data_address
AND g.variant_id=t.variant_id
WHERE t.variant_id = v.id -- refers to parent query
) AS coverage
FROM result_GenericExperimentMessage r
INNER JOIN fspgroup g ON g.pilot_id=r.pilot_id
INNER JOIN trace t ON g.instr2=t.instr2
AND g.data_address=t.data_address
AND g.variant_id=t.variant_id
INNER JOIN variant v ON t.variant_id=v.id
WHERE v.variant="$VARIANT"
AND v.benchmark="$BENCHMARK"
GROUP BY r.resulttype
ORDER BY r.resulttype;
EOT
fi

38
tools/analysis/data-aggregator/symbol-occurrences.sh
View File

@ -1,33 +1,15 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then source $(dirname $0)/fail-analysis-common.inc.sh
FORMAT=-t
shift
else
FORMAT=-B
fi
if [ ! $# -eq 3 ]; then
echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
$MYSQL << EOT $MYSQL << EOT
SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences
FROM variant v FROM variant v
INNER JOIN symbol s ON v.id = s.variant_id INNER JOIN symbol s ON v.id = s.variant_id
INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1
INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2
INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
WHERE v.variant="$VARIANT" WHERE $FILTER
AND v.benchmark="$BENCHMARK" GROUP BY v.id, s.name, r.resulttype
GROUP BY v.benchmark, v.variant, s.name, r.resulttype ORDER BY v.benchmark, v.variant, s.name, r.resulttype;
ORDER BY v.benchmark, v.variant, s.name, r.resulttype;
EOT EOT

58
tools/analysis/data-aggregator/symbol-occurrences_coverage.sh
View File

@ -1,44 +1,24 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then source $(dirname $0)/fail-analysis-common.inc.sh
FORMAT=-t
shift
else
FORMAT=-B
fi
if [ ! $# -eq 3 ]; then
echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
$MYSQL << EOT $MYSQL << EOT
SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype, SELECT v.benchmark, v.variant, s.name, s.size, r.resulttype,
SUM(t.time2-t.time1+1) SUM(t.time2-t.time1+1)
/ /
(SELECT SUM(t.time2-t.time1+1) (SELECT SUM(t.time2-t.time1+1)
FROM variant v FROM symbol s
INNER JOIN symbol s ON v.id = s.variant_id INNER JOIN trace t ON t.variant_id = s.variant_id AND t.data_address BETWEEN s.address AND s.address + s.size - 1
INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2
INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id WHERE t.variant_id = v.id -- refers to parent query
WHERE v.variant="$VARIANT" ) AS coverage
AND v.benchmark="$BENCHMARK" FROM variant v
) AS coverage INNER JOIN symbol s ON v.id = s.variant_id
FROM variant v INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1
INNER JOIN symbol s ON v.id = s.variant_id INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2
INNER JOIN trace t ON t.variant_id = v.id AND t.data_address BETWEEN s.address AND s.address + s.size - 1 INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id
INNER JOIN fspgroup g ON t.variant_id = g.variant_id AND t.data_address = g.data_address AND t.instr2 = g.instr2 WHERE $FILTER
INNER JOIN result_GenericExperimentMessage r ON r.pilot_id = g.pilot_id GROUP BY v.id, s.name, r.resulttype
WHERE v.variant="$VARIANT" ORDER BY v.benchmark, v.variant, s.name, r.resulttype;
AND v.benchmark="$BENCHMARK"
GROUP BY v.benchmark, v.variant, s.name, r.resulttype
ORDER BY v.benchmark, v.variant, s.name, r.resulttype;
EOT EOT

36
tools/analysis/data-aggregator/translation-unit-occurrences.sh
View File

@ -1,39 +1,25 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then source $(dirname $0)/fail-analysis-common.inc.sh
FORMAT=-t
shift
else
FORMAT=-B
fi
if [ ! $# -eq 3 ]; then
echo "usage: $0 [ -t ] DATABASE VARIANT BENCHMARK" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
$MYSQL << EOT $MYSQL << EOT
SELECT v.benchmark, v.variant, f.path, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences SELECT v.benchmark, v.variant, f.path, r.resulttype, SUM(t.time2-t.time1+1) AS occurrences
FROM variant v FROM variant v
JOIN dbg_filename f JOIN dbg_filename f
ON f.variant_id = v.id ON f.variant_id = v.id
JOIN dbg_mapping m JOIN dbg_mapping m
ON f.variant_id = m.variant_id AND f.file_id = m.file_id ON f.variant_id = m.variant_id
AND f.file_id = m.file_id
JOIN trace t JOIN trace t
ON t.variant_id = m.variant_id AND t.instr2_absolute BETWEEN m.instr_absolute AND m.instr_absolute + m.line_range_size - 1 ON t.variant_id = m.variant_id
AND t.instr2_absolute BETWEEN m.instr_absolute AND m.instr_absolute + m.line_range_size - 1
JOIN fspgroup g JOIN fspgroup g
ON g.variant_id = t.variant_id AND g.data_address = t.data_address AND g.instr2 = t.instr2 ON g.variant_id = t.variant_id
AND g.data_address = t.data_address
AND g.instr2 = t.instr2
JOIN result_GenericExperimentMessage r JOIN result_GenericExperimentMessage r
ON r.pilot_id = g.pilot_id ON r.pilot_id = g.pilot_id
WHERE v.variant="$VARIANT" WHERE $FILTER
AND v.benchmark="$BENCHMARK"
GROUP BY v.id, f.file_id, r.resulttype GROUP BY v.id, f.file_id, r.resulttype
ORDER BY v.benchmark, v.variant, f.path, r.resulttype ORDER BY v.benchmark, v.variant, f.path, r.resulttype
; ;

45
tools/analysis/data-aggregator/variant-durations.sh
View File

@ -1,44 +1,15 @@
#!/bin/bash #!/bin/bash
set -e
if [ "$1" = -t ]; then MUST_FILTER=1
FORMAT=-t source $(dirname $0)/fail-analysis-common.inc.sh
shift
else
FORMAT=-B
fi
if [ $# -ne 3 -a $# -ne 1 ]; then
echo "usage: $0 [ -t ] DATABASE [ VARIANT BENCHMARK ]" >&2
echo " -t Display output in table format (tab-separated CSV otherwise)" >&2
exit 1
fi
DATABASE=$1
VARIANT=$2
BENCHMARK=$3
MYSQL="mysql $FORMAT $DATABASE"
if [ -z "$VARIANT" ]; then
$MYSQL << EOT $MYSQL << EOT
SET sql_mode = 'NO_UNSIGNED_SUBTRACTION'; SET sql_mode = 'NO_UNSIGNED_SUBTRACTION';
SELECT v.benchmark, v.variant, SELECT v.benchmark, v.variant,
MAX(t.time2)-MIN(t.time1)+1 AS duration, MAX(t.time2)-MIN(t.time1)+1 AS duration,
MAX(t.instr2)-MIN(t.instr1)+1 AS dyn_instr MAX(t.instr2)-MIN(t.instr1)+1 AS dyn_instr
FROM trace t FROM trace t
JOIN variant v ON t.variant_id = v.id JOIN variant v
GROUP BY v.id ON t.variant_id = v.id
ORDER BY v.benchmark, v.variant; WHERE $FILTER
EOT EOT
else
$MYSQL << EOT
SET sql_mode = 'NO_UNSIGNED_SUBTRACTION';
SELECT
MAX(t.time2)-MIN(t.time1)+1 AS duration,
MAX(t.instr2)-MIN(t.instr1)+1 AS dyn_instr
FROM trace t
JOIN variant v ON t.variant_id = v.id
WHERE v.variant = "$VARIANT"
AND v.benchmark = "$BENCHMARK"
EOT
fi