christoph/wamr
1
Fork 0
Code Releases Activity

Update cmake files and wamr-test-suites to support collect code coverage (#1992)

Browse Source 
Support collecting code coverage with wamr-test-suites script by using
lcov and genhtml tools, eg.:
  cd tests/wamr-test-suites
  ./test_wamr.sh -s spec -b -P -C

The default code coverage and html files are generated at:
  tests/wamr-test-suites/workspace/wamr.lcov
  tests/wamr-test-suites/workspace/wamr-lcov.zip

And update wamr-test-suites scripts to support testing GC spec cases to
avoid frequent synchronization conflicts between branch main and dev/gc.
This commit is contained in:
Wenyong Huang
2023-02-28 17:38:18 +08:00
committed by GitHub
parent b4f0228497
commit 9b9ae0cfac
14 changed files with 342 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
tests/wamr-test-suites/test_wamr.sh
View File

@ -14,19 +14,23 @@ function help()
{
echo "test_wamr.sh [options]"
echo "-c clean previous test results, not start test"
echo "-s {suite_name} test only one suite (spec)"
echo "-m set compile target of iwasm(x86_64\x86_32\armv7_vfp\thumbv7_vfp\riscv64_lp64d\riscv64_lp64)"
echo "-t set compile type of iwasm(classic-interp\fast-interp\jit\aot\fast-jit\multi-tier-jit)"
echo "-s {suite_name} test only one suite (spec|wasi_certification)"
echo "-m set compile target of iwasm(x86_64|x86_32|armv7_vfp|thumbv7_vfp|riscv64_lp64d|riscv64_lp64)"
echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)"
echo "-M enable multi module feature"
echo "-p enable multi thread feature"
echo "-S enable SIMD feature"
echo "-G enable GC feature"
echo "-X enable XIP feature"
echo "-x test SGX"
echo "-w enable WASI threads"
echo "-b use the wabt binary release package instead of compiling from the source code"
echo "-g build iwasm with debug version"
echo "-v enable GC heap verification"
echo "-P run the spec test parallelly"
echo "-Q enable qemu"
echo "-F set the firmware path used by qemu"
echo "-w enable WASI threads"
echo "-C enable code coverage collect"
}
OPT_PARSED=""
@ -40,7 +44,10 @@ ENABLE_MULTI_MODULE=0
ENABLE_MULTI_THREAD=0
COLLECT_CODE_COVERAGE=0
ENABLE_SIMD=0
ENABLE_GC=0
ENABLE_XIP=0
ENABLE_DEBUG_VERSION=0
ENABLE_GC_HEAP_VERIFY=0
#unit test case arrary
TEST_CASE_ARR=()
SGX_OPT=""
@ -50,7 +57,7 @@ ENABLE_QEMU=0
QEMU_FIRMWARE=""
WASI_TESTSUITE_COMMIT="b18247e2161bea263fe924b8189c67b1d2d10a10"
while getopts ":s:cabt:m:wMCpSXxPQF:" opt
while getopts ":s:cabgvt:m:MCpSXxwPGQF:" opt
do
OPT_PARSED="TRUE"
case $opt in
@ -72,8 +79,9 @@ do
c)
read -t 5 -p "Are you sure to delete all reports. y/n " cmd
if [[ $cmd == "y" && $(ls -A workspace/report) ]];then
rm -r workspace/report/*
echo "cleaned all reports"
rm -fr workspace/report/*
rm -fr /tmp/*.wasm /tmp/*.wast /tmp/*.aot
echo "cleaned all reports and temp files"
fi
exit 0;;
a)
@ -128,6 +136,18 @@ do
echo "test SGX"
SGX_OPT="--sgx"
;;
g)
echo "enable build iwasm with debug version"
ENABLE_DEBUG_VERSION=1
;;
v)
echo "enable GC heap verification"
ENABLE_GC_HEAP_VERIFY=1
;;
G)
echo "enable GC feature"
ENABLE_GC=1
;;
P)
PARALLELISM=1
;;
@ -164,7 +184,6 @@ readonly DATE=$(date +%Y-%m-%d_%H:%M:%S)
readonly REPORT_DIR=${WORK_DIR}/report/${DATE}
mkdir -p ${REPORT_DIR}
# TODO: a strong assumation about a link to the WAMR project
readonly WAMR_DIR=${WORK_DIR}/../../..
if [[ ${SGX_OPT} == "--sgx" ]];then
@ -198,14 +217,16 @@ readonly ORC_EAGER_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \
-DWAMR_BUILD_LAZY_JIT=0 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly ORC_LAZY_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_TARGET=${TARGET} \
-DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \
-DWAMR_BUILD_LAZY_JIT=1 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly AOT_COMPILE_FLAGS="\
-DWAMR_BUILD_TARGET=${TARGET} \
@ -219,13 +240,15 @@ readonly FAST_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \
-DWAMR_BUILD_FAST_JIT=1 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly MULTI_TIER_JIT_COMPILE_FLAGS="\
-DWAMR_BUILD_TARGET=${TARGET} \
-DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \
-DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \
-DWAMR_BUILD_SPEC_TEST=1"
-DWAMR_BUILD_SPEC_TEST=1 \
-DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
readonly COMPILE_FLAGS=(
"${CLASSIC_INTERP_COMPILE_FLAGS}"
@ -237,39 +260,19 @@ readonly COMPILE_FLAGS=(
"${MULTI_TIER_JIT_COMPILE_FLAGS}"
)
# TODO: with libiwasm.so only
function unit_test()
{
echo "Now start unit tests"
cd ${WORK_DIR}
readonly UNIT_CASES="wasm-vm host-tool utils"
rm -fr unittest-build && mkdir unittest-build
cd unittest-build
echo "Build unit test"
touch ${REPORT_DIR}/unit_test_report.txt
for compile_flag in "${COMPILE_FLAGS[@]}"; do
echo "Build unit test with compile flags with " ${compile_flag}
# keep going and do not care if it is success or not
make -ki clean | true
cmake ${compile_flag} ${WORK_DIR}/../../unit && make -j 4
if [ "$?" != 0 ];then
echo -e "build unit test failed, you may need to change wamr into dev/aot branch and ensure llvm is built"
exit 1
fi
echo ${compile_flag} >> ${REPORT_DIR}/unit_test_report.txt
for case in ${UNIT_CASES}
do
echo "run ${case} ..."
cd ./${case}/
./${case/-/_}"_test" | tee -a ${REPORT_DIR}/unit_test_report.txt
cd -
echo "finish ${case}"
done
done
cmake ${WORK_DIR}/../../unit -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}
make -j
make test | tee -a ${REPORT_DIR}/unit_test_report.txt
echo "Finish unit tests"
}
@ -347,6 +350,27 @@ function spec_test()
git apply ../../spec-test-script/thread_proposal_fix_atomic_case.patch
fi
# update GC cases
if [[ ${ENABLE_GC} == 1 ]]; then
echo "checkout spec for GC proposal"
popd
rm -fr spec
# check spec test cases for GC
git clone -b main --single-branch https://github.com/WebAssembly/gc.git spec
pushd spec
git restore . && git clean -ffd .
# Sync constant expression descriptions
git reset --hard 62beb94ddd41987517781732f17f213d8b866dcc
git apply ../../spec-test-script/gc_ignore_cases.patch
echo "compile the reference intepreter"
pushd interpreter
make opt
popd
fi
popd
echo $(pwd)
@ -446,6 +470,10 @@ function spec_test()
ARGS_FOR_SPEC_TEST+="--parl "
fi
if [[ ${ENABLE_GC} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="--gc "
fi
if [[ ${ENABLE_QEMU} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="--qemu "
ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} "
@ -484,7 +512,7 @@ function wasi_test()
function wasi_certification_test()
{
echo "Now start wasi tests"
echo "Now start wasi certification tests"
cd ${WORK_DIR}
if [ ! -d "wasi-testsuite" ]; then
@ -514,7 +542,6 @@ function polybench_test()
if [[ $1 == "aot" || $1 == "jit" ]];then
./build.sh AOT ${SGX_OPT}
./test_aot.sh $1 ${SGX_OPT}
else
./build.sh
./test_interp.sh ${SGX_OPT}
@ -524,6 +551,22 @@ function polybench_test()
echo "Finish polybench tests"
}
function libsodium_test()
{
echo "Now start libsodium tests"
cd ${WORK_DIR}/../libsodium
if [[ $1 == "aot" || $1 == "jit" ]];then
./build.sh ${SGX_OPT}
./test_aot.sh $1 ${SGX_OPT}
else
./test_interp.sh ${SGX_OPT}
fi
cp report.txt ${REPORT_DIR}/libsodium_$1_test_report.txt
echo "Finish libsodium tests"
}
function malformed_test()
{
# build iwasm firstly
@ -535,14 +578,13 @@ function standalone_test()
{
cd ${WORK_DIR}/../../standalone
args=""
args="--$1"
[[ $1 == "aot" ]] && args="$args --aot" || args="$args --no-aot"
[[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx"
if [[ ${ENABLE_MULTI_THREAD} == 1 ]];then
args="$args --thread"
fi
[[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" && args="$args --no-thread"
[[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" && args="$args --no-simd"
./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt
}
@ -589,7 +631,7 @@ function build_wamrc()
&& ./build_llvm.sh \
&& if [ -d build ]; then rm -r build/*; else mkdir build; fi \
&& cd build \
&& cmake .. \
&& cmake .. -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \
&& make -j 4
}
@ -602,15 +644,33 @@ function build_wamrc()
function collect_coverage()
{
if [[ ${COLLECT_CODE_COVERAGE} == 1 ]];then
cd ${IWASM_LINUX_ROOT_DIR}/build
lcov -t "iwasm code coverage" -o iwasm.info -c -d .
genhtml -o iwasm-gcov iwasm.info
[[ -d iwasm-gcov ]] && \
cp -r iwasm-gcov ${REPORT_DIR}/$1_iwasm_gcov || \
echo "generate code coverage html failed"
if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then
ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR}
CODE_COV_FILE=""
if [[ -z "${COV_FILE}" ]]; then
CODE_COV_FILE="${WORK_DIR}/wamr.lcov"
else
CODE_COV_FILE="${COV_FILE}"
fi
pushd ${WORK_DIR} > /dev/null 2>&1
echo "Collect code coverage of iwasm"
./collect_coverage.sh ${CODE_COV_FILE} ${IWASM_LINUX_ROOT_DIR}/build
if [[ $1 == "llvm-aot" ]]; then
echo "Collect code coverage of wamrc"
./collect_coverage.sh ${CODE_COV_FILE} ${WAMR_DIR}/wamr-compiler/build
fi
for suite in "${TEST_CASE_ARR[@]}"; do
if [[ ${suite} = "unit" ]]; then
echo "Collect code coverage of unit test"
./collect_coverage.sh ${CODE_COV_FILE} ${WORK_DIR}/unittest-build
break
fi
done
popd > /dev/null 2>&1
else
echo "will not collect code coverage"
echo "code coverage isn't collected"
fi
}
@ -639,10 +699,22 @@ function trigger()
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SIMD=0"
fi
if [[ ${ENABLE_GC} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC=1"
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_REF_TYPES=1"
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_BULK_MEMORY=1"
fi
if [[ ${ENABLE_DEBUG_VERSION} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DCMAKE_BUILD_TYPE=Debug"
fi
if [[ ${ENABLE_GC_HEAP_VERIFY} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC_HEAP_VERIFY=1"
fi
if [[ ${ENABLE_WASI_THREADS} == 1 ]]; then
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=1"
else
EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=0"
fi
for t in "${TYPE[@]}"; do
@ -692,18 +764,18 @@ function trigger()
echo "work in orc jit eager compilation mode"
BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS
build_wamrc
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" jit
done
collect_coverage llvm-jit
echo "work in orc jit lazy compilation mode"
BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
build_iwasm_with_cfg $BUILD_FLAGS
build_wamrc
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" jit
done
collect_coverage llvm-jit
;;
"aot")
@ -717,7 +789,7 @@ function trigger()
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" aot
done
collect_coverage aot
collect_coverage llvm-aot
;;
"fast-jit")
@ -728,6 +800,7 @@ function trigger()
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" fast-jit
done
collect_coverage fast-jit
;;
"multi-tier-jit")
@ -738,6 +811,7 @@ function trigger()
for suite in "${TEST_CASE_ARR[@]}"; do
$suite"_test" multi-tier-jit
done
collect_coverage multi-tier-jit
;;
*)
@ -748,11 +822,19 @@ function trigger()
}
# if collect code coverage, ignore -s, test all test cases.
if [[ $TEST_CASE_ARR && $COLLECT_CODE_COVERAGE != 1 ]];then
if [[ $TEST_CASE_ARR ]];then
trigger || (echo "TEST FAILED"; exit 1)
else
# test all suite, ignore polybench because of long time cost
TEST_CASE_ARR=("sightglass" "spec" "wasi" "malformed" "standalone")
# test all suite, ignore polybench and libsodium because of long time cost
TEST_CASE_ARR=("spec")
: '
if [[ $COLLECT_CODE_COVERAGE == 1 ]];then
# add polybench if collecting code coverage data
TEST_CASE_ARR+=("polybench")
# add libsodium if needed, which takes long time to run
TEST_CASE_ARR+=("libsodium")
fi
'
trigger || (echo "TEST FAILED"; exit 1)
# Add more suites here
fi