christoph/wamr
1
Fork 0
Code Releases Activity

Update cmake files and wamr-test-suites to support collect code coverage (#1992)

Browse Source 
Support collecting code coverage with wamr-test-suites script by using
lcov and genhtml tools, eg.:
  cd tests/wamr-test-suites
  ./test_wamr.sh -s spec -b -P -C

The default code coverage and html files are generated at:
  tests/wamr-test-suites/workspace/wamr.lcov
  tests/wamr-test-suites/workspace/wamr-lcov.zip

And update wamr-test-suites scripts to support testing GC spec cases to
avoid frequent synchronization conflicts between branch main and dev/gc.
This commit is contained in:
Wenyong Huang
2023-02-28 17:38:18 +08:00
committed by GitHub
parent b4f0228497
commit 9b9ae0cfac
14 changed files with 342 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tests/wamr-test-suites/spec-test-script/all.py
View File

@ -14,6 +14,18 @@ import time
"""
The script itself has to be put under the same directory with the "spec".
To run a single non-GC case with interpreter mode:
cd workspace
python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
spec/test/core/xxx.wast
To run a single non-GC case with aot mode:
cd workspace
python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
--aot-compiler wamrc spec/test/core/xxx.wast
To run a single GC case:
cd workspace
python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \
--aot-compiler wamrc --gc spec/test/core/xxx.wast
"""
PLATFORM_NAME = os.uname().sysname.lower()
@ -22,9 +34,9 @@ IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
IWASM_QEMU_CMD = "iwasm"
SPEC_TEST_DIR = "spec/test/core"
WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
class TargetAction(argparse.Action):
TARGET_MAP = {
"ARMV7_VFP": "armv7",
@ -51,6 +63,7 @@ def ignore_the_case(
multi_module_flag=False,
multi_thread_flag=False,
simd_flag=False,
gc_flag=False,
xip_flag=False,
qemu_flag=False
):
@ -63,6 +76,10 @@ def ignore_the_case(
if "i386" == target and case_name in ["float_exprs"]:
return True
if gc_flag:
if case_name in ["type-canon", "type-equivalence", "type-rec"]:
return True;
if sgx_flag:
if case_name in ["conversions", "f32_bitwise", "f64_bitwise"]:
return True
@ -76,7 +93,9 @@ def ignore_the_case(
return True
if qemu_flag:
if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", "conversions", "f32", "f32_cmp", "float_exprs", "float_misc", "select", "memory_grow"]:
if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp",
"conversions", "f32", "f32_cmp", "float_exprs",
"float_misc", "select", "memory_grow"]:
return True
return False
@ -109,6 +128,7 @@ def test_case(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
gc_flag=False,
qemu_flag=False,
qemu_firmware='',
log='',
@ -124,6 +144,7 @@ def test_case(
multi_module_flag,
multi_thread_flag,
simd_flag,
gc_flag,
xip_flag,
qemu_flag
):
@ -131,7 +152,7 @@ def test_case(
CMD = ["python3", "runtest.py"]
CMD.append("--wast2wasm")
CMD.append(WAST2WASM_CMD)
CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD)
CMD.append("--interpreter")
if sgx_flag:
CMD.append(IWASM_SGX_CMD)
@ -171,6 +192,9 @@ def test_case(
if not clean_up_flag:
CMD.append("--no_cleanup")
if gc_flag:
CMD.append("--gc")
if log != '':
CMD.append("--log-dir")
CMD.append(log)
@ -231,6 +255,7 @@ def test_suite(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
gc_flag=False,
parl_flag=False,
qemu_flag=False,
qemu_firmware='',
@ -246,6 +271,10 @@ def test_suite(
simd_case_list = sorted(suite_path.glob("simd/*.wast"))
case_list.extend(simd_case_list)
if gc_flag:
gc_case_list = sorted(suite_path.glob("gc/*.wast"))
case_list.extend(gc_case_list)
case_count = len(case_list)
failed_case = 0
successful_case = 0
@ -268,6 +297,7 @@ def test_suite(
xip_flag,
clean_up_flag,
verbose_flag,
gc_flag,
qemu_flag,
qemu_firmware,
log,
@ -304,6 +334,7 @@ def test_suite(
xip_flag,
clean_up_flag,
verbose_flag,
gc_flag,
qemu_flag,
qemu_firmware,
log,
@ -414,6 +445,13 @@ def main():
dest="verbose_flag",
help="Close real time output while running cases, only show last words of failed ones",
)
parser.add_argument(
"--gc",
action="store_true",
default=False,
dest="gc_flag",
help="Running with GC feature",
)
parser.add_argument(
"cases",
metavar="path_to__case",
@ -446,6 +484,7 @@ def main():
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.gc_flag,
options.parl_flag,
options.qemu_flag,
options.qemu_firmware,
@ -469,6 +508,7 @@ def main():
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.gc_flag,
options.qemu_flag,
options.qemu_firmware,
options.log

80
tests/wamr-test-suites/spec-test-script/collect_coverage.sh Executable file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env bash
#
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
readonly WORK_DIR=$PWD
readonly WAMR_DIR=${WORK_DIR}/../../..
readonly DST_COV_FILE=$1
readonly SRC_COV_DIR=$2
readonly SRC_TEMP_COV_FILE=wamr_temp.lcov
readonly SRC_COV_FILE=wamr.lcov
# get dest folder
dir=$(dirname ${DST_COV_FILE})
pushd ${dir} > /dev/null 2>&1
readonly DST_COV_DIR=${PWD}
popd > /dev/null 2>&1
if [[ ! -d ${SRC_COV_DIR} ]]; then
echo "${SRC_COV_DIR} doesn't exist, ignore code coverage collection"
exit
fi
echo "Start to collect code coverage of ${SRC_COV_DIR} .."
pushd ${SRC_COV_DIR} > /dev/null 2>&1
# collect all code coverage data
lcov -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1
# extract code coverage data of WAMR source files
lcov -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \
-rc lcov_branch_coverage=1 \
"*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \
"*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \
"*/app-framework/*" "*/test-tools/*"
if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
if [[ -s ${DST_COV_FILE} ]]; then
# merge code coverage data
lcov --rc lcov_branch_coverage=1 \
--add-tracefile ${SRC_TEMP_COV_FILE} \
-a ${DST_COV_FILE} -o ${SRC_COV_FILE}
# backup the original lcov file
cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig"
# replace the lcov file
cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
else
cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE}
cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
fi
# get ignored prefix path
dir=$(dirname ${WAMR_DIR}/../..)
pushd ${dir} > /dev/null 2>&1
prefix_full_path=${PWD}
popd > /dev/null 2>&1
# generate html output for merged code coverage data
rm -fr ${DST_COV_DIR}/wamr-lcov
genhtml -t "WAMR Code Coverage" \
--rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \
-o ${DST_COV_DIR}/wamr-lcov \
${DST_COV_FILE}
cd ${DST_COV_DIR}
rm -f wamr-lcov.zip
zip -r -q -o wamr-lcov.zip wamr-lcov
rm -fr wamr-lcov
echo "Code coverage file ${DST_COV_FILE} was generated or appended"
echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated"
else
echo "generate code coverage html failed"
fi
echo ""
popd > /dev/null 2>&1

77
tests/wamr-test-suites/spec-test-script/runtest.py
View File

@ -38,7 +38,7 @@ log_file = None
temp_file_repo = []
# to save the mapping of module files in /tmp by name
temp_module_table = {}
temp_module_table = {}
def debug(data):
if debug_file:
@ -230,6 +230,9 @@ parser.add_argument('--multi-module', default=False, action='store_true',
parser.add_argument('--multi-thread', default=False, action='store_true',
help="Enable Multi-thread")
parser.add_argument('--gc', default=False, action='store_true',
help='Test with GC')
parser.add_argument('--qemu', default=False, action='store_true',
help="Enable QEMU")
@ -420,11 +423,20 @@ def parse_simple_const_w_type(number, type):
number = float.fromhex(number) if '0x' in number else float(number)
return number, "{:.7g}:{}".format(number, type)
elif type == "ref.null":
# hard coding
return "extern", "extern:ref.null"
if number == "func":
return "func", "func:ref.null"
elif number == "extern":
return "extern", "extern:ref.null"
elif number == "any":
return "any", "any:ref.null"
else:
raise Exception("invalid value {} and type {}".format(number, type))
elif type == "ref.extern":
number = int(number, 16) if '0x' in number else int(number)
return number, "0x{:x}:ref.extern".format(number)
elif type == "ref.host":
number = int(number, 16) if '0x' in number else int(number)
return number, "0x{:x}:ref.host".format(number)
else:
raise Exception("invalid value {} and type {}".format(number, type))
@ -440,6 +452,10 @@ def parse_assertion_value(val):
type.const val
ref.extern val
ref.null ref_type
ref.array
ref.struct
ref.func
ref.i31
"""
if not val:
return None, ""
@ -453,6 +469,8 @@ def parse_assertion_value(val):
if type in ["i32", "i64", "f32", "f64"]:
return parse_simple_const_w_type(numbers[0], type)
elif type == "ref":
if splitted[0] in ["ref.array", "ref.struct", "ref.func", "ref.i31"]:
return splitted[0]
# need to distinguish between "ref.null" and "ref.extern"
return parse_simple_const_w_type(numbers[0], splitted[0])
else:
@ -615,6 +633,9 @@ def simple_value_comparison(out, expected):
elif "ref.extern" == expected_type:
out_val_binary = out_val
expected_val_binary = expected_val
elif "ref.host" == expected_type:
out_val_binary = out_val
expected_val_binary = expected_val
else:
assert(0), "unknown 'expected_type' {}".format(expected_type)
@ -637,8 +658,10 @@ def value_comparison(out, expected):
if not expected:
return False
assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out)
assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected)
if not out in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]:
assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out)
if not expected in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]:
assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected)
if 'v128' in out:
return vector_value_comparison(out, expected)
@ -761,6 +784,9 @@ def test_assert_return(r, opts, form):
elif "ref.extern" == splitted[0]:
number, _ = parse_simple_const_w_type(splitted[1], splitted[0])
args.append(str(number))
elif "ref.host" == splitted[0]:
number, _ = parse_simple_const_w_type(splitted[1], splitted[0])
args.append(str(number))
else:
assert(0), "an unkonwn parameter type"
@ -769,7 +795,15 @@ def test_assert_return(r, opts, form):
else:
returns = re.split("\)\s*\(", m.group(3)[1:-1])
# processed numbers in strings
expected = [parse_assertion_value(v)[1] for v in returns]
if len(returns) == 1 and returns[0] in ["ref.array", "ref.struct", "ref.i31",
"ref.eq", "ref.any", "ref.extern",
"ref.func", "ref.null"]:
expected = [returns[0]]
elif len(returns) == 1 and returns[0] in ["func:ref.null", "any:ref.null",
"extern:ref.null"]:
expected = [returns[0]]
else:
expected = [parse_assertion_value(v)[1] for v in returns]
expected = ",".join(expected)
test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected)
@ -800,10 +834,10 @@ def test_assert_return(r, opts, form):
if n.group(3) == '':
args=[]
else:
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n.group(3)[1:-1])]
# a workaround for "ref.null extern" and "ref.null func"
args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args]
# convert (ref.null extern/func) into (ref.null null)
n1 = n.group(3).replace("(ref.null extern)", "(ref.null null)")
n1 = n1.replace("ref.null func)", "(ref.null null)")
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n1[1:-1])]
_, expected = parse_assertion_value(n.group(4)[1:-1])
test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected)
@ -828,10 +862,10 @@ def test_assert_trap(r, opts, form):
if m.group(2) == '':
args = []
else:
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])]
# workaround for "ref.null extern"
args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args]
# convert (ref.null extern/func) into (ref.null null)
m1 = m.group(2).replace("(ref.null extern)", "(ref.null null)")
m1 = m1.replace("ref.null func)", "(ref.null null)")
args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m1[1:-1])]
expected = "Exception: %s" % m.group(3)
test_assert(r, opts, "trap", "%s %s" % (func, " ".join(args)), expected)
@ -918,10 +952,11 @@ def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts):
log("Compiling WASM to '%s'" % wasm_tempfile)
# default arguments
cmd = [opts.wast2wasm,
"--enable-thread",
"--no-check",
wast_tempfile, "-o", wasm_tempfile ]
if opts.gc:
cmd = [opts.wast2wasm, "-u", "-d", wast_tempfile, "-o", wasm_tempfile]
else:
cmd = [opts.wast2wasm, "--enable-thread", "--no-check",
wast_tempfile, "-o", wasm_tempfile ]
# remove reference-type and bulk-memory enabling options since a WABT
# commit 30c1e983d30b33a8004b39fd60cbd64477a7956c
@ -1023,18 +1058,18 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
if (r != None):
r.cleanup()
r = Runner(cmd, no_pty=opts.no_pty)
if opts.qemu:
r.read_to_prompt(['nsh> '], 10)
r.writeline("mount -t hostfs -o fs={} /tmp".format(tempfile.gettempdir()))
r.read_to_prompt(['nsh> '], 10)
r.writeline(" ".join(cmd_iwasm))
return r
def create_tmpfiles(wast_name):
tempfiles = []
(t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
(t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
tempfiles.append(wast_tempfile)