Update cmake files and wamr-test-suites to support collect code coverage (#1992)

Support collecting code coverage with wamr-test-suites script by using
lcov and genhtml tools, eg.:
  cd tests/wamr-test-suites
  ./test_wamr.sh -s spec -b -P -C

The default code coverage and html files are generated at:
  tests/wamr-test-suites/workspace/wamr.lcov
  tests/wamr-test-suites/workspace/wamr-lcov.zip

And update wamr-test-suites scripts to support testing GC spec cases to
avoid frequent synchronization conflicts between branch main and dev/gc.
This commit is contained in:
Wenyong Huang
2023-02-28 17:38:18 +08:00
committed by GitHub
parent b4f0228497
commit 9b9ae0cfac
14 changed files with 342 additions and 116 deletions

View File

@ -14,6 +14,18 @@ import time
"""
The script itself has to be put under the same directory with the "spec".
To run a single non-GC case with interpreter mode:
cd workspace
python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
spec/test/core/xxx.wast
To run a single non-GC case with aot mode:
cd workspace
python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
--aot-compiler wamrc spec/test/core/xxx.wast
To run a single GC case:
cd workspace
python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \
--aot-compiler wamrc --gc spec/test/core/xxx.wast
"""
PLATFORM_NAME = os.uname().sysname.lower()
@ -22,9 +34,9 @@ IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
IWASM_QEMU_CMD = "iwasm"
SPEC_TEST_DIR = "spec/test/core"
WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
class TargetAction(argparse.Action):
TARGET_MAP = {
"ARMV7_VFP": "armv7",
@ -51,6 +63,7 @@ def ignore_the_case(
multi_module_flag=False,
multi_thread_flag=False,
simd_flag=False,
gc_flag=False,
xip_flag=False,
qemu_flag=False
):
@ -63,6 +76,10 @@ def ignore_the_case(
if "i386" == target and case_name in ["float_exprs"]:
return True
if gc_flag:
if case_name in ["type-canon", "type-equivalence", "type-rec"]:
return True;
if sgx_flag:
if case_name in ["conversions", "f32_bitwise", "f64_bitwise"]:
return True
@ -76,7 +93,9 @@ def ignore_the_case(
return True
if qemu_flag:
if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", "conversions", "f32", "f32_cmp", "float_exprs", "float_misc", "select", "memory_grow"]:
if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp",
"conversions", "f32", "f32_cmp", "float_exprs",
"float_misc", "select", "memory_grow"]:
return True
return False
@ -109,6 +128,7 @@ def test_case(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
gc_flag=False,
qemu_flag=False,
qemu_firmware='',
log='',
@ -124,6 +144,7 @@ def test_case(
multi_module_flag,
multi_thread_flag,
simd_flag,
gc_flag,
xip_flag,
qemu_flag
):
@ -131,7 +152,7 @@ def test_case(
CMD = ["python3", "runtest.py"]
CMD.append("--wast2wasm")
CMD.append(WAST2WASM_CMD)
CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD)
CMD.append("--interpreter")
if sgx_flag:
CMD.append(IWASM_SGX_CMD)
@ -171,6 +192,9 @@ def test_case(
if not clean_up_flag:
CMD.append("--no_cleanup")
if gc_flag:
CMD.append("--gc")
if log != '':
CMD.append("--log-dir")
CMD.append(log)
@ -231,6 +255,7 @@ def test_suite(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
gc_flag=False,
parl_flag=False,
qemu_flag=False,
qemu_firmware='',
@ -246,6 +271,10 @@ def test_suite(
simd_case_list = sorted(suite_path.glob("simd/*.wast"))
case_list.extend(simd_case_list)
if gc_flag:
gc_case_list = sorted(suite_path.glob("gc/*.wast"))
case_list.extend(gc_case_list)
case_count = len(case_list)
failed_case = 0
successful_case = 0
@ -268,6 +297,7 @@ def test_suite(
xip_flag,
clean_up_flag,
verbose_flag,
gc_flag,
qemu_flag,
qemu_firmware,
log,
@ -304,6 +334,7 @@ def test_suite(
xip_flag,
clean_up_flag,
verbose_flag,
gc_flag,
qemu_flag,
qemu_firmware,
log,
@ -414,6 +445,13 @@ def main():
dest="verbose_flag",
help="Close real time output while running cases, only show last words of failed ones",
)
parser.add_argument(
"--gc",
action="store_true",
default=False,
dest="gc_flag",
help="Running with GC feature",
)
parser.add_argument(
"cases",
metavar="path_to__case",
@ -446,6 +484,7 @@ def main():
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.gc_flag,
options.parl_flag,
options.qemu_flag,
options.qemu_firmware,
@ -469,6 +508,7 @@ def main():
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.gc_flag,
options.qemu_flag,
options.qemu_firmware,
options.log