Add benchmarks (#979)

Add scripts to build and run benchmark of coremark, polybench, sightglass and jetstream2. And add documents.
2022-01-25 10:52:48 +08:00
parent d925369a1f
commit 4bc6074273
15 changed files with 595 additions and 0 deletions
--- a/tests/benchmarks/sightglass/README.md
+++ b/tests/benchmarks/sightglass/README.md
@ -0,0 +1,21 @@
+# Introduction
+
+[Sightglass](https://github.com/bytecodealliance/sightglass) is a benchmarking suite and tooling to test WebAssembly applications.
+
+**Source**: https://github.com/bytecodealliance/sightglass
+
+# Building
+
+Please build iwasm and wamrc, refer to:
+- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos)
+- [build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler)
+
+And install WASI SDK, please download the [wasi-sdk release](https://github.com/CraneStation/wasi-sdk/releases) and extract the archive to default path `/opt/wasi-sdk`.
+
+And then run `./build.sh` to build the source code, the folder `out` will be created and files will be generated under it.
+
+# Running
+
+Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated.
+
+Run `./run_interp.sh` to test the benchmark, the native mode and iwasm interpreter mode will be tested for each workload, and the file `report.txt` will be generated.
--- a/tests/benchmarks/sightglass/build.sh
+++ b/tests/benchmarks/sightglass/build.sh
@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+OUT_DIR=$PWD/out
+WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc
+SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \
+                nestedloop2 nestedloop3 random seqhash sieve strchr \
+                switch2"
+
+if [ ! -d sightglass ]; then
+    git clone https://github.com/wasm-micro-runtime/sightglass.git
+fi
+
+mkdir -p ${OUT_DIR}
+
+cd sightglass/benchmarks/shootout
+
+for bench in $SHOOTOUT_CASES
+do
+    echo "Build ${bench}_native"
+    gcc -O3 -o ${OUT_DIR}/${bench}_native -Dblack_box=set_res -Dbench=${bench} \
+        -I../../include ${bench}.c main/main_${bench}.c main/my_libc.c
+
+    echo "Build ${bench}.wasm"
+    /opt/wasi-sdk/bin/clang -O3 -nostdlib \
+        -Wno-unknown-attributes \
+        -Dblack_box=set_res \
+        -I../../include -DNOSTDLIB_MODE \
+        -Wl,--initial-memory=1310720,--allow-undefined \
+        -Wl,--strip-all,--no-entry \
+        -o ${OUT_DIR}/${bench}.wasm \
+        -Wl,--export=app_main -Wl,--export=_start \
+        ${bench}.c main/main_${bench}.c main/my_libc.c
+
+
+    echo "Compile ${bench}.wasm into ${bench}.aot"
+    ${WAMRC_CMD} -o ${OUT_DIR}/${bench}.aot ${OUT_DIR}/${bench}.wasm
+done
+
+cd ..
+
+echo "Done"
--- a/tests/benchmarks/sightglass/run_aot.sh
+++ b/tests/benchmarks/sightglass/run_aot.sh
@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+CUR_DIR=$PWD
+OUT_DIR=$CUR_DIR/out
+REPORT=$CUR_DIR/report.txt
+TIME=/usr/bin/time
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+
+BENCH_NAME_MAX_LEN=20
+
+SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \
+                nestedloop2 nestedloop3 random seqhash sieve strchr \
+                switch2"
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+echo "Start to run cases, the result is written to report.txt"
+
+#run benchmarks
+cd $OUT_DIR
+echo -en "\t\t\t\t\t  native\tiwasm-aot\n" >> $REPORT
+
+for t in $SHOOTOUT_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo -en "\n" >> $REPORT
+done
--- a/tests/benchmarks/sightglass/run_interp.sh
+++ b/tests/benchmarks/sightglass/run_interp.sh
@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+CUR_DIR=$PWD
+OUT_DIR=$CUR_DIR/out
+REPORT=$CUR_DIR/report.txt
+TIME=/usr/bin/time
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+
+BENCH_NAME_MAX_LEN=20
+
+SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \
+                nestedloop2 nestedloop3 random seqhash sieve strchr \
+                switch2"
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+echo "Start to run cases, the result is written to report.txt"
+
+#run benchmarks
+cd $OUT_DIR
+echo -en "\t\t\t\t\t  native\tiwasm-interp\n" >> $REPORT
+
+for t in $SHOOTOUT_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo -en "\n" >> $REPORT
+done