Add benchmarks (#979)

Add scripts to build and run benchmark of coremark, polybench, sightglass and jetstream2. And add documents.
2022-01-25 10:52:48 +08:00
parent d925369a1f
commit 4bc6074273
15 changed files with 595 additions and 0 deletions
--- a/tests/benchmarks/jetstream/README.md
+++ b/tests/benchmarks/jetstream/README.md
@ -0,0 +1,29 @@
+# Introduction
+
+[JetStream 2](https://browserbench.org/JetStream) is a JavaScript and WebAssembly benchmark suite focused on the most advanced web applications. It rewards browsers that start up quickly, execute code quickly, and run smoothly.
+
+**Source**: https://browserbench.org/JetStream/in-depth.html
+
+# Building
+
+Please build iwasm and wamrc, refer to:
+- [Build iwasm on Linux](../../../doc/build_wamr.md#linux), or [Build iwasm on MacOS](../../../doc/build_wamr.md#macos)
+- [build wamrc AOT compiler](../../../README.md#build-wamrc-aot-compiler)
+
+And install emsdk, refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate
+ emsdk and set up environment variables. For example, use instructions below to install it under /opt and activate it:
+``` bash
+$ cd /opt
+$ git clone https://github.com/emscripten-core/emsdk.git
+$ cd emsdk
+$ git pull
+$ ./emsdk install latest
+$ ./emsdk activate latest
+$ echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc
+```
+
+And then run `./build.sh` to build the source code, the folder `out` will be created and files will be generated under it.
+
+# Running
+
+Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated.
--- a/tests/benchmarks/jetstream/build.sh
+++ b/tests/benchmarks/jetstream/build.sh
@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+OUT_DIR=$PWD/out
+WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc
+
+mkdir -p jetstream
+mkdir -p ${OUT_DIR}
+
+cd jetstream
+
+echo "Download source files .."
+wget https://browserbench.org/JetStream/wasm/gcc-loops.cpp
+wget https://browserbench.org/JetStream/wasm/quicksort.c
+wget https://browserbench.org/JetStream/wasm/HashSet.cpp
+wget https://browserbench.org/JetStream/simple/float-mm.c
+
+patch -p1 < ../jetstream.patch
+
+echo "Build gcc-loops with g++ .."
+g++ -O3 -msse2 -msse3 -msse4 -o ${OUT_DIR}/gcc-loops_native gcc-loops.cpp
+
+echo "Build gcc-loops with em++ .."
+em++ -O3 -s STANDALONE_WASM=1 -msimd128 \
+         -s INITIAL_MEMORY=1048576 \
+         -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_STACK=32768 \
+         -s "EXPORTED_FUNCTIONS=['_main']" \
+         -o ${OUT_DIR}/gcc-loops.wasm gcc-loops.cpp
+
+echo "Compile gcc-loops.wasm to gcc-loops.aot"
+${WAMRC_CMD} -o ${OUT_DIR}/gcc-loops.aot ${OUT_DIR}/gcc-loops.wasm
+
+echo "Build quicksort with gcc .."
+gcc -O3 -msse2 -msse3 -msse4 -o ${OUT_DIR}/quicksort_native quicksort.c
+
+echo "Build quicksort with emcc .."
+emcc -O3 -s STANDALONE_WASM=1 -msimd128 \
+         -s INITIAL_MEMORY=1048576 \
+         -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_STACK=32768 \
+         -s "EXPORTED_FUNCTIONS=['_main']" \
+         -o ${OUT_DIR}/quicksort.wasm quicksort.c
+
+echo "Compile quicksort.wasm to quicksort.aot"
+${WAMRC_CMD} -o ${OUT_DIR}/quicksort.aot ${OUT_DIR}/quicksort.wasm
+
+echo "Build HashSet with g++ .."
+g++ -O3 -msse2 -msse3 -msse4 -o ${OUT_DIR}/HashSet_native HashSet.cpp \
+        -lstdc++
+
+echo "Build HashSet with em++ .."
+em++ -O3 -s STANDALONE_WASM=1 -msimd128 \
+         -s INITIAL_MEMORY=1048576 \
+         -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_STACK=32768 \
+         -s "EXPORTED_FUNCTIONS=['_main']" \
+         -o ${OUT_DIR}/HashSet.wasm HashSet.cpp
+
+echo "Compile HashSet.wasm to HashSet.aot"
+${WAMRC_CMD} -o ${OUT_DIR}/HashSet.aot ${OUT_DIR}/HashSet.wasm
+
+echo "Build float-mm with gcc .."
+gcc -O3 -msse2 -msse3 -msse4 -o ${OUT_DIR}/float-mm_native float-mm.c
+
+echo "Build float-mm with emcc .."
+emcc -O3 -s STANDALONE_WASM=1 -msimd128 \
+         -s INITIAL_MEMORY=1048576 \
+         -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_STACK=32768 \
+         -s "EXPORTED_FUNCTIONS=['_main']" \
+         -o ${OUT_DIR}/float-mm.wasm float-mm.c
+
+echo "Compile float-mm.wasm to float-mm.aot"
+${WAMRC_CMD} -o ${OUT_DIR}/float-mm.aot ${OUT_DIR}/float-mm.wasm
--- a/tests/benchmarks/jetstream/jetstream.patch
+++ b/tests/benchmarks/jetstream/jetstream.patch
@ -0,0 +1,20 @@
+diff -urN jetstream-org/HashSet.cpp jetstream/HashSet.cpp
+--- jetstream-org/HashSet.cpp	2020-10-30 04:12:42.000000000 +0800
+++ jetstream/HashSet.cpp	2022-01-24 17:11:08.619831711 +0800
+@@ -24,6 +24,7 @@
+ #include <memory>
+ #include <stdio.h>
+ #include <stdlib.h>
+#include <string.h>
+ #include <sys/time.h>
+ 
+ // Compile with: xcrun clang++ -o HashSet HashSet.cpp -O2 -W -framework Foundation -licucore -std=c++11 -fvisibility=hidden -DNDEBUG=1
+@@ -76,7 +77,7 @@
+ inline ToType bitwise_cast(FromType from)
+ {
+     typename std::remove_const<ToType>::type to { };
+-    std::memcpy(&to, &from, sizeof(to));
+    memcpy(&to, &from, sizeof(to));
+     return to;
+ }
+ 
--- a/tests/benchmarks/jetstream/run_aot.sh
+++ b/tests/benchmarks/jetstream/run_aot.sh
@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+CUR_DIR=$PWD
+OUT_DIR=$CUR_DIR/out
+REPORT=$CUR_DIR/report.txt
+TIME=/usr/bin/time
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+
+BENCH_NAME_MAX_LEN=20
+
+JETSTREAM_CASES="gcc-loops quicksort HashSet float-mm"
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+echo "Start to run cases, the result is written to report.txt"
+
+#run benchmarks
+cd $OUT_DIR
+echo -en "\t\t\t\t\t  native\tiwasm-aot\n" >> $REPORT
+
+for t in $JETSTREAM_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo -en "\n" >> $REPORT
+done