Implement AOT static PGO (#2243)

LLVM PGO (Profile-Guided Optimization) allows the compiler to better optimize code for how it actually runs. This PR implements the AOT static PGO, and is tested on Linux x86-64 and x86-32. The basic steps are: 1. Use `wamrc --enable-llvm-pgo -o <aot_file_of_pgo> <wasm_file>` to generate an instrumented aot file. 2. Compile iwasm with `cmake -DWAMR_BUILD_STATIC_PGO=1` and run `iwasm --gen-prof-file=<raw_profile_file> <aot_file_of_pgo>` to generate the raw profile file. 3. Run `llvm-profdata merge -output=<profile_file> <raw_profile_file>` to merge the raw profile file into the profile file. 4. Run `wamrc --use-prof-file=<profile_file> -o <aot_file> <wasm_file>` to generate the optimized aot file. 5. Run the optimized aot_file: `iwasm <aot_file>`. The test scripts are also added for each benchmark, run `test_pgo.sh` under each benchmark's folder to test the AOT static pgo.
2023-06-05 09:17:39 +08:00
parent f1e9029ebc
commit 8d88471c46
29 changed files with 2000 additions and 53 deletions
--- a/tests/benchmarks/README.md
+++ b/tests/benchmarks/README.md
@ -0,0 +1,62 @@
+# WAMR test benchmarks
+
+This folder contains test benchmarks for wamr.
+
+## Build and Run
+
+Refer to the `README.md` under each folder for how to build and run the benchmark.
+
+## Install `llvm-profdata`
+
+The tool `llvm-profdata` is used when running the `test_pgo.sh` script under the benchmark folder. There are two ways to install it:
+
+1. Refer to https://apt.llvm.org/, e.g. in Ubuntu 20.04, add lines below to /etc/apt/source.list
+
+```bash
+deb http://apt.llvm.org/focal/ llvm-toolchain-focal main
+deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main
+# 15
+deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main
+deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main
+```
+
+Then run `sudo apt update`, `sudo apt install llvm`. And after installing:
+
+```bash
+cd /usr/bin
+sudo ln -s llvm-profdata-15 llvm-profdata
+```
+
+2. Build manually
+
+```bash
+git clone --depth 1 --branch release/15.x https://github.com/llvm/llvm-project.git
+cd llvm-project
+mkdir build && cd build
+cmake ../llvm \
+    -DCMAKE_BUILD_TYPE:STRING="Release" \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+    -DLLVM_APPEND_VC_REV:BOOL=ON \
+    -DLLVM_BUILD_EXAMPLES:BOOL=OFF \
+    -DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF \
+    -DLLVM_BUILD_TESTS:BOOL=OFF \
+    -DLLVM_CCACHE_BUILD:BOOL=ON \
+    -DLLVM_ENABLE_BINDINGS:BOOL=OFF \
+    -DLLVM_ENABLE_IDE:BOOL=OFF \
+    -DLLVM_ENABLE_LIBEDIT=OFF \
+    -DLLVM_ENABLE_TERMINFO:BOOL=OFF \
+    -DLLVM_ENABLE_ZLIB:BOOL=ON \
+    -DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF \
+    -DLLVM_INCLUDE_DOCS:BOOL=OFF \
+    -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF \
+    -DLLVM_INCLUDE_UTILS:BOOL=OFF \
+    -DLLVM_INCLUDE_TESTS:BOOL=OFF \
+    -DLLVM_BUILD_TESTS:BOOL=OFF \
+    -DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON \
+    -DLLVM_ENABLE_LIBXML2:BOOL=OFF \
+    -DLLVM_TARGETS_TO_BUILD:STRING="X86" \
+    -DLLVM_INCLUDE_TOOLS:BOOL=ON \
+    -G'Ninja'
+ninja -j 8
+# tool `llvm-profdata` is generated under this folder.
+```
--- a/tests/benchmarks/coremark/README.md
+++ b/tests/benchmarks/coremark/README.md
@ -17,3 +17,5 @@ And then run `./build.sh` to build the source code, file `coremark.exe`, `corema
 # Running

 Run `./run.sh` to test the benchmark, the native mode, iwasm aot mode and iwasm interpreter mode will be tested respectively.
+
+Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`.
--- a/tests/benchmarks/coremark/test_pgo.sh
+++ b/tests/benchmarks/coremark/test_pgo.sh
@ -0,0 +1,50 @@
+#!/bin/sh
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+
+IWASM="../../../product-mini/platforms/${PLATFORM}/build/iwasm"
+WAMRC="../../../wamr-compiler/build/wamrc"
+
+if [ ! -e "coremark.wasm" ]; then
+    echo "coremark.wasm doesn't exist, please run build.sh first"
+    exit
+fi
+
+echo ""
+echo "Compile coremark.wasm to coremark.aot .."
+${WAMRC} -o coremark.aot coremark.wasm
+
+echo ""
+echo "Compile coremark.wasm to coremark_pgo.aot .."
+${WAMRC} --enable-llvm-pgo -o coremark_pgo.aot coremark.wasm
+
+echo ""
+echo "Run coremark_pgo.aot to generate the raw profile data .."
+${IWASM} --gen-prof-file=coremark.profraw coremark_pgo.aot
+
+echo ""
+echo "Merge the raw profile data to coremark.profdata .."
+rm -f coremark.profdata && llvm-profdata merge -output=coremark.profdata coremark.profraw
+
+echo ""
+echo "Compile coremark.wasm to coremark_opt.aot with the profile data .."
+${WAMRC} --use-prof-file=coremark.profdata -o coremark_opt.aot coremark.wasm
+
+echo ""
+echo "Run the coremark native"
+./coremark.exe
+
+echo ""
+echo "Run the original aot file coremark.aot"
+${IWASM} coremark.aot
+
+echo ""
+echo "Run the PGO optimized aot file coremark_opt.aot"
+${IWASM} coremark_opt.aot
+
+# Show the profile data:
+# llvm-profdata show --all-functions --detailed-summary --binary-ids --counts \
+# --hot-func-list --memop-sizes --show-prof-sym-list coremark.profraw
--- a/tests/benchmarks/dhrystone/test_pgo.sh
+++ b/tests/benchmarks/dhrystone/test_pgo.sh
@ -0,0 +1,50 @@
+#!/bin/sh
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+
+IWASM="../../../product-mini/platforms/${PLATFORM}/build/iwasm"
+WAMRC="../../../wamr-compiler/build/wamrc"
+
+if [ ! -e "dhrystone.wasm" ]; then
+    echo "dhrystone.wasm doesn't exist, please run build.sh first"
+    exit
+fi
+
+echo ""
+echo "Compile dhrystone.wasm to dhrystone.aot .."
+${WAMRC} -o dhrystone.aot dhrystone.wasm
+
+echo ""
+echo "Compile dhrystone.wasm to dhrystone_pgo.aot .."
+${WAMRC} --enable-llvm-pgo -o dhrystone_pgo.aot dhrystone.wasm
+
+echo ""
+echo "Run dhrystone_pgo.aot to generate the raw profile data .."
+${IWASM} --gen-prof-file=dhrystone.profraw dhrystone_pgo.aot
+
+echo ""
+echo "Merge the raw profile data to dhrystone.profdata .."
+rm -f dhrystone.profdata && llvm-profdata merge -output=dhrystone.profdata dhrystone.profraw
+
+echo ""
+echo "Compile dhrystone.wasm to dhrystone_opt.aot with the profile data .."
+${WAMRC} --use-prof-file=dhrystone.profdata -o dhrystone_opt.aot dhrystone.wasm
+
+echo ""
+echo "Run the dhrystone native"
+./dhrystone_native
+
+echo ""
+echo "Run the original aot file dhrystone.aot"
+${IWASM} dhrystone.aot
+
+echo ""
+echo "Run the PGO optimized aot file dhrystone_opt.aot"
+${IWASM} dhrystone_opt.aot
+
+# Show the profile data:
+# llvm-profdata show --all-functions --detailed-summary --binary-ids --counts \
+# --hot-func-list --memop-sizes --show-prof-sym-list dhrystone.profraw
--- a/tests/benchmarks/jetstream/README.md
+++ b/tests/benchmarks/jetstream/README.md
@ -27,3 +27,5 @@ And then run `./build.sh` to build the source code, the folder `out` will be cre
 # Running

 Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated.
+
+Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`.
--- a/tests/benchmarks/jetstream/test_pgo.sh
+++ b/tests/benchmarks/jetstream/test_pgo.sh
@ -0,0 +1,87 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+CUR_DIR=$PWD
+OUT_DIR=$CUR_DIR/out
+REPORT=$CUR_DIR/report.txt
+TIME=/usr/bin/time
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+WAMRC_CMD=$CUR_DIR/../../../wamr-compiler/build/wamrc
+
+BENCH_NAME_MAX_LEN=20
+
+JETSTREAM_CASES="gcc-loops HashSet tsf float-mm quicksort"
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+pushd $OUT_DIR > /dev/null 2>&1
+for t in $JETSTREAM_CASES
+do
+    if [ ! -e "${t}.wasm" ]; then
+        echo "${t}.wasm doesn't exist, please run build.sh first"
+        exit
+    fi
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}.aot .."
+    ${WAMRC_CMD} -o ${t}.aot ${t}.wasm
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_pgo.aot .."
+    ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm
+
+    echo ""
+    echo "Run ${t}_pgo.aot to generate the raw profile data .."
+    ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot
+
+    echo ""
+    echo "Merge the raw profile data to ${t}.profdata .."
+    rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .."
+    ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm
+done
+popd > /dev/null 2>&1
+
+echo "Start to run cases, the result is written to report.txt"
+
+#run benchmarks
+cd $OUT_DIR
+echo -en "\t\t\t\t\t  native\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT
+
+for t in $JETSTREAM_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD --dir=. ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot opt .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD --dir=. ${t}_opt.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo -en "\n" >> $REPORT
+done
--- a/tests/benchmarks/libsodium/test_pgo.sh
+++ b/tests/benchmarks/libsodium/test_pgo.sh
@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chacha20poly1305 \
+                 aead_xchacha20poly1305 auth2 auth3 auth5 auth6 auth7 auth box2 box7 box8 \
+                 box_easy2 box_easy box_seal box_seed box chacha20 codecs core1 core2 core3 \
+                 core4 core5 core6 core_ed25519 core_ristretto255 ed25519_convert generichash2 \
+                 generichash3 generichash hash3 hash kdf keygen kx metamorphic misuse \
+                 onetimeauth2 onetimeauth7 onetimeauth pwhash_argon2id pwhash_argon2i \
+                 pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \
+                 scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \
+                 scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \
+                 secretbox secretstream shorthash sign siphashx24 sodium_core sodium_utils2 \
+                 sodium_utils stream2 stream3 stream4 stream verify1 xchacha20"
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+
+readonly OUT_DIR=$PWD/libsodium/zig-out/bin
+readonly REPORT=$PWD/report.txt
+readonly IWASM_CMD=$PWD/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+readonly WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc
+readonly TIME=/usr/bin/time
+
+BENCH_NAME_MAX_LEN=20
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+pushd $OUT_DIR > /dev/null 2>&1
+for t in $libsodium_CASES
+do
+    if [ ! -e "${t}.wasm" ]; then
+        echo "${t}.wasm doesn't exist, please run build.sh first"
+        exit
+    fi
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}.aot .."
+    ${WAMRC_CMD} -o ${t}.aot ${t}.wasm
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_pgo.aot .."
+    ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm
+
+    echo ""
+    echo "Run ${t}_pgo.aot to generate the raw profile data .."
+    ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot
+
+    echo ""
+    echo "Merge the raw profile data to ${t}.profdata .."
+    rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .."
+    ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm
+done
+
+# run benchmarks
+cd $OUT_DIR
+
+echo -en "\t\t\t\t\t\tnative\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT
+
+for t in $libsodium_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native..."
+    echo -en "\t" >> $REPORT
+    if [[ $t != "sodium_utils2" ]]; then
+        ./${t} | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT
+    else
+        # sodium_utils2 doesn't print the result,
+        # use time command to get result instead
+        $TIME -f "real-%e-time" ./${t} 2>&1 | grep "real-.*-time" |
+            awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT
+    fi
+
+    echo "run $t with iwasm aot..."
+    echo -en "\t  \t" >> $REPORT
+    if [[ $t != "sodium_utils2" ]]; then
+        $IWASM_CMD ${t}.aot | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT
+    else
+        # sodium_utils2 doesn't print the result,
+        # use time command to get result instead
+        $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" |
+            awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT
+    fi
+
+    echo "run $t with iwasm aot opt..."
+    echo -en "\t  \t" >> $REPORT
+    if [[ $t != "sodium_utils2" ]]; then
+        $IWASM_CMD ${t}_opt.aot | awk '{printf "%-10.2f", $0/1000000.0}' >> $REPORT
+    else
+        # sodium_utils2 doesn't print the result,
+        # use time command to get result instead
+        $TIME -f "real-%e-time" $IWASM_CMD ${t}_opt.aot 2>&1 | grep "real-.*-time" |
+            awk -F '-' '{printf "%-10.2f", $2}' >> $REPORT
+    fi
+
+    echo -en "\n" >> $REPORT
+done
+
--- a/tests/benchmarks/polybench/test_pgo.sh
+++ b/tests/benchmarks/polybench/test_pgo.sh
@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+CUR_DIR=$PWD
+OUT_DIR=$CUR_DIR/out
+REPORT=$CUR_DIR/report.txt
+TIME=/usr/bin/time
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+WAMRC_CMD=$CUR_DIR/../../../wamr-compiler/build/wamrc
+
+BENCH_NAME_MAX_LEN=20
+
+POLYBENCH_CASES="2mm 3mm adi atax bicg cholesky correlation covariance \
+                 deriche doitgen durbin fdtd-2d floyd-warshall gemm gemver \
+                 gesummv gramschmidt heat-3d jacobi-1d jacobi-2d ludcmp lu \
+                 mvt nussinov seidel-2d symm syr2k syrk trisolv trmm"
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+pushd $OUT_DIR > /dev/null 2>&1
+for t in $POLYBENCH_CASES
+do
+    if [ ! -e "${t}.wasm" ]; then
+        echo "${t}.wasm doesn't exist, please run build.sh first"
+        exit
+    fi
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}.aot .."
+    ${WAMRC_CMD} -o ${t}.aot ${t}.wasm
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_pgo.aot .."
+    ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm
+
+    echo ""
+    echo "Run ${t}_pgo.aot to generate the raw profile data .."
+    ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot
+
+    echo ""
+    echo "Merge the raw profile data to ${t}.profdata .."
+    rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .."
+    ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm
+done
+popd > /dev/null 2>&1
+
+echo "Start to run cases, the result is written to report.txt"
+
+#run benchmarks
+cd $OUT_DIR
+echo -en "\t\t\t\t\t  native\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT
+
+for t in $POLYBENCH_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot opt .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}_opt.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo -en "\n" >> $REPORT
+done
--- a/tests/benchmarks/sightglass/README.md
+++ b/tests/benchmarks/sightglass/README.md
@ -19,3 +19,5 @@ And then run `./build.sh` to build the source code, the folder `out` will be cre
 Run `./run_aot.sh` to test the benchmark, the native mode and iwasm aot mode will be tested for each workload, and the file `report.txt` will be generated.

 Run `./run_interp.sh` to test the benchmark, the native mode and iwasm interpreter mode will be tested for each workload, and the file `report.txt` will be generated.
+
+Run `./test_pgo.sh` to test the benchmark with AOT static PGO (Profile-Guided Optimization) enabled, please refer [here](../README.md#install-llvm-profdata) to install tool `llvm-profdata` and build `iwasm` with `cmake -DWAMR_BUILD_STATIC_PGO=1`.
--- a/tests/benchmarks/sightglass/test_pgo.sh
+++ b/tests/benchmarks/sightglass/test_pgo.sh
@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+CUR_DIR=$PWD
+OUT_DIR=$CUR_DIR/out
+REPORT=$CUR_DIR/report.txt
+TIME=/usr/bin/time
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+IWASM_CMD=$CUR_DIR/../../../product-mini/platforms/${PLATFORM}/build/iwasm
+WAMRC_CMD=$CUR_DIR/../../../wamr-compiler/build/wamrc
+
+BENCH_NAME_MAX_LEN=20
+
+SHOOTOUT_CASES="base64 fib2 gimli heapsort matrix memmove nestedloop \
+                nestedloop2 nestedloop3 random seqhash sieve strchr \
+                switch2"
+
+rm -f $REPORT
+touch $REPORT
+
+function print_bench_name()
+{
+    name=$1
+    echo -en "$name" >> $REPORT
+    name_len=${#name}
+    if [ $name_len -lt $BENCH_NAME_MAX_LEN ]
+    then
+        spaces=$(( $BENCH_NAME_MAX_LEN - $name_len ))
+        for i in $(eval echo "{1..$spaces}"); do echo -n " " >> $REPORT; done
+    fi
+}
+
+pushd $OUT_DIR > /dev/null 2>&1
+for t in $SHOOTOUT_CASES
+do
+    if [ ! -e "${t}.wasm" ]; then
+        echo "${t}.wasm doesn't exist, please run build.sh first"
+        exit
+    fi
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}.aot .."
+    ${WAMRC_CMD} -o ${t}.aot ${t}.wasm
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_pgo.aot .."
+    ${WAMRC_CMD} --enable-llvm-pgo -o ${t}_pgo.aot ${t}.wasm
+
+    echo ""
+    echo "Run ${t}_pgo.aot to generate the raw profile data .."
+    ${IWASM_CMD} --gen-prof-file=${t}.profraw --dir=. ${t}_pgo.aot
+
+    echo ""
+    echo "Merge the raw profile data to ${t}.profdata .."
+    rm -f ${t}.profdata && llvm-profdata merge -output=${t}.profdata ${t}.profraw
+
+    echo ""
+    echo "Compile ${t}.wasm to ${t}_opt.aot with the profile data .."
+    ${WAMRC_CMD} --use-prof-file=${t}.profdata -o ${t}_opt.aot ${t}.wasm
+done
+popd > /dev/null 2>&1
+
+echo "Start to run cases, the result is written to report.txt"
+
+#run benchmarks
+cd $OUT_DIR
+echo -en "\t\t\t\t\t  native\tiwasm-aot\tiwasm-aot-pgo\n" >> $REPORT
+
+for t in $SHOOTOUT_CASES
+do
+    print_bench_name $t
+
+    echo "run $t with native .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" ./${t}_native 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo "run $t with iwasm aot opt .."
+    echo -en "\t" >> $REPORT
+    $TIME -f "real-%e-time" $IWASM_CMD ${t}_opt.aot 2>&1 | grep "real-.*-time" | awk -F '-' '{ORS=""; print $2}' >> $REPORT
+
+    echo -en "\n" >> $REPORT
+done