From 2f530e67fceb192eef14dbf759bf1dcec698f674 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 4 Dec 2020 15:35:45 +0800 Subject: [PATCH] Fix app manager install atomics app issue and optimize workload scripts (#458) --- core/app-framework/base/app/bh_platform.h | 14 ++ core/app-mgr/app-manager/module_wasm_app.c | 13 +- core/shared/platform/zephyr/zephyr_thread.c | 6 +- samples/workload/README.md | 4 + samples/workload/XNNPACK/.gitignore | 2 + samples/workload/XNNPACK/docker_build.sh | 1 + samples/workload/bwa/docker_build.sh | 1 + samples/workload/docker/Dockerfile | 22 ++- .../docker/{build.sh => docker_build.sh} | 28 +++- samples/workload/docker/run.sh | 10 -- .../workload/meshoptimizer/docker_build.sh | 1 + samples/workload/preparation.sh | 129 ++++++++++++++++++ samples/workload/wasm-av1/docker_build.sh | 1 + 13 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 samples/workload/XNNPACK/.gitignore create mode 120000 samples/workload/XNNPACK/docker_build.sh create mode 120000 samples/workload/bwa/docker_build.sh rename samples/workload/docker/{build.sh => docker_build.sh} (69%) delete mode 100755 samples/workload/docker/run.sh create mode 120000 samples/workload/meshoptimizer/docker_build.sh create mode 100755 samples/workload/preparation.sh create mode 120000 samples/workload/wasm-av1/docker_build.sh diff --git a/core/app-framework/base/app/bh_platform.h b/core/app-framework/base/app/bh_platform.h index 2539ec58..70760e9c 100755 --- a/core/app-framework/base/app/bh_platform.h +++ b/core/app-framework/base/app/bh_platform.h @@ -45,4 +45,18 @@ uint16 ntohs(uint16 value); // We are not worried for the WASM world since the sandbox will catch it. #define bh_memcpy_s(dst, dst_len, src, src_len) memcpy(dst, src, src_len) +#ifdef NDEBUG +#define bh_assert(v) (void)0 +#else +#define bh_assert(v) do { \ + if (!(v)) { \ + int _count; \ + printf("ASSERTION FAILED: %s, at %s, line %d",\ + #v, __FILE__, __LINE__); \ + _count = printf("\n"); \ + printf("%d\n", _count / (_count - 1)); \ + } \ + } while (0) +#endif + #endif /* DEPS_IWASM_APP_LIBS_BASE_BH_PLATFORM_H_ */ diff --git a/core/app-mgr/app-manager/module_wasm_app.c b/core/app-mgr/app-manager/module_wasm_app.c index a84e760d..6fda5355 100644 --- a/core/app-mgr/app-manager/module_wasm_app.c +++ b/core/app-mgr/app-manager/module_wasm_app.c @@ -703,7 +703,11 @@ wasm_app_module_install(request_t * msg) SECTION_TYPE_GLOBAL, SECTION_TYPE_EXPORT, SECTION_TYPE_START, - SECTION_TYPE_ELEM + SECTION_TYPE_ELEM, +#if WASM_ENABLE_BULK_MEMORY != 0 + SECTION_TYPE_DATACOUNT +#endif + }; /* Sections to be released after instantiating */ uint8 sections2[] = { SECTION_TYPE_DATA }; @@ -1174,7 +1178,12 @@ wasm_app_module_on_install_request_byte_arrive(uint8 ch, } else if (recv_ctx.phase == Phase_Wasm_Section_Type) { uint8 section_type = ch; - if (section_type <= SECTION_TYPE_DATA) { +#if WASM_ENABLE_BULK_MEMORY == 0 + uint8 section_type_max = SECTION_TYPE_DATA; +#else + uint8 section_type_max = SECTION_TYPE_DATACOUNT; +#endif + if (section_type <= section_type_max) { wasm_section_t *new_section; if (!(new_section = (wasm_section_t *) APP_MGR_MALLOC(sizeof(wasm_section_t)))) { app_manager_printf("Allocate memory failed!\n"); diff --git a/core/shared/platform/zephyr/zephyr_thread.c b/core/shared/platform/zephyr/zephyr_thread.c index ebcb37b4..56032f97 100644 --- a/core/shared/platform/zephyr/zephyr_thread.c +++ b/core/shared/platform/zephyr/zephyr_thread.c @@ -8,13 +8,9 @@ #define bh_assert(v) do { \ if (!(v)) { \ - int _count; \ printf("\nASSERTION FAILED: %s, at %s, line %d\n", \ #v, __FILE__, __LINE__); \ - _count = printf(" "); \ - /* divived by 0 to make it abort */ \ - printf("%d\n", _count / (_count - 1)); \ - while (1); \ + abort(); \ } \ } while (0) diff --git a/samples/workload/README.md b/samples/workload/README.md index f32fb4b9..2af6ec6b 100644 --- a/samples/workload/README.md +++ b/samples/workload/README.md @@ -6,6 +6,10 @@ Ubuntu 18.04 as an example. ## Installation instructions +use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload. + +for details, the script includes below steps: + - **wasi-sdk**. Install [latest release](https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sdk-11.0-linux.tar.gz) in */opt/wasi-sdk* or */opt/wasi-sdk-11* diff --git a/samples/workload/XNNPACK/.gitignore b/samples/workload/XNNPACK/.gitignore new file mode 100644 index 00000000..09f3fe92 --- /dev/null +++ b/samples/workload/XNNPACK/.gitignore @@ -0,0 +1,2 @@ +xnnpack +build diff --git a/samples/workload/XNNPACK/docker_build.sh b/samples/workload/XNNPACK/docker_build.sh new file mode 120000 index 00000000..3c6de9bc --- /dev/null +++ b/samples/workload/XNNPACK/docker_build.sh @@ -0,0 +1 @@ +../docker/docker_build.sh \ No newline at end of file diff --git a/samples/workload/bwa/docker_build.sh b/samples/workload/bwa/docker_build.sh new file mode 120000 index 00000000..3c6de9bc --- /dev/null +++ b/samples/workload/bwa/docker_build.sh @@ -0,0 +1 @@ +../docker/docker_build.sh \ No newline at end of file diff --git a/samples/workload/docker/Dockerfile b/samples/workload/docker/Dockerfile index cb96768c..8afa4738 100644 --- a/samples/workload/docker/Dockerfile +++ b/samples/workload/docker/Dockerfile @@ -23,7 +23,7 @@ RUN cd /opt \ && tar zxf ${WASI_SDK_FILE} \ && rm ${WASI_SDK_FILE} \ && ln -sf /opt/wasi-sdk-${WASI_SDK_VER} /opt/wasi-sdk \ - && ln -sf /opt/wasi-sdk/lib/clang/10.0.0/lib/wasi/ /usr/lib/llvm-11/lib/clang/11.0.0/lib/ + && ln -sf /opt/wasi-sdk/lib/clang/10.0.0/lib/wasi/ /usr/lib/llvm-11/lib/clang/11.0.1/lib/ # # install wabt @@ -46,7 +46,8 @@ RUN cd /tmp \ # # install tools -RUN apt install -y git tree +RUN apt update \ + && apt install -y git tree # # install emsdk @@ -85,4 +86,19 @@ RUN apt-get autoremove -y \ && rm -rf /tmp/* VOLUME /data -WORKDIR /data + +# +# +RUN touch /build.sh \ + && echo "\ +#!/bin/bash \n\ +if [[ -d /data/project/build ]]; then \n\ + rm -r /data/project/build \n\ +fi \n\ +mkdir /data/project/build \n\ +cd /data/project/build \n\ +source /opt/emsdk/emsdk_env.sh \n\ +cmake .. \n\ +make \n\ +cd - > /dev/null" > /build.sh \ + && chmod a+x /build.sh diff --git a/samples/workload/docker/build.sh b/samples/workload/docker/docker_build.sh similarity index 69% rename from samples/workload/docker/build.sh rename to samples/workload/docker/docker_build.sh index d14606c8..a5fb54bb 100755 --- a/samples/workload/docker/build.sh +++ b/samples/workload/docker/docker_build.sh @@ -5,8 +5,10 @@ #!/bin/bash -if [[ ! -d build_scripts ]]; then - mkdir build_scripts +BUILD_CONTENT="/tmp/build_content" + +if [[ ! -d ${BUILD_CONTENT} ]]; then + mkdir ${BUILD_CONTENT} fi WASI_SDK_VER=11.0 @@ -15,7 +17,7 @@ CMAKE_VER=3.16.2 BINARYEN_VER=version_97 BAZEL_VER=3.7.0 -cd build_scripts +cd ${BUILD_CONTENT} if [[ ! -f wasi-sdk-${WASI_SDK_VER}-linux.tar.gz ]]; then wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sdk-${WASI_SDK_VER}-linux.tar.gz fi @@ -39,16 +41,30 @@ fi if [[ ! -f bazel-${BAZEL_VER}-installer-linux-x86_64.sh ]]; then wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VER}/bazel-${BAZEL_VER}-installer-linux-x86_64.sh fi -cd - +cd - > /dev/null + +DOCKERFILE_PATH=$(dirname $(realpath $0)) docker build \ --build-arg http_proxy=${http_proxy} \ --build-arg https_proxy=${https_proxy} \ --build-arg HTTP_PROXY=${http_proxy} \ --build-arg HTTPS_PROXY=${https_proxy} \ - --build-arg WASI_SDK_VER=11.0 \ + --build-arg WASI_SDK_VER=${WASI_SDK_VER} \ --build-arg WABT_VER=${WABT_VER} \ --build-arg CMAKE_VER=${CMAKE_VER} \ --build-arg BINARYEN_VER=${BINARYEN_VER} \ --build-arg BAZEL_VER=${BAZEL_VER} \ - -t clang_env:0.1 -f Dockerfile build_scripts + -t clang_env:0.1 -f ${DOCKERFILE_PATH}/Dockerfile ${BUILD_CONTENT} + +docker run --rm -it \ + -e http_proxy=${http_proxy} \ + -e https_proxy=${https_proxy} \ + -e HTTP_PROXY=${http_proxy} \ + -e HTTPS_PROXY=${htpps_proxy} \ + --name workload_w_clang \ + --mount type=bind,source=$(pwd),target=/data/project \ + --mount type=bind,source=$(pwd)/../cmake,target=/data/cmake \ + -w /data/project \ + clang_env:0.1 \ + /bin/bash -c /build.sh diff --git a/samples/workload/docker/run.sh b/samples/workload/docker/run.sh deleted file mode 100755 index 8c594302..00000000 --- a/samples/workload/docker/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -docker run --rm -it \ - -e http_proxy=${http_proxy} \ - -e https_proxy=${https_proxy} \ - -e HTTP_PROXY=${http_proxy} \ - -e HTTPS_PROXY=${htpps_proxy} \ - --name workload_w_clang \ - --mount type=bind,source=$(pwd)/..,target=/data \ - clang_env:0.1 diff --git a/samples/workload/meshoptimizer/docker_build.sh b/samples/workload/meshoptimizer/docker_build.sh new file mode 120000 index 00000000..3c6de9bc --- /dev/null +++ b/samples/workload/meshoptimizer/docker_build.sh @@ -0,0 +1 @@ +../docker/docker_build.sh \ No newline at end of file diff --git a/samples/workload/preparation.sh b/samples/workload/preparation.sh new file mode 100755 index 00000000..81b1aea2 --- /dev/null +++ b/samples/workload/preparation.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +readonly BUILD_CONTENT="/tmp/build_content" +readonly WASI_SDK_VER=11.0 +readonly WASI_SDK_FILE="wasi-sdk-${WASI_SDK_VER}-linux.tar.gz" +readonly WABT_VER=1.0.19 +readonly WABT_FILE="wabt-${WABT_VER}-ubuntu.tar.gz" +readonly CMAKE_VER=3.16.2 +readonly CMAKE_FILE="cmake-${CMAKE_VER}-Linux-x86_64.sh" +readonly BINARYEN_VER=version_97 +readonly BINARYEN_FILE="binaryen-${BINARYEN_VER}-x86_64-linux.tar.gz" +readonly BAZEL_VER=3.7.0 +readonly BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh + +function DEBUG() { + [[ -n $(env | grep "\") ]] +} + +# +# install dependency +function install_deps() { + apt update + apt install -y lsb-release wget software-properties-common \ + build-essential git tree zip unzip +} + +# +# install clang +function install_clang() { + if [[ ! -f llvm.sh ]]; then + wget https://apt.llvm.org/llvm.sh + fi + + chmod a+x llvm.sh + ./llvm.sh 11 +} + +# +# install wasi-sdk +function install_wasi-sdk() { + if [[ ! -f ${WASI_SDK_FILE} ]]; then + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/${WASI_SDK_FILE} + fi + + tar zxf ${WASI_SDK_FILE} -C /opt + ln -sf /opt/wasi-sdk-${WASI_SDK_VER} /opt/wasi-sdk + ln -sf /opt/wasi-sdk/lib/clang/10.0.0/lib/wasi/ /usr/lib/llvm-11/lib/clang/11.0.1/lib/ +} + +# +# install wabt +function install_wabt() { + if [[ ! -f ${WABT_FILE} ]]; then + wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/${WABT_FILE} + fi + + tar zxf ${WABT_FILE} -C /opt + ln -sf /opt/wabt-${WABT_VER} /opt/wabt +} + +# +# install cmake +function install_cmake() { + if [[ ! -f cmake-${CMAKE_VER}-Linux-x86_64.sh ]]; then + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/${CMAKE_FILE} + fi + + chmod a+x ${CMAKE_FILE} + mkdir /opt/cmake + ./${CMAKE_FILE} --prefix=/opt/cmake --skip-license + ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake +} + +# +# install emsdk +function install_emsdk() { + cd /opt + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk + git pull + ./emsdk install latest + ./emsdk activate latest + echo "source /opt/emsdk/emsdk_env.sh" >> ${HOME}/.bashrc +} + +# +# install binaryen +function install_binaryen() { + if [[ ! -f ${BINARYEN_FILE} ]]; then + wget https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VER}/${BINARYEN_FILE} + fi + + tar zxf ${BINARYEN_FILE} -C /opt + ln -sf /opt/binaryen-${BINARYEN_VER} /opt/binaryen +} + +# +# install bazel +function install_bazel() { + if [[ ! -f ${BAZEL_FILE} ]]; then + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VER}/${BAZEL_FILE} + fi + + chmod a+x ${BAZEL_FILE} + ./${BAZEL_FILE} +} + +# +# MAIN +DEBUG && set -xevu +if [[ ! -d ${BUILD_CONTENT} ]]; then + mkdir ${BUILD_CONTENT} +fi + +cd ${BUILD_CONTENT} +if DEBUG; then + $@ +else + install_deps \ + && install_clang \ + && install_wasi \ + && install_wabt \ + && install_cmake \ + && install_emsdk \ + && install_binaryen \ + && install_bazel +fi +cd - > /dev/null +DEBUG && set +xevu diff --git a/samples/workload/wasm-av1/docker_build.sh b/samples/workload/wasm-av1/docker_build.sh new file mode 120000 index 00000000..3c6de9bc --- /dev/null +++ b/samples/workload/wasm-av1/docker_build.sh @@ -0,0 +1 @@ +../docker/docker_build.sh \ No newline at end of file