From 365ec6360becd551ffd11cc10022b5069459e0b6 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 29 Dec 2020 00:41:09 -0600 Subject: [PATCH] Update build scripts and documents of workload samples for wasi-sdk-12.0 (#484) Update the build scripts of sample workloads (meshoptimizer/bwa/wasm-av1) to use the wasi-sdk-12.0 firstly to build the workload and discard clang-11, as wasi-sdk-12 supports wasi, simd and pthread better. And update the related documents. Also modify wasm mini loader to sync up with the change of wasm normal loader. Signed-off-by: Wenyong Huang --- core/iwasm/interpreter/wasm_mini_loader.c | 53 ++------- samples/workload/README.md | 30 ++---- samples/workload/XNNPACK/README.md | 3 +- samples/workload/bwa/CMakeLists.bwa_wasm.txt | 1 + samples/workload/bwa/CMakeLists.txt | 37 ++++++- samples/workload/bwa/README.md | 7 +- samples/workload/cmake/toolchain.cmake | 102 ------------------ samples/workload/docker/Dockerfile | 46 ++++---- samples/workload/docker/docker_build.sh | 45 +++----- samples/workload/meshoptimizer/CMakeLists.txt | 37 ++++++- samples/workload/meshoptimizer/README.md | 16 ++- .../workload/meshoptimizer/codecbench.patch | 5 +- samples/workload/preparation.sh | 44 ++++---- samples/workload/tensorflow/README.md | 3 +- samples/workload/wasm-av1/CMakeLists.txt | 37 ++++++- samples/workload/wasm-av1/README.md | 44 ++++---- 16 files changed, 227 insertions(+), 283 deletions(-) delete mode 100644 samples/workload/cmake/toolchain.cmake diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 27085b0a..d082d46a 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -33,42 +33,10 @@ set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) bh_assert(buf + length <= buf_end); \ } while (0) -static void -skip_leb(const uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, - char* error_buf, uint32 error_buf_size) -{ - const uint8 *buf = *p_buf; - uint32 offset = 0, bcnt = 0; - uint64 byte; - - while (true) { - bh_assert(bcnt + 1 <= (maxbits + 6) / 7); - CHECK_BUF(buf, buf_end, offset + 1); - byte = buf[offset]; - offset += 1; - bcnt += 1; - if ((byte & 0x80) == 0) { - break; - } - } - - *p_buf += offset; -} - -#define skip_leb_int64(p, p_end) do { \ - skip_leb(&p, p_end, 64, \ - error_buf, error_buf_size); \ - } while (0) - -#define skip_leb_uint32(p, p_end) do { \ - skip_leb(&p, p_end, 32, \ - error_buf, error_buf_size); \ - } while (0) - -#define skip_leb_int32(p, p_end) do { \ - skip_leb(&p, p_end, 32, \ - error_buf, error_buf_size); \ - } while (0) +#define skip_leb(p) while (*p++ & 0x80) +#define skip_leb_int64(p, p_end) skip_leb(p) +#define skip_leb_uint32(p, p_end) skip_leb(p) +#define skip_leb_int32(p, p_end) skip_leb(p) static void read_leb(uint8 **p_buf, const uint8 *buf_end, @@ -2149,17 +2117,17 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, const uint8 *code_end_addr, uint8 label_type, uint8 **p_else_addr, - uint8 **p_end_addr, - char *error_buf, - uint32 error_buf_size) + uint8 **p_end_addr) { const uint8 *p = start_addr, *p_end = code_end_addr; uint8 *else_addr = NULL; + char error_buf[128]; uint32 block_nested_depth = 1, count, i, j, t; + uint32 error_buf_size = sizeof(error_buf); uint8 opcode, u8; BlockAddr block_stack[16] = { 0 }, *block; - i = ((uintptr_t)start_addr) % BLOCK_ADDR_CACHE_SIZE; + i = ((uintptr_t)start_addr) & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); block = block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) { @@ -2185,7 +2153,6 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, case WASM_OP_BLOCK: case WASM_OP_LOOP: case WASM_OP_IF: - CHECK_BUF(p, p_end, 1); /* block result type: 0x40/0x7F/0x7E/0x7D/0x7C */ u8 = read_uint8(p); if (block_nested_depth < sizeof(block_stack)/sizeof(BlockAddr)) { @@ -2224,7 +2191,8 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, for (t = 0; t < sizeof(block_stack)/sizeof(BlockAddr); t++) { start_addr = block_stack[t].start_addr; if (start_addr) { - i = ((uintptr_t)start_addr) % BLOCK_ADDR_CACHE_SIZE; + i = ((uintptr_t)start_addr) + & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); block = block_addr_cache + BLOCK_ADDR_CONFLICT_SIZE * i; for (j = 0; j < BLOCK_ADDR_CONFLICT_SIZE; j++) if (!block[j].start_addr) @@ -4669,7 +4637,6 @@ handle_op_block_and_loop: #endif POP_I32(); - /* TODO: check the const */ for (i = 0; i <= count; i++) { if (!(frame_csp_tmp = check_branch_block(loader_ctx, &p, p_end, diff --git a/samples/workload/README.md b/samples/workload/README.md index 2af6ec6b..624126cb 100644 --- a/samples/workload/README.md +++ b/samples/workload/README.md @@ -1,8 +1,8 @@ -All workloads have similar a requirment of software dependencies. It includes -**wasi-sdk**, **clang-11**, **emsdk**, **wabt** and **binaryen** +All workloads have similar requirment of software dependencies, including +**wasi-sdk**, **emsdk**, **wabt** and **binaryen** -> It might slightly different when using MacOS, and other linux distro than Ubuntu. This document only target -Ubuntu 18.04 as an example. +> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document only target +Ubuntu 18.04 as example. ## Installation instructions @@ -11,28 +11,16 @@ use [preparation.sh](./preparation.sh) to install all dependencies before compil for details, the script includes below steps: - **wasi-sdk**. Install - [latest release](https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sdk-11.0-linux.tar.gz) - in */opt/wasi-sdk* or */opt/wasi-sdk-11* + [latest release](https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz) + to */opt/wasi-sdk* - **wabt**. Install - [latest release](https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/wabt-1.0.19-ubuntu.tar.gz) - in */opt/wabt* or */opt/wabt-1.0.19* - -- **clang-11**. Refer to [the guide](https://apt.llvm.org/). + [latest release](https://github.com/WebAssembly/wabt/releases/download/1.0.20/wabt-1.0.20-ubuntu.tar.gz) + to */opt/wabt* or */opt/wabt-1.0.20* - **emsdk**. Refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate emsdk and set up environment variables. Verify it with `echo ${EMSDK}`. -- **libclang_rt.builtins-wasm32.a**. *wasi* has its private rt library. Put it under clang search path - -``` shell -# copy it -$ cp -r /opt/wasi-sdk-11.0/lib/clang/10.0.0/lib/wasi /usr/lib/llvm-11/lib/clang/11.0.0/lib/ - -# or just link it -$ ln -sf /opt/wasi-sdk-11.0/lib/clang/10.0.0/lib/wasi/ /usr/lib/llvm-11/lib/clang/11.0.0/lib/ -``` - - **binaryen**. Install [latest release](https://github.com/WebAssembly/binaryen/releases/download/version_97/binaryen-version_97-x86_64-linux.tar.gz) - in */opt/binaryen* or */opt/binaryen-version_97* + to */opt/binaryen* or */opt/binaryen-version_97* diff --git a/samples/workload/XNNPACK/README.md b/samples/workload/XNNPACK/README.md index a34f6621..ffbe2361 100644 --- a/samples/workload/XNNPACK/README.md +++ b/samples/workload/XNNPACK/README.md @@ -1,6 +1,7 @@ "XNNPACK" sample introduction ============== -This sample demonstrates how to build [XNNPACK](https://github.com/google/XNNPACK) benchmarks into WebAssembly with emcc toolchain and run them with iwasm. + +This sample demonstrates how to build [XNNPACK](https://github.com/google/XNNPACK) benchmarks into WebAssembly with emsdk toolchain and run them with iwasm. ## Installation toolchains diff --git a/samples/workload/bwa/CMakeLists.bwa_wasm.txt b/samples/workload/bwa/CMakeLists.bwa_wasm.txt index 8cda6b69..f315bf9c 100644 --- a/samples/workload/bwa/CMakeLists.bwa_wasm.txt +++ b/samples/workload/bwa/CMakeLists.bwa_wasm.txt @@ -102,6 +102,7 @@ target_compile_options(${PROJECT_NAME} PRIVATE -Wno-unused-function -Wno-unused-variable + -msimd128 ) target_link_options(${PROJECT_NAME} diff --git a/samples/workload/bwa/CMakeLists.txt b/samples/workload/bwa/CMakeLists.txt index 6e785a14..5c1d4a8f 100644 --- a/samples/workload/bwa/CMakeLists.txt +++ b/samples/workload/bwa/CMakeLists.txt @@ -5,6 +5,38 @@ cmake_minimum_required (VERSION 3.0) project(bwa_wasm) +################ WASI-SDK ################ +find_path(WASI_SDK_HOME + NAMES wasi-sdk + PATHS /opt/ + REQUIRED +) + +if (NOT WASI_SDK_HOME) + message(FATAL_ERROR + "can not find wasi-sdk. " + "please download it from " + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz " + "and install it under /opt/wasi-sdk" + ) +endif() + +# +# check clang version +execute_process(COMMAND + ${WASI_SDK_HOME}/wasi-sdk/bin/clang --version + OUTPUT_VARIABLE clang_full_version_string +) +string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" + CLANG_VERSION_STRING ${clang_full_version_string} +) +message("cur clang version is ${CLANG_VERSION_STRING}") +if(CLANG_VERSION_STRING VERSION_LESS 11.0) + message(FATAL_ERROR + "please install latest wai-sdk to get a clang-11 at least" + ) +endif() + ################ EMCC ################ if(NOT DEFINED ENV{EMSDK}) message(FATAL_ERROR @@ -85,7 +117,10 @@ ExternalProject_Add(bwa UPDATE_COMMAND git clean -fd && git checkout -- * && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt" && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.bwa_wasm.txt CMakeLists.txt - CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/../cmake/toolchain.cmake ${CMAKE_CURRENT_SOURCE_DIR}/bwa + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASI_SDK_HOME}/wasi-sdk + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/wasi-sdk/share/cmake/wasi-sdk.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/bwa BUILD_COMMAND make bwa_wasm_opt INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./bwa.opt.wasm ${CMAKE_CURRENT_SOURCE_DIR}/build/bwa.wasm ) diff --git a/samples/workload/bwa/README.md b/samples/workload/bwa/README.md index f9688f6c..e826bb61 100644 --- a/samples/workload/bwa/README.md +++ b/samples/workload/bwa/README.md @@ -40,8 +40,7 @@ $ make Then compile wasm file to aot file and run: ``` shell -$ cd /wamr-compiler/build -$ ./wamrc --enable-simd -o bwa.aot ./bwa.wasm -$ cd /product-mini/platforms/linux/ -$ ./iwasm --dir=. ./bwa.aot index hs38DH.fa +$ cd /samples/workload/bwa/build +$ /wamr-compiler/build/wamrc --enable-simd -o bwa.aot bwa.wasm +$ /product-mini/platforms/linux/iwasm --dir=. bwa.aot index hs38DH.fa ``` diff --git a/samples/workload/cmake/toolchain.cmake b/samples/workload/cmake/toolchain.cmake deleted file mode 100644 index 85591323..00000000 --- a/samples/workload/cmake/toolchain.cmake +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -cmake_minimum_required (VERSION 3.0) - -if(DEFINED _WAMR_TOOLCHAIN_CMAKE_) - return() -else() - set(_WAMR_TOOLCHAIN_CMAKE_ 1) -endif() - -SET(CMAKE_SYSTEM_NAME Linux) - -################ COMPILER ################ -find_program(CLANG_11 NAMES clang clang-11 REQUIRED) -find_program(CLANG++_11 NAMES clang++ clang++-11 REQUIRED) - -if(NOT CLANG_11) - message(FATAL_ERROR "clang not found") -else() - message(STATUS "use ${CLANG_11} as the c compiler") -endif() - -if(NOT CLANG++_11) - message(FATAL_ERROR "clang++ not found") -else() - message(STATUS "use ${CLANG++_11} as the c++ compiler") -endif() - -set(CMAKE_C_COMPILER "${CLANG_11}" CACHE STRING "C compiler" FORCE) -set(CMAKE_C_COMPILER_ID Clang CACHE STRING "C compiler ID" FORCE) - -set(CMAKE_CXX_COMPILER "${CLANG++_11}" CACHE STRING "C++ compiler" FORCE) -set(CMAKE_CXX_COMPILER_ID Clang CACHE STRING "C++ compiler ID" FORCE) - -################ WASI AS SYSROOT ################ -find_path(WASI_SYSROOT - wasi-sysroot - PATHS /opt/wasi-sdk-11.0/share /opt/wasi-sdk/share - REQUIRED -) - -if(NOT WASI_SYSROOT) - message(FATAL_ERROR - "can not find wasi sysroot. " - "please download it from " - "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sdk-11.0-linux.tar.gz " - "and install it under /opt" - ) -endif() - -set(CMAKE_SYSROOT ${WASI_SYSROOT}/wasi-sysroot CACHE STRING "--sysroot to compiler" FORCE) - -add_compile_options( - --target=wasm32-wasi - -msimd128 - $,-O0,-O3> - $<$:-g> - $<$:-v> -) - -# need users to create their own additional include files - -################ AR ################ -find_program(LLVM_AR NAMES llvm-ar llvm-ar-11 REQUIRED) - -if(NOT LLVM_AR) - message(FATAL_ERROR "llvm-ar not found") -else() - message(STATUS "use ${LLVM_AR} as the AR") -endif() - -set(CMAKE_AR "${LLVM_AR}" CACHE STRING "AR" FORCE) - -################ RANLIB ################ -find_program(LLVM_RANLIB NAMES llvm-ranlib llvm-ranlib-11 REQUIRED) - -if(NOT LLVM_RANLIB) - message(FATAL_ERROR "llvm-ranlib not found") -else() - message(STATUS "use ${LLVM_RANLIB} as the ranlib") -endif() - -set(CMAKE_RANLIB "${LLVM_RANLIB}" CACHE STRING "RANLIB" FORCE) - -################ LD ################ -find_program(WASM_LD NAMES wasm-ld wasm-ld-11 REQUIRED) - -if(NOT WASM_LD) - message(FATAL_ERROR "wasm-ld not found") -else() - message(STATUS "use ${WASM_LD} as the linker") -endif() - -add_link_options( - --target=wasm32-wasi - -fuse-ld=${WASM_LD} - LINKER:--allow-undefined - $,-O0,-O3> - $<$:-g> - $<$:-v> -) diff --git a/samples/workload/docker/Dockerfile b/samples/workload/docker/Dockerfile index 8afa4738..4c57c668 100644 --- a/samples/workload/docker/Dockerfile +++ b/samples/workload/docker/Dockerfile @@ -1,29 +1,32 @@ FROM ubuntu:18.04 as builder +RUN apt update \ + && apt install -y lsb-release software-properties-common build-essential \ + wget curl git tree zip unzip + # # install clang and llvm -COPY llvm.sh /tmp -RUN apt update \ - && apt install -y lsb-release wget software-properties-common build-essential \ - && cd /tmp \ - && chmod a+x llvm.sh \ - && ./llvm.sh 11 +# COPY llvm.sh /tmp +# RUN apt update \ +# && apt install -y lsb-release wget software-properties-common build-essential \ +# && cd /tmp \ +# && chmod a+x llvm.sh \ +# && ./llvm.sh 11 -ARG WASI_SDK_VER=11.0 -ARG WABT_VER=1.0.19 +ARG WASI_SDK_VER=12 +ARG WABT_VER=1.0.20 ARG CMAKE_VER=3.16.2 ARG BINARYEN_VER=version_97 ARG BAZEL_VER=3.7.0 # # install wasi-sdk -ARG WASI_SDK_FILE="wasi-sdk-${WASI_SDK_VER}-linux.tar.gz" +ARG WASI_SDK_FILE="wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz" COPY ${WASI_SDK_FILE} /opt RUN cd /opt \ && tar zxf ${WASI_SDK_FILE} \ && rm ${WASI_SDK_FILE} \ - && ln -sf /opt/wasi-sdk-${WASI_SDK_VER} /opt/wasi-sdk \ - && ln -sf /opt/wasi-sdk/lib/clang/10.0.0/lib/wasi/ /usr/lib/llvm-11/lib/clang/11.0.1/lib/ + && ln -sf /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk # # install wabt @@ -44,11 +47,6 @@ RUN cd /tmp \ && ./${CMAKE_FILE} --prefix=/opt/cmake --skip-license \ && ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake -# -# install tools -RUN apt update \ - && apt install -y git tree - # # install emsdk RUN cd /opt \ @@ -68,15 +66,13 @@ RUN cd /opt \ && rm ${BINARYEN_FILE} \ && ln -sf /opt/binaryen-${BINARYEN_VER} /opt/binaryen -RUN apt install -y unzip zip - -# -# install bazel -ARG BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh -COPY ${BAZEL_FILE} /tmp -RUN cd /tmp \ - && chmod a+x ${BAZEL_FILE} \ - && ./${BAZEL_FILE} +# # +# # install bazel +# ARG BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh +# COPY ${BAZEL_FILE} /tmp +# RUN cd /tmp \ +# && chmod a+x ${BAZEL_FILE} \ +# && ./${BAZEL_FILE} # # Clean up diff --git a/samples/workload/docker/docker_build.sh b/samples/workload/docker/docker_build.sh index a5fb54bb..ec502d55 100755 --- a/samples/workload/docker/docker_build.sh +++ b/samples/workload/docker/docker_build.sh @@ -1,34 +1,32 @@ +#!/usr/bin/env bash # # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -#!/bin/bash - -BUILD_CONTENT="/tmp/build_content" - +readonly BUILD_CONTENT="/tmp/build_content" if [[ ! -d ${BUILD_CONTENT} ]]; then mkdir ${BUILD_CONTENT} fi -WASI_SDK_VER=11.0 -WABT_VER=1.0.19 -CMAKE_VER=3.16.2 -BINARYEN_VER=version_97 -BAZEL_VER=3.7.0 +readonly WASI_SDK_VER=12 +readonly WABT_VER=1.0.20 +readonly CMAKE_VER=3.16.2 +readonly BINARYEN_VER=version_97 +readonly BAZEL_VER=3.7.0 -cd ${BUILD_CONTENT} -if [[ ! -f wasi-sdk-${WASI_SDK_VER}-linux.tar.gz ]]; then - wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/wasi-sdk-${WASI_SDK_VER}-linux.tar.gz +cd ${BUILD_CONTENT} || exit +if [[ ! -f wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz ]]; then + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VER}/wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz fi if [[ ! -f wabt-${WABT_VER}-ubuntu.tar.gz ]]; then wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/wabt-${WABT_VER}-ubuntu.tar.gz fi -if [[ ! -f llvm.sh ]]; then - wget https://apt.llvm.org/llvm.sh -fi +# if [[ ! -f llvm.sh ]]; then +# wget https://apt.llvm.org/llvm.sh +# fi if [[ ! -f cmake-${CMAKE_VER}-Linux-x86_64.sh ]]; then wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-Linux-x86_64.sh @@ -41,30 +39,21 @@ fi if [[ ! -f bazel-${BAZEL_VER}-installer-linux-x86_64.sh ]]; then wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VER}/bazel-${BAZEL_VER}-installer-linux-x86_64.sh fi -cd - > /dev/null +cd - > /dev/null || exit -DOCKERFILE_PATH=$(dirname $(realpath $0)) +DOCKERFILE_PATH=$(dirname "$(realpath "$0")") docker build \ - --build-arg http_proxy=${http_proxy} \ - --build-arg https_proxy=${https_proxy} \ - --build-arg HTTP_PROXY=${http_proxy} \ - --build-arg HTTPS_PROXY=${https_proxy} \ --build-arg WASI_SDK_VER=${WASI_SDK_VER} \ --build-arg WABT_VER=${WABT_VER} \ --build-arg CMAKE_VER=${CMAKE_VER} \ --build-arg BINARYEN_VER=${BINARYEN_VER} \ --build-arg BAZEL_VER=${BAZEL_VER} \ - -t clang_env:0.1 -f ${DOCKERFILE_PATH}/Dockerfile ${BUILD_CONTENT} + -t clang_env:0.1 -f "${DOCKERFILE_PATH}"/Dockerfile ${BUILD_CONTENT} docker run --rm -it \ - -e http_proxy=${http_proxy} \ - -e https_proxy=${https_proxy} \ - -e HTTP_PROXY=${http_proxy} \ - -e HTTPS_PROXY=${htpps_proxy} \ --name workload_w_clang \ - --mount type=bind,source=$(pwd),target=/data/project \ - --mount type=bind,source=$(pwd)/../cmake,target=/data/cmake \ + --mount type=bind,source="$(pwd)",target=/data/project \ -w /data/project \ clang_env:0.1 \ /bin/bash -c /build.sh diff --git a/samples/workload/meshoptimizer/CMakeLists.txt b/samples/workload/meshoptimizer/CMakeLists.txt index 1270582d..1ac8ee18 100644 --- a/samples/workload/meshoptimizer/CMakeLists.txt +++ b/samples/workload/meshoptimizer/CMakeLists.txt @@ -5,6 +5,38 @@ cmake_minimum_required (VERSION 3.0) project(bench-meshoptimizer) +################ WASI-SDK ################ +find_path(WASI_SDK_HOME + NAMES wasi-sdk + PATHS /opt/ + REQUIRED +) + +if (NOT WASI_SDK_HOME) + message(FATAL_ERROR + "can not find wasi-sdk. " + "please download it from " + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz " + "and install it under /opt/wasi-sdk" + ) +endif() + +# +# check clang version +execute_process(COMMAND + ${WASI_SDK_HOME}/wasi-sdk/bin/clang --version + OUTPUT_VARIABLE clang_full_version_string +) +string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" + CLANG_VERSION_STRING ${clang_full_version_string} +) +message("cur clang version is ${CLANG_VERSION_STRING}") +if(CLANG_VERSION_STRING VERSION_LESS 11.0) + message(FATAL_ERROR + "please install latest wai-sdk to get a clang-11 at least" + ) +endif() + ################ BINARYEN ################ find_program(WASM_OPT NAMES wasm-opt @@ -33,7 +65,10 @@ ExternalProject_Add(codecbench UPDATE_COMMAND git clean -fd && git checkout -- * && ${CMAKE_COMMAND} -E echo "Applying patch" && git apply ${CMAKE_CURRENT_SOURCE_DIR}/codecbench.patch - CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/../cmake/toolchain.cmake ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASI_SDK_HOME}/wasi-sdk + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/wasi-sdk/share/cmake/wasi-sdk.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer BUILD_COMMAND make codecbench.opt INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./codecbench.opt.wasm ${CMAKE_CURRENT_SOURCE_DIR}/build/codecbench.wasm ) diff --git a/samples/workload/meshoptimizer/README.md b/samples/workload/meshoptimizer/README.md index c29a07af..92e10391 100644 --- a/samples/workload/meshoptimizer/README.md +++ b/samples/workload/meshoptimizer/README.md @@ -8,7 +8,7 @@ WebAssembly with simd support and run it with iwasm. please refer to [installation instructions](../README.md). -## Build with clang-11 and wasi-sdk +## Build with wasi-sdk ``` shell $ mkdir build && cd build @@ -18,15 +18,15 @@ $ make $ ls codecbench.wasm ``` -## Or build with EMCC +## Or build with EMSDK -EMCC is another toolchain to compile C code to WASM. In this case, will have -a higher performance with EMCC. +EMSDK is another toolchain to compile C/C++ code to WASM. In this case, the output wasm file +might have a higher performance than the file generated by wasi-sdk. ``` shell $ git clone https://github.com/zeux/meshoptimizer.git $ cd messoptimizer -$ emcc tools/codecbench.cpp src/vertexcodec.cpp src/vertexfilter.cpp \ +$ em++ tools/codecbench.cpp src/vertexcodec.cpp src/vertexfilter.cpp \ src/overdrawanalyzer.cpp src/indexgenerator.cpp src/vcacheoptimizer.cpp \ src/clusterizer.cpp src/indexcodec.cpp src/vfetchanalyzer.cpp \ src/spatialorder.cpp src/allocator.cpp src/vcacheanalyzer.cpp \ @@ -51,9 +51,7 @@ $ make Then compile wasm file to aot file and run: ``` shell -$ cd /wamr-compiler/build -$ ./wamrc --enable-simd -o codecbench.aot codecbench.wasm -$ cd /product-mini/platforms/linux/ -$ ./iwasm codecbench.aot +$ /wamr-compiler/build/wamrc --enable-simd -o codecbench.aot codecbench.wasm +$ /product-mini/platforms/linux/build/iwasm codecbench.aot ``` diff --git a/samples/workload/meshoptimizer/codecbench.patch b/samples/workload/meshoptimizer/codecbench.patch index 4adebed4..2d6e9d4f 100644 --- a/samples/workload/meshoptimizer/codecbench.patch +++ b/samples/workload/meshoptimizer/codecbench.patch @@ -1,8 +1,8 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index eccc49e..dac126c 100644 +index ffdb4da..536a5c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -127,3 +127,42 @@ install(FILES +@@ -127,3 +127,43 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/meshoptimizerConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/meshoptimizerConfigVersion.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/meshoptimizer) @@ -16,6 +16,7 @@ index eccc49e..dac126c 100644 + +target_compile_options(codecbench + PUBLIC ++ -O3 -msimd128 + -std=c++11 + -Wno-unused-function + -Wno-unused-variable diff --git a/samples/workload/preparation.sh b/samples/workload/preparation.sh index 81b1aea2..1d086f10 100755 --- a/samples/workload/preparation.sh +++ b/samples/workload/preparation.sh @@ -1,9 +1,13 @@ #!/bin/bash +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# readonly BUILD_CONTENT="/tmp/build_content" -readonly WASI_SDK_VER=11.0 -readonly WASI_SDK_FILE="wasi-sdk-${WASI_SDK_VER}-linux.tar.gz" -readonly WABT_VER=1.0.19 +readonly WASI_SDK_VER=12 +readonly WASI_SDK_FILE="wasi-sdk-${WASI_SDK_VER}.0-linux.tar.gz" +readonly WABT_VER=1.0.20 readonly WABT_FILE="wabt-${WABT_VER}-ubuntu.tar.gz" readonly CMAKE_VER=3.16.2 readonly CMAKE_FILE="cmake-${CMAKE_VER}-Linux-x86_64.sh" @@ -13,7 +17,7 @@ readonly BAZEL_VER=3.7.0 readonly BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh function DEBUG() { - [[ -n $(env | grep "\") ]] + env | grep -q "\" } # @@ -26,25 +30,24 @@ function install_deps() { # # install clang -function install_clang() { - if [[ ! -f llvm.sh ]]; then - wget https://apt.llvm.org/llvm.sh - fi - - chmod a+x llvm.sh - ./llvm.sh 11 -} +#function install_clang() { +# if [[ ! -f llvm.sh ]]; then +# wget https://apt.llvm.org/llvm.sh +# fi +# +# chmod a+x llvm.sh +# ./llvm.sh 11 +#} # # install wasi-sdk function install_wasi-sdk() { if [[ ! -f ${WASI_SDK_FILE} ]]; then - wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-11/${WASI_SDK_FILE} + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VER}/${WASI_SDK_FILE} fi tar zxf ${WASI_SDK_FILE} -C /opt - ln -sf /opt/wasi-sdk-${WASI_SDK_VER} /opt/wasi-sdk - ln -sf /opt/wasi-sdk/lib/clang/10.0.0/lib/wasi/ /usr/lib/llvm-11/lib/clang/11.0.1/lib/ + ln -sf /opt/wasi-sdk-${WASI_SDK_VER}.0 /opt/wasi-sdk } # @@ -80,7 +83,7 @@ function install_emsdk() { git pull ./emsdk install latest ./emsdk activate latest - echo "source /opt/emsdk/emsdk_env.sh" >> ${HOME}/.bashrc + echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc } # @@ -112,18 +115,17 @@ if [[ ! -d ${BUILD_CONTENT} ]]; then mkdir ${BUILD_CONTENT} fi -cd ${BUILD_CONTENT} +cd ${BUILD_CONTENT} || exit if DEBUG; then - $@ + "$@" else install_deps \ - && install_clang \ - && install_wasi \ + && install_wasi-sdk \ && install_wabt \ && install_cmake \ && install_emsdk \ && install_binaryen \ && install_bazel fi -cd - > /dev/null +cd - > /dev/null || exit DEBUG && set +xevu diff --git a/samples/workload/tensorflow/README.md b/samples/workload/tensorflow/README.md index 8883a57e..11fa28b8 100644 --- a/samples/workload/tensorflow/README.md +++ b/samples/workload/tensorflow/README.md @@ -1,6 +1,7 @@ "tensorflow" sample introduction ============== -This sample demonstrates how to build [tensorflow](https://github.com/tensorflow/tensorflow) into WebAssembly with emcc toolchain and run it with iwasm. Please first install [emsdk](https://github.com/emscripten-core/emsdk): + +This sample demonstrates how to build [tensorflow](https://github.com/tensorflow/tensorflow) into WebAssembly with emsdk toolchain and run it with iwasm. Please first install [emsdk](https://github.com/emscripten-core/emsdk): ```bash git clone https://github.com/emscripten-core/emsdk.git cd emsdk diff --git a/samples/workload/wasm-av1/CMakeLists.txt b/samples/workload/wasm-av1/CMakeLists.txt index ae33e6ff..873af607 100644 --- a/samples/workload/wasm-av1/CMakeLists.txt +++ b/samples/workload/wasm-av1/CMakeLists.txt @@ -5,6 +5,38 @@ cmake_minimum_required (VERSION 3.0) project(av1_wasm) +################ WASI-SDK ################ +find_path(WASI_SDK_HOME + NAMES wasi-sdk + PATHS /opt/ + REQUIRED +) + +if (NOT WASI_SDK_HOME) + message(FATAL_ERROR + "can not find wasi-sdk. " + "please download it from " + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz " + "and install it under /opt/wasi-sdk" + ) +endif() + +# +# check clang version +execute_process(COMMAND + ${WASI_SDK_HOME}/wasi-sdk/bin/clang --version + OUTPUT_VARIABLE clang_full_version_string +) +string(REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" + CLANG_VERSION_STRING ${clang_full_version_string} +) +message("cur clang version is ${CLANG_VERSION_STRING}") +if(CLANG_VERSION_STRING VERSION_LESS 11.0) + message(FATAL_ERROR + "please install latest wai-sdk to get a clang-11 at least" + ) +endif() + ################ BINARYEN ################ find_program(WASM_OPT NAMES wasm-opt @@ -55,7 +87,10 @@ ExternalProject_Add(av1 && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt" && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.avx_wasm.txt CMakeLists.txt && git apply ../av1-clang.patch - CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/../cmake/toolchain.cmake ${CMAKE_CURRENT_SOURCE_DIR}/av1 + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASI_SDK_HOME}/wasi-sdk + -DCMAKE_TOOLCHAIN_FILE=${WASI_SDK_HOME}/wasi-sdk/share/cmake/wasi-sdk.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/av1 BUILD_COMMAND make testavx_opt INSTALL_COMMAND ${CMAKE_COMMAND} -E copy testavx.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/testavx.wasm ) diff --git a/samples/workload/wasm-av1/README.md b/samples/workload/wasm-av1/README.md index 78481241..e7a92f2a 100644 --- a/samples/workload/wasm-av1/README.md +++ b/samples/workload/wasm-av1/README.md @@ -8,21 +8,7 @@ WebAssembly with simd support and run it with iwasm. please refer to [installation instructions](../README.md). -## Build with EMSDK - -just run the convenience script: - -```bash -./build.sh -``` - -it is going to build wasm-av1 and run it with iwasm, which basically contains the following steps: -- hack emcc to delete some objects in libc.a -- patch wasm-av1 and build it with emcc compiler -- build iwasm with simd and libc-emcc support -- run testav1.aot with iwasm - -## Or build with clang-11 and wasi-sdk +## Build with wasi-sdk ``` shell $ mkdir build && cd build @@ -32,6 +18,20 @@ $ make $ ls testavx.wasm ``` +## Or build with EMSDK + +just run the convenience script: + +```bash +./build.sh +``` + +the script builds wasm-av1 and runs it with iwasm, which basically contains the following steps: +- hack emcc to delete some objects in libc.a +- patch wasm-av1 and build it with emcc compiler +- build iwasm with simd and libc-emcc support +- run testav1.aot with iwasm + ### Run workload Firstly please build iwasm with simd support: @@ -46,11 +46,9 @@ $ make Then compile wasm file to aot file and run: ``` shell -$ cd /wamr-compiler/build -$ ./wamrc --enable-simd -o testavx.aot testavx.wasm -$ cd /product-mini/platforms/linux/ -$ # copy sample data like /samples/workload/wasm-av1/av1/third_party/samples/elephants_dream_480p24.ivf -$ # copy testavx.aot -$ # make sure you declare the access priority of the directory in which the sample data is -$ ./iwasm --dir=. ./testavx.aot ./elephants_dream_480p24.ivf -``` \ No newline at end of file +$ cd +$ /wamr-compiler/build/wamrc --enable-simd -o testavx.aot testavx.wasm +# copy sample data like /samples/workload/wasm-av1/av1/third_party/samples/elephants_dream_480p24.ivf +# make sure you declare the access priority of the directory in which the sample data is +$ /product-mini/platforms/linux/build/iwasm --dir=. testavx.aot elephants_dream_480p24.ivf +```