Import SIMD feature and add some workload samples (#438)

2020-11-05 18:15:15 +08:00
parent 667282eea9
commit a3074df21b
84 changed files with 7780 additions and 318 deletions
--- a/samples/workload/bwa/.gitignore
+++ b/samples/workload/bwa/.gitignore
@ -0,0 +1,4 @@
+build
+libz
+bwa
+include
--- a/samples/workload/bwa/CMakeLists.bwa_wasm.txt
+++ b/samples/workload/bwa/CMakeLists.bwa_wasm.txt
@ -0,0 +1,134 @@
+# Copyright (C) 2019 Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+cmake_minimum_required (VERSION 3.0)
+
+project(bwa_wasm C)
+
+################ LIBZ ################
+set(LIBZ_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libz)
+add_library(z_wasm STATIC
+  ${LIBZ_SRC_DIR}/adler32.c
+  ${LIBZ_SRC_DIR}/compress.c
+  ${LIBZ_SRC_DIR}/crc32.c
+  ${LIBZ_SRC_DIR}/deflate.c
+  ${LIBZ_SRC_DIR}/gzclose.c
+  ${LIBZ_SRC_DIR}/gzlib.c
+  ${LIBZ_SRC_DIR}/gzread.c
+  ${LIBZ_SRC_DIR}/gzwrite.c
+  ${LIBZ_SRC_DIR}/infback.c
+  ${LIBZ_SRC_DIR}/inffast.c
+  ${LIBZ_SRC_DIR}/inflate.c
+  ${LIBZ_SRC_DIR}/inftrees.c
+  ${LIBZ_SRC_DIR}/trees.c
+  ${LIBZ_SRC_DIR}/uncompr.c
+  ${LIBZ_SRC_DIR}/zutil.c
+)
+
+set_target_properties(z_wasm PROPERTIES LINKER_LANGUAGE C)
+
+target_compile_definitions(z_wasm PRIVATE Z_HAVE_UNISTD_H _LARGEFILE64_SOURCE=1)
+
+target_compile_options(z_wasm
+  PRIVATE
+    -Wno-unused-function
+    -Wno-unused-variable
+)
+
+target_include_directories(z_wasm
+  PUBLIC
+    ${LIBZ_SRC_DIR}
+)
+
+################ BWA_WASM ################
+set(BWA_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(BWA_SOURCE
+  ${BWA_SRC_DIR}/utils.c
+  ${BWA_SRC_DIR}/kthread.c
+  ${BWA_SRC_DIR}/kstring.c
+  ${BWA_SRC_DIR}/ksw.c
+  ${BWA_SRC_DIR}/bwt.c
+  ${BWA_SRC_DIR}/bntseq.c
+  ${BWA_SRC_DIR}/bwa.c
+  ${BWA_SRC_DIR}/bwamem.c
+  ${BWA_SRC_DIR}/bwamem_pair.c
+  ${BWA_SRC_DIR}/bwamem_extra.c
+  ${BWA_SRC_DIR}/malloc_wrap.c
+  ${BWA_SRC_DIR}/QSufSort.c
+  ${BWA_SRC_DIR}/bwt_gen.c
+  ${BWA_SRC_DIR}/rope.c
+  ${BWA_SRC_DIR}/rle.c
+  ${BWA_SRC_DIR}/is.c
+  ${BWA_SRC_DIR}/bwtindex.c
+  ${BWA_SRC_DIR}/bwashm.c
+  ${BWA_SRC_DIR}/bwase.c
+  ${BWA_SRC_DIR}/bwaseqio.c
+  ${BWA_SRC_DIR}/bwtgap.c
+  ${BWA_SRC_DIR}/bwtaln.c
+  ${BWA_SRC_DIR}/bamlite.c
+  ${BWA_SRC_DIR}/bwape.c
+  ${BWA_SRC_DIR}/kopen.c
+  ${BWA_SRC_DIR}/pemerge.c
+  ${BWA_SRC_DIR}/maxk.c
+  ${BWA_SRC_DIR}/bwtsw2_core.c
+  ${BWA_SRC_DIR}/bwtsw2_main.c
+  ${BWA_SRC_DIR}/bwtsw2_aux.c
+  ${BWA_SRC_DIR}/bwt_lite.c
+  ${BWA_SRC_DIR}/bwtsw2_chain.c
+  ${BWA_SRC_DIR}/fastmap.c
+  ${BWA_SRC_DIR}/bwtsw2_pair.c
+  ${BWA_SRC_DIR}/main.c
+)
+
+add_executable(${PROJECT_NAME} ${BWA_SOURCE})
+
+set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME bwa.wasm)
+
+target_include_directories(${PROJECT_NAME}
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../include
+    ${CMAKE_CURRENT_SOURCE_DIR}/../include/SSE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../include/pthread
+)
+
+target_compile_definitions(${PROJECT_NAME}
+  PRIVATE
+    USE_MALLOC_WRAPPERS
+    __SSE__ __SSE2__ __SSE4_1__
+    _WASI_EMULATED_MMAN _WASI_EMULATED_SIGNAL
+)
+
+target_compile_options(${PROJECT_NAME}
+  PRIVATE
+    -Wno-unused-function
+    -Wno-unused-variable
+)
+
+target_link_options(${PROJECT_NAME}
+  PRIVATE
+    -Wno-unused-command-line-argument
+    LINKER:--allow-undefined,--export=__heap_base,--export=__data_end
+    LINKER:-z,stack-size=1048576
+)
+
+target_link_libraries(${PROJECT_NAME} z_wasm)
+
+find_program(WASM_OPT
+    NAMES wasm-opt
+    PATHS /opt/binaryen-version_97/bin /opt/binaryen/bin
+)
+
+if (NOT WASM_OPT)
+  message(WARNING "can not find wasm-opt and will not optimize any wasm module")
+endif()
+
+add_custom_target(bwa_wasm_opt ALL
+  COMMAND
+    ${WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm
+  BYPRODUCTS
+    ${CMAKE_CURRENT_BINARY_DIR}/bwa.opt.wasm
+  WORKING_DIRECTORY
+    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+add_dependencies(bwa_wasm_opt ${PROJECT_NAME})
--- a/samples/workload/bwa/CMakeLists.txt
+++ b/samples/workload/bwa/CMakeLists.txt
@ -0,0 +1,91 @@
+# Copyright (C) 2019 Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+cmake_minimum_required (VERSION 3.0)
+
+project(bwa_wasm)
+
+################  EMCC ################
+if(NOT DEFINED ENV{EMSDK})
+  message(FATAL_ERROR
+    "can not find emsdk. "
+    "please refer to https://emscripten.org/docs/getting_started/downloads.html "
+    "and install it, "
+    "or active emsdk by 'source ./emsdk_env.sh'"
+  )
+endif()
+
+################  BINARYEN ################
+find_program(WASM_OPT
+    NAMES wasm-opt
+    PATHS /opt/binaryen-version_97/bin /opt/binaryen/bin
+)
+
+if (NOT WASM_OPT)
+  message(FATAL_ERROR
+    "can not find wasm-opt. "
+    "please download it from "
+    "https://github.com/WebAssembly/binaryen/releases/download/version_97/binaryen-version_97-x86_64-linux.tar.gz "
+    "and install it under /opt"
+  )
+endif()
+
+#######################################
+include(ExternalProject)
+
+################ HEADERS ################
+ExternalProject_Add(headers_from_emcc
+   PREFIX headers
+   SOURCE_DIR "$ENV{EMSDK}/upstream/emscripten/system/include/SSE"
+   CONFIGURE_COMMAND ""
+   BUILD_COMMAND  ""
+   INSTALL_COMMAND mkdir -p ${CMAKE_CURRENT_SOURCE_DIR}/include/SSE
+                     && ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_SOURCE_DIR}/include/pthread/sys
+                     && ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_SOURCE_DIR}/include/emscripten
+                     # copy emscripten SSE header files
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/SSE/immintrin.h ${CMAKE_CURRENT_SOURCE_DIR}/include/SSE/
+                     # SSE
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/SSE/xmmintrin.h ${CMAKE_CURRENT_SOURCE_DIR}/include/SSE/
+                     # SSE2
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/SSE/emmintrin.h ${CMAKE_CURRENT_SOURCE_DIR}/include/SSE/
+                     # SSE4.1
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/SSE/smmintrin.h ${CMAKE_CURRENT_SOURCE_DIR}/include/SSE/
+                     # a fake empty header to aovid further depenency
+                     && ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_SOURCE_DIR}/include/emscripten/emscripten.h
+                     # copy emscripten pthread related header files
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/libc/pthread.h ${CMAKE_CURRENT_SOURCE_DIR}/include/pthread/
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/libc/signal.h ${CMAKE_CURRENT_SOURCE_DIR}/include/pthread/
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/libc/netdb.h ${CMAKE_CURRENT_SOURCE_DIR}/include/pthread/
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/libc/sys/wait.h ${CMAKE_CURRENT_SOURCE_DIR}/include/pthread/sys/
+                     && ${CMAKE_COMMAND} -E copy $ENV{EMSDK}/upstream/emscripten/system/include/libc/sys/socket.h ${CMAKE_CURRENT_SOURCE_DIR}/include/pthread/sys/
+)
+
+################ libz ################
+ExternalProject_Add(libz_src
+  PREFIX libz
+  GIT_REPOSITORY https://github.com/madler/zlib.git
+  GIT_TAG        master
+  GIT_PROGRESS   ON
+  GIT_SHALLOW    ON
+  SOURCE_DIR     ${CMAKE_CURRENT_SOURCE_DIR}/libz
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND  ""
+  INSTALL_COMMAND ""
+)
+
+################ bwa ################
+ExternalProject_Add(bwa
+  PREFIX bwa
+  GIT_REPOSITORY https://github.com/lh3/bwa.git
+  GIT_TAG        master
+  GIT_PROGRESS   ON
+  GIT_SHALLOW    ON
+  SOURCE_DIR     ${CMAKE_CURRENT_SOURCE_DIR}/bwa
+  DEPENDS        libz_src headers_from_emcc
+  UPDATE_COMMAND git clean -fd && git checkout -- *
+                  && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt"
+                  && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.bwa_wasm.txt CMakeLists.txt
+  CONFIGURE_COMMAND  ${CMAKE_COMMAND} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/../cmake/toolchain.cmake ${CMAKE_CURRENT_SOURCE_DIR}/bwa
+  BUILD_COMMAND  make bwa_wasm_opt
+  INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ./bwa.opt.wasm ${CMAKE_CURRENT_SOURCE_DIR}/build/bwa.wasm
+)
--- a/samples/workload/bwa/README.md
+++ b/samples/workload/bwa/README.md
@ -0,0 +1,47 @@
+"bwa" sample introduction
+==============
+
+This sample demonstrates how to build [bwa](https://github.com/lh3/bwa) into
+WebAssembly with simd support and run it with iwasm.
+
+## Preparation
+
+please refer to [installation instructions](../README.md).
+
+## Build
+
+``` shell
+$ mkdir build && cd build
+$ cmake ..
+$ make
+# to verify
+$ ls bwa.wasm
+```
+
+## Download sample data
+
+Download the bwa-0.7.15 binary package from
+[such an address](https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download),
+a sample data file named **hs38DH.fa** will be used later.
+
+If want more data, please refer to http://hgdownload.cse.ucsc.edu/goldenpath/hg19/bigZips/
+
+## Run workload
+
+Firstly please build iwasm with simd support:
+
+``` shell
+$ cd <wamr dir>/product-mini/platforms/linux/
+$ mkdir build && cd build
+$ cmake .. -DWAMR_BUILD_SIMD=1
+$ make
+```
+
+Then compile wasm file to aot file and run:
+
+``` shell
+$ cd <wamr dir>/wamr-compiler/build
+$ ./wamrc --enable-simd -o bwa.aot ./bwa.wasm
+$ cd <wamr dir>/product-mini/platforms/linux/
+$ ./iwasm --dir=. ./bwa.aot index hs38DH.fa
+```