From c515fb1b7514015e8c1a2a54c443df197e3c2633 Mon Sep 17 00:00:00 2001
From: Wang Ning <justdoitwn@163.com>
Date: Thu, 22 Oct 2020 16:18:37 +0800
Subject: [PATCH] Add tensorflow sample under samples/workload/tensorflow
 (#427)

---
 build-scripts/config_common.cmake             |   3 +
 build-scripts/runtime_lib.cmake               |   5 +
 core/config.h                                 |   5 +
 core/iwasm/common/wasm_native.c               |  11 +
 .../iwasm/libraries/libc-emcc/libc_emcc.cmake |  12 ++
 .../libraries/libc-emcc/libc_emcc_wrapper.c   | 196 ++++++++++++++++++
 samples/workload/tensorflow/README.md         |  19 ++
 samples/workload/tensorflow/build.sh          |  97 +++++++++
 samples/workload/tensorflow/tf_lite.patch     |  78 +++++++
 9 files changed, 426 insertions(+)
 create mode 100644 core/iwasm/libraries/libc-emcc/libc_emcc.cmake
 create mode 100644 core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c
 create mode 100644 samples/workload/tensorflow/README.md
 create mode 100755 samples/workload/tensorflow/build.sh
 create mode 100644 samples/workload/tensorflow/tf_lite.patch

diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake
index 8ac8e4de..9f3d5bb6 100644
--- a/build-scripts/config_common.cmake
+++ b/build-scripts/config_common.cmake
@@ -152,6 +152,9 @@ endif ()
 if (WAMR_BUILD_LIB_PTHREAD EQUAL 1)
   message ("     Lib pthread enabled")
 endif ()
+if (WAMR_BUILD_LIBC_EMCC EQUAL 1)
+  message ("     Libc emcc enabled")
+endif ()
 if (WAMR_BUILD_MINI_LOADER EQUAL 1)
   add_definitions (-DWASM_ENABLE_MINI_LOADER=1)
   message ("     WASM mini loader enabled")
diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake
index a87a2ff6..aaddc004 100644
--- a/build-scripts/runtime_lib.cmake
+++ b/build-scripts/runtime_lib.cmake
@@ -81,6 +81,10 @@ if (WAMR_BUILD_THREAD_MGR EQUAL 1)
     include (${IWASM_DIR}/libraries/thread-mgr/thread_mgr.cmake)
 endif ()
 
+if (WAMR_BUILD_LIBC_EMCC EQUAL 1)
+    include (${IWASM_DIR}/libraries/libc-emcc/libc_emcc.cmake)
+endif()
+
 ####################### Common sources #######################
 if (NOT MSVC)
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \
@@ -120,6 +124,7 @@ set (source_all
     ${APP_MGR_SOURCE}
     ${LIB_PTHREAD_SOURCE}
     ${THREAD_MGR_SOURCE}
+    ${LIBC_EMCC_SOURCE}
 )
 
 set (WAMR_RUNTIME_LIB_SOURCE ${source_all})
diff --git a/core/config.h b/core/config.h
index d21396aa..1b474b0b 100644
--- a/core/config.h
+++ b/core/config.h
@@ -82,6 +82,11 @@
 #define WASM_ENABLE_LIBC_WASI 0
 #endif
 
+/* Default disable libc emcc */
+#ifndef WASM_ENABLE_LIBC_EMCC
+#define WASM_ENABLE_LIBC_EMCC 0
+#endif
+
 #ifndef WASM_ENABLE_LIB_PTHREAD
 #define WASM_ENABLE_LIB_PTHREAD 0
 #endif
diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c
index ff694645..ad9ebae0 100644
--- a/core/iwasm/common/wasm_native.c
+++ b/core/iwasm/common/wasm_native.c
@@ -50,6 +50,9 @@ uint32
 get_lib_pthread_export_apis(NativeSymbol **p_lib_pthread_apis);
 #endif
 
+uint32
+get_libc_emcc_export_apis(NativeSymbol **p_libc_emcc_apis);
+
 static bool
 check_symbol_signature(const WASMType *type, const char *signature)
 {
@@ -386,6 +389,14 @@ wasm_native_init()
         return false;
 #endif
 
+#if WASM_ENABLE_LIBC_EMCC != 0
+    n_native_symbols = get_libc_emcc_export_apis(&native_symbols);
+    if (n_native_symbols > 0
+        && !wasm_native_register_natives("env",
+                                         native_symbols, n_native_symbols))
+        return false;
+#endif /* WASM_ENABLE_LIBC_EMCC */
+
     return true;
 }
 
diff --git a/core/iwasm/libraries/libc-emcc/libc_emcc.cmake b/core/iwasm/libraries/libc-emcc/libc_emcc.cmake
new file mode 100644
index 00000000..d237a16e
--- /dev/null
+++ b/core/iwasm/libraries/libc-emcc/libc_emcc.cmake
@@ -0,0 +1,12 @@
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set (LIBC_EMCC_DIR ${CMAKE_CURRENT_LIST_DIR})
+
+add_definitions (-DWASM_ENABLE_LIBC_EMCC=1)
+
+include_directories(${LIBC_EMCC_DIR})
+
+file (GLOB source_all ${LIBC_EMCC_DIR}/*.c)
+
+set (LIBC_EMCC_SOURCE ${source_all})
\ No newline at end of file
diff --git a/core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c b/core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c
new file mode 100644
index 00000000..0e3652b3
--- /dev/null
+++ b/core/iwasm/libraries/libc-emcc/libc_emcc_wrapper.c
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2019 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "bh_common.h"
+#include "bh_log.h"
+#include "wasm_export.h"
+#include "../interpreter/wasm.h"
+
+#define get_module_inst(exec_env) \
+    wasm_runtime_get_module_inst(exec_env)
+
+#define validate_native_addr(addr, size) \
+    wasm_runtime_validate_native_addr(module_inst, addr, size)
+
+#define module_malloc(size, p_native_addr) \
+    wasm_runtime_module_malloc(module_inst, size, p_native_addr)
+
+#define module_free(offset) \
+    wasm_runtime_module_free(module_inst, offset)
+
+#define REG_NATIVE_FUNC(func_name, signature)  \
+    { #func_name, func_name##_wrapper, signature, NULL }
+
+struct timespec_emcc {
+    int tv_sec;
+    int tv_nsec;
+};
+
+struct stat_emcc {
+    unsigned st_dev;
+    int __st_dev_padding;
+    unsigned __st_ino_truncated;
+    unsigned st_mode;
+    unsigned st_nlink;
+    unsigned st_uid;
+    unsigned st_gid;
+    unsigned st_rdev;
+    int __st_rdev_padding;
+    int64 st_size;
+    int st_blksize;
+    int st_blocks;
+    struct timespec_emcc st_atim;
+    struct timespec_emcc st_mtim;
+    struct timespec_emcc st_ctim;
+    int64 st_ino;
+};
+
+static int
+open_wrapper(wasm_exec_env_t exec_env, const char *pathname,
+             int flags, int mode)
+{
+    if (pathname == NULL)
+        return -1;
+    return open(pathname, flags, mode);
+}
+
+static int
+__sys_read_wrapper(wasm_exec_env_t exec_env,
+                   int fd, void *buf, uint32 count)
+{
+    return read(fd, buf, count);
+}
+
+static void
+statbuf_native2app(const struct stat *statbuf_native,
+                   struct stat_emcc *statbuf_app)
+{
+    statbuf_app->st_dev = (unsigned)statbuf_native->st_dev;
+    statbuf_app->__st_ino_truncated = (unsigned)statbuf_native->st_ino;
+    statbuf_app->st_mode = (unsigned)statbuf_native->st_mode;
+    statbuf_app->st_nlink = (unsigned)statbuf_native->st_nlink;
+    statbuf_app->st_uid = (unsigned)statbuf_native->st_uid;
+    statbuf_app->st_gid = (unsigned)statbuf_native->st_gid;
+    statbuf_app->st_rdev = (unsigned)statbuf_native->st_rdev;
+    statbuf_app->st_size = (int64)statbuf_native->st_size;
+    statbuf_app->st_blksize = (unsigned)statbuf_native->st_blksize;
+    statbuf_app->st_blocks = (unsigned)statbuf_native->st_blocks;
+    statbuf_app->st_ino = (int64)statbuf_native->st_ino;
+    statbuf_app->st_atim.tv_sec = (int)statbuf_native->st_atim.tv_sec;
+    statbuf_app->st_atim.tv_nsec = (int)statbuf_native->st_atim.tv_nsec;
+    statbuf_app->st_mtim.tv_sec = (int)statbuf_native->st_mtim.tv_sec;
+    statbuf_app->st_mtim.tv_nsec = (int)statbuf_native->st_mtim.tv_nsec;
+    statbuf_app->st_ctim.tv_sec = (int)statbuf_native->st_ctim.tv_sec;
+    statbuf_app->st_ctim.tv_nsec = (int)statbuf_native->st_ctim.tv_nsec;
+}
+
+static int
+__sys_stat64_wrapper(wasm_exec_env_t exec_env,
+                     const char *pathname,
+                     struct stat_emcc *statbuf_app)
+{
+    wasm_module_inst_t module_inst = get_module_inst(exec_env);
+    int ret;
+    struct stat statbuf;
+
+    if (!validate_native_addr((void*)statbuf_app, sizeof(struct stat_emcc)))
+        return -1;
+
+    if (pathname == NULL)
+        return -1;
+
+    ret = stat(pathname, &statbuf);
+    if (ret == 0)
+        statbuf_native2app(&statbuf, statbuf_app);
+    return ret;
+}
+
+static int
+__sys_fstat64_wrapper(wasm_exec_env_t exec_env,
+                      int fd, struct stat_emcc *statbuf_app)
+{
+    wasm_module_inst_t module_inst = get_module_inst(exec_env);
+    int ret;
+    struct stat statbuf;
+
+    if (!validate_native_addr((void*)statbuf_app, sizeof(struct stat_emcc)))
+        return -1;
+
+    if (fd <= 0)
+        return -1;
+
+    ret = fstat(fd, &statbuf);
+    if (ret == 0)
+        statbuf_native2app(&statbuf, statbuf_app);
+    return ret;
+}
+
+static int
+mmap_wrapper(wasm_exec_env_t exec_env,
+             void *addr, int length, int prot, int flags,
+             int fd, int64 offset)
+{
+    wasm_module_inst_t module_inst = get_module_inst(exec_env);
+    uint32 buf_offset;
+    char *buf;
+    int size_read;
+
+    buf_offset = module_malloc(length, (void**)&buf);
+    if (buf_offset == 0)
+        return -1;
+
+    if (fd <= 0)
+        return -1;
+
+    if (lseek(fd, offset, SEEK_SET) == -1)
+        return -1;
+
+    size_read = read(fd, buf, length);
+    (void)size_read;
+    return buf_offset;
+}
+
+static int
+munmap_wrapper(wasm_exec_env_t exec_env, uint32 buf_offset, int length)
+{
+    wasm_module_inst_t module_inst = get_module_inst(exec_env);
+    module_free(buf_offset);
+    return 0;
+}
+
+static int
+__munmap_wrapper(wasm_exec_env_t exec_env, uint32 buf_offset, int length)
+{
+    return munmap_wrapper(exec_env, buf_offset, length);
+}
+
+static int
+getentropy_wrapper(wasm_exec_env_t exec_env, void *buffer, uint32 length)
+{
+    if (buffer == NULL)
+        return -1;
+    return getentropy(buffer, length);
+}
+
+#define REG_NATIVE_FUNC(func_name, signature)  \
+    { #func_name, func_name##_wrapper, signature, NULL }
+
+static NativeSymbol native_symbols_libc_emcc[] = {
+    REG_NATIVE_FUNC(open, "($ii)i"),
+    REG_NATIVE_FUNC(__sys_read, "(i*~)i"),
+    REG_NATIVE_FUNC(__sys_stat64, "($*)i"),
+    REG_NATIVE_FUNC(__sys_fstat64, "(i*)i"),
+    REG_NATIVE_FUNC(mmap, "(*iiiiI)i"),
+    REG_NATIVE_FUNC(munmap, "(ii)i"),
+    REG_NATIVE_FUNC(__munmap, "(ii)i"),
+    REG_NATIVE_FUNC(getentropy, "(*~)i"),
+};
+
+uint32
+get_libc_emcc_export_apis(NativeSymbol **p_libc_emcc_apis)
+{
+    *p_libc_emcc_apis = native_symbols_libc_emcc;
+    return sizeof(native_symbols_libc_emcc) / sizeof(NativeSymbol);
+}
diff --git a/samples/workload/tensorflow/README.md b/samples/workload/tensorflow/README.md
new file mode 100644
index 00000000..d34f51ec
--- /dev/null
+++ b/samples/workload/tensorflow/README.md
@@ -0,0 +1,19 @@
+"tensorflow" sample introduction
+==============
+This sample demonstrates how to build [tensorflow](https://github.com/tensorflow/tensorflow) into WebAssembly with emcc toolchain and run it with iwasm. Please first install [emsdk](https://github.com/emscripten-core/emsdk):
+```bash
+git clone https://github.com/emscripten-core/emsdk.git
+cd emsdk
+./emsdk install latest
+./emsdk activate latest
+```
+And set up ensdk environment:
+```bash
+source emsdk_env.sh
+```
+Then run ./build.sh to build tensorflow and run it with iwasm, which basically contains the following steps:
+- hack emcc to delete some objects in libc.a
+- build tf-lite with emcc compiler
+- build iwasm with pthread enable and include libiary under libc-emcc
+- run benchmark model with iwasm:
+  --max-secs 300: means the max training time cost is 5 minutes, you can adjust by yourself
diff --git a/samples/workload/tensorflow/build.sh b/samples/workload/tensorflow/build.sh
new file mode 100755
index 00000000..d1fdde04
--- /dev/null
+++ b/samples/workload/tensorflow/build.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+####################################
+#   build tensorflow-lite sample   #
+####################################
+set -x
+set -e
+
+EMSDK_WASM_DIR="$EM_CACHE/wasm"
+BUILD_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUT_DIR=${BUILD_SCRIPT_DIR}/out
+TENSORFLOW_DIR="${BUILD_SCRIPT_DIR}/tensorflow"
+TF_LITE_BUILD_DIR=${TENSORFLOW_DIR}/tensorflow/lite/tools/make
+WAMR_DIR="${BUILD_SCRIPT_DIR}/../../../product-mini/platforms/linux"
+
+function Clear_Before_Exit
+{
+    [[ -f ${TENSORFLOW_DIR}/tf_lite.patch ]] &&
+       rm -f ${TENSORFLOW_DIR}/tf_lite.patch
+    # resume the libc.a under EMSDK_WASM_DIR
+    cd ${EMSDK_WASM_DIR}
+    mv libc.a.bak libc.a
+}
+
+# 1.hack emcc
+cd ${EMSDK_WASM_DIR}
+# back up libc.a
+cp libc.a libc.a.bak
+# delete some objects in libc.a
+emar d libc.a open.o
+emar d libc.a mmap.o
+emar d libc.a munmap.o
+emranlib libc.a
+
+# 2. build tf-lite
+cd ${BUILD_SCRIPT_DIR}
+# 2.1 clone tf repo from Github and checkout to 2303ed commit
+if [ ! -d "tensorflow" ]; then
+    git clone https://github.com/tensorflow/tensorflow.git
+fi
+
+cd ${TENSORFLOW_DIR}
+git checkout 2303ed4bdb344a1fc4545658d1df6d9ce20331dd
+
+# 2.2 copy the tf-lite.patch to tensorflow_root_dir and apply
+cd ${TENSORFLOW_DIR}
+cp ${BUILD_SCRIPT_DIR}/tf_lite.patch .
+git checkout tensorflow/lite/tools/make/Makefile
+git checkout tensorflow/lite/tools/make/targets/linux_makefile.inc
+
+if [[ $(git apply tf_lite.patch 2>&1) =~ "error" ]]; then
+    echo "git apply patch failed, please check tf-lite related changes..."
+    Clear_Before_Exit
+    exit 0
+fi
+
+cd ${TF_LITE_BUILD_DIR}
+# 2.3 download dependencies
+if [ ! -d "${TF_LITE_BUILD_DIR}/downloads" ]; then
+    source download_dependencies.sh
+fi
+
+# 2.4 build tf-lite target
+if [ -d "${TF_LITE_BUILD_DIR}/gen" ]; then
+    rm -fr ${TF_LITE_BUILD_DIR}/gen
+fi
+make -j 4 -C "${TENSORFLOW_DIR}" -f ${TF_LITE_BUILD_DIR}/Makefile $@
+
+# 2.5 copy /make/gen target files to out/
+rm -rf ${OUT_DIR}
+mkdir ${OUT_DIR}
+cp -r ${TF_LITE_BUILD_DIR}/gen/linux_x86_64/bin/. ${OUT_DIR}/
+
+# 3. build iwasm with pthread and libc_emcc enable
+cd ${WAMR_DIR}
+rm -fr build && mkdir build
+cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1
+make
+
+# 4. run tensorflow with iwasm
+cd ${BUILD_SCRIPT_DIR}
+# 4.1 download tf-lite model
+if [ ! -f mobilenet_quant_v1_224.tflite ]; then
+    wget "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip"
+    unzip mobilenet_v1_224_android_quant_2017_11_08.zip
+fi
+
+# 4.2 run tf-lite model with iwasm
+echo "---> run tensorflow benchmark model with iwasm"
+${WAMR_DIR}/build/iwasm --heap-size=10475860 \
+                        ${OUT_DIR}/benchmark_model.wasm \
+                        --graph=mobilenet_quant_v1_224.tflite --max_secs=300
+
+Clear_Before_Exit
+
+
+
diff --git a/samples/workload/tensorflow/tf_lite.patch b/samples/workload/tensorflow/tf_lite.patch
new file mode 100644
index 00000000..85700778
--- /dev/null
+++ b/samples/workload/tensorflow/tf_lite.patch
@@ -0,0 +1,78 @@
+diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
+index c7ddff5844..1082644043 100644
+--- a/tensorflow/lite/tools/make/Makefile
++++ b/tensorflow/lite/tools/make/Makefile
+@@ -48,11 +48,7 @@ INCLUDES += -I/usr/local/include
+ 
+ # These are the default libraries needed, but they can be added to or
+ # overridden by the platform-specific settings in target makefiles.
+-LIBS := \
+--lstdc++ \
+--lpthread \
+--lm \
+--lz \
++LIBS := -lm \
+ -ldl
+ 
+ # There are no rules for compiling objects for the host system (since we don't
+@@ -84,14 +80,18 @@ endif # ifeq ($(HOST_ARCH),$(TARGET_ARCH))
+ endif # ifeq ($(HOST_OS),$(TARGET))
+ endif
+ 
++LIBFLAGS += -s TOTAL_STACK=1048576 \
++            -Wl,--export=__data_end -Wl,--export=__heap_base \
++            -s ERROR_ON_UNDEFINED_SYMBOLS=0
++
+ # This library is the main target for this makefile. It will contain a minimal
+ # runtime that can be linked in to other programs.
+ LIB_NAME := libtensorflow-lite.a
+ 
+ # Benchmark static library and binary
+ BENCHMARK_LIB_NAME := benchmark-lib.a
+-BENCHMARK_BINARY_NAME := benchmark_model
+-BENCHMARK_PERF_OPTIONS_BINARY_NAME := benchmark_model_performance_options
++BENCHMARK_BINARY_NAME := benchmark_model.wasm
++BENCHMARK_PERF_OPTIONS_BINARY_NAME := benchmark_model_performance_options.wasm
+ 
+ # A small example program that shows how to link against the library.
+ MINIMAL_SRCS := \
+@@ -277,12 +277,16 @@ LIB_PATH := $(LIBDIR)$(LIB_NAME)
+ BENCHMARK_LIB := $(LIBDIR)$(BENCHMARK_LIB_NAME)
+ BENCHMARK_BINARY := $(BINDIR)$(BENCHMARK_BINARY_NAME)
+ BENCHMARK_PERF_OPTIONS_BINARY := $(BINDIR)$(BENCHMARK_PERF_OPTIONS_BINARY_NAME)
+-MINIMAL_BINARY := $(BINDIR)minimal
++MINIMAL_BINARY := $(BINDIR)minimal.wasm
+ LABEL_IMAGE_BINARY := $(BINDIR)label_image
+ 
+-CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++
+-CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc
+-AR := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}ar
++# CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++
++# CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc
++# AR := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}ar
++
++CXX := em++
++CC := emcc
++AR := emar
+ 
+ MINIMAL_OBJS := $(addprefix $(OBJDIR), \
+ $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MINIMAL_SRCS))))
+diff --git a/tensorflow/lite/tools/make/targets/linux_makefile.inc b/tensorflow/lite/tools/make/targets/linux_makefile.inc
+index 222cef9e5f..eea89a38f0 100644
+--- a/tensorflow/lite/tools/make/targets/linux_makefile.inc
++++ b/tensorflow/lite/tools/make/targets/linux_makefile.inc
+@@ -2,12 +2,10 @@
+ ifeq ($(TARGET), linux)
+   CXXFLAGS += \
+     -fPIC \
+-    -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \
+-    -pthread
++    -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK
+   CFLAGS += \
+     -fPIC \
+-    -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \
+-    -pthread    
++    -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK
+   # TODO(petewarden): In the future we may want to add architecture-specific
+   # flags like -msse4.2
+ 	LIBS += -ldl