wasi-nn: Add a new target for llama.cpp as a wasi-nn backend (#3709)
Minimum support: - [x] accept (WasmEdge) customized model parameters. metadata. - [x] Target [wasmedge-ggml examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master/wasmedge-ggml) - [x] basic - [x] chatml - [x] gemma - [x] llama - [x] qwen --- In the future, to support if required: - [ ] Target [wasmedge-ggml examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master/wasmedge-ggml) - [ ] command-r. (>70G memory requirement) - [ ] embedding. (embedding mode) - [ ] grammar. (use the grammar option to constrain the model to generate the JSON output) - [ ] llama-stream. (new APIS `compute_single`, `get_output_single`, `fini_single`) - [ ] llava. (image representation) - [ ] llava-base64-stream. (image representation) - [ ] multimodel. (image representation) - [ ] Target [llamaedge](https://github.com/LlamaEdge/LlamaEdge)
This commit is contained in:
liang.he
17
core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake
Normal file
17
core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake
Normal file
@ -0,0 +1,17 @@
|
||||
# Copyright (C) 2019 Intel Corporation. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
include(FetchContent)
|
||||
|
||||
set(CJSON_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/cjson")
|
||||
|
||||
FetchContent_Declare(
|
||||
cjson
|
||||
GIT_REPOSITORY https://github.com/DaveGamble/cJSON.git
|
||||
GIT_TAG v1.7.18
|
||||
SOURCE_DIR ${CJSON_SOURCE_DIR}
|
||||
)
|
||||
|
||||
set(ENABLE_CJSON_TEST OFF CACHE INTERNAL "Turn off tests")
|
||||
set(ENABLE_CJSON_UNINSTALL OFF CACHE INTERNAL "Turn off uninstall to avoid targets conflict")
|
||||
FetchContent_MakeAvailable(cjson)
|
||||
18
core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake
Normal file
18
core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake
Normal file
@ -0,0 +1,18 @@
|
||||
# Copyright (C) 2019 Intel Corporation. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
include(FetchContent)
|
||||
|
||||
set(LLAMA_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/llama.cpp")
|
||||
|
||||
FetchContent_Declare(
|
||||
llamacpp
|
||||
GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git
|
||||
GIT_TAG b3573
|
||||
SOURCE_DIR ${LLAMA_SOURCE_DIR}
|
||||
)
|
||||
|
||||
set(LLAMA_BUILD_TESTS OFF)
|
||||
set(LLAMA_BUILD_EXAMPLES OFF)
|
||||
set(LLAMA_BUILD_SERVER OFF)
|
||||
FetchContent_MakeAvailable(llamacpp)
|
||||
@ -1,47 +1,25 @@
|
||||
# Copyright (C) 2019 Intel Corporation. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
find_library(TENSORFLOW_LITE
|
||||
NAMES tensorflow-lite
|
||||
HINTS ${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite
|
||||
NO_DEFAULT_PATHS
|
||||
include(FetchContent)
|
||||
|
||||
set(TFLITE_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src")
|
||||
|
||||
FetchContent_Declare(
|
||||
tensorflow_lite
|
||||
GIT_REPOSITORY https://github.com/tensorflow/tensorflow.git
|
||||
GIT_TAG v2.12.0
|
||||
GIT_SHALLOW ON
|
||||
GIT_PROGRESS ON
|
||||
SOURCE_DIR ${TFLITE_SOURCE_DIR}
|
||||
SOURCE_SUBDIR tensorflow/lite
|
||||
)
|
||||
|
||||
if(NOT TENSORFLOW_LITE)
|
||||
if(NOT EXISTS "${WAMR_ROOT_DIR}/core/deps/tensorflow-src")
|
||||
execute_process(
|
||||
COMMAND "${WAMR_ROOT_DIR}/core/deps/install_tensorflow.sh"
|
||||
RESULT_VARIABLE TENSORFLOW_RESULT
|
||||
)
|
||||
else()
|
||||
message("Tensorflow is already downloaded.")
|
||||
endif()
|
||||
|
||||
set(TENSORFLOW_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src")
|
||||
|
||||
if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1)
|
||||
# Tensorflow specific:
|
||||
# * https://www.tensorflow.org/lite/guide/build_cmake#available_options_to_build_tensorflow_lite
|
||||
set (TFLITE_ENABLE_GPU ON)
|
||||
endif()
|
||||
|
||||
if (CMAKE_SIZEOF_VOID_P EQUAL 4)
|
||||
set (TFLITE_ENABLE_XNNPACK OFF)
|
||||
endif()
|
||||
|
||||
add_subdirectory(
|
||||
"${TENSORFLOW_SOURCE_DIR}/tensorflow/lite"
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite"
|
||||
EXCLUDE_FROM_ALL
|
||||
)
|
||||
else ()
|
||||
message(STATUS "TensorFlow Lite library found: ${TENSORFLOW_LITE}")
|
||||
set(TENSORFLOW_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src")
|
||||
if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1)
|
||||
set(TFLITE_ENABLE_GPU ON)
|
||||
endif()
|
||||
if (CMAKE_SIZEOF_VOID_P EQUAL 4)
|
||||
set(TFLITE_ENABLE_XNNPACK OFF)
|
||||
endif()
|
||||
|
||||
set(TENSORFLOW_LITE_INCLUDE_DIR "${TENSORFLOW_SOURCE_DIR}/tensorflow/lite")
|
||||
set(FLATBUFFER_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers/include")
|
||||
|
||||
include_directories(${TENSORFLOW_SOURCE_DIR})
|
||||
include_directories(${FLATBUFFER_INCLUDE_DIR})
|
||||
link_directories(${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite)
|
||||
FetchContent_MakeAvailable(tensorflow_lite)
|
||||
|
||||
@ -3,27 +3,6 @@
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
|
||||
|
||||
if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1)
|
||||
# Find tensorflow-lite
|
||||
find_package(tensorflow_lite REQUIRED)
|
||||
endif()
|
||||
|
||||
if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
|
||||
if(NOT DEFINED ENV{OpenVINO_DIR})
|
||||
message(FATAL_ERROR
|
||||
"OpenVINO_DIR is not defined. "
|
||||
"Please follow https://docs.openvino.ai/2024/get-started/install-openvino.html,"
|
||||
"install openvino, and set environment variable OpenVINO_DIR."
|
||||
"Like OpenVINO_DIR=/usr/lib/openvino-2023.2/ cmake ..."
|
||||
"Or OpenVINO_DIR=/opt/intel/openvino/ cmake ..."
|
||||
)
|
||||
endif()
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH $ENV{OpenVINO_DIR})
|
||||
# Find OpenVINO
|
||||
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
|
||||
endif()
|
||||
|
||||
#
|
||||
# wasi-nn general
|
||||
set(WASI_NN_ROOT ${CMAKE_CURRENT_LIST_DIR}/..)
|
||||
@ -42,22 +21,46 @@ add_compile_definitions(
|
||||
#
|
||||
# - tflite
|
||||
if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1)
|
||||
find_package(tensorflow_lite REQUIRED)
|
||||
|
||||
add_library(
|
||||
wasi_nn_tflite
|
||||
SHARED
|
||||
${WASI_NN_ROOT}/src/wasi_nn_tensorflowlite.cpp
|
||||
)
|
||||
|
||||
target_include_directories(
|
||||
wasi_nn_tflite
|
||||
PUBLIC
|
||||
${tensorflow_lite_SOURCE_DIR}
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
wasi_nn_tflite
|
||||
PUBLIC
|
||||
libiwasm
|
||||
tensorflow-lite
|
||||
)
|
||||
|
||||
install(TARGETS wasi_nn_tflite DESTINATION lib)
|
||||
endif()
|
||||
|
||||
# - openvino
|
||||
if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
|
||||
if(NOT DEFINED ENV{OpenVINO_DIR})
|
||||
message(FATAL_ERROR
|
||||
"OpenVINO_DIR is not defined. "
|
||||
"Please follow https://docs.openvino.ai/2024/get-started/install-openvino.html,"
|
||||
"install openvino, and set environment variable OpenVINO_DIR."
|
||||
"Like OpenVINO_DIR=/usr/lib/openvino-2023.2/ cmake ..."
|
||||
"Or OpenVINO_DIR=/opt/intel/openvino/ cmake ..."
|
||||
)
|
||||
endif()
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH $ENV{OpenVINO_DIR})
|
||||
# Find OpenVINO
|
||||
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
|
||||
|
||||
add_library(
|
||||
wasi_nn_openvino
|
||||
SHARED
|
||||
@ -71,4 +74,37 @@ if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
|
||||
openvino::runtime
|
||||
openvino::runtime::c
|
||||
)
|
||||
endif()
|
||||
|
||||
install(TARGETS wasi_nn_openvino DESTINATION lib)
|
||||
endif()
|
||||
|
||||
# - llamacpp
|
||||
|
||||
if(WAMR_BUILD_WASI_NN_LLAMACPP EQUAL 1)
|
||||
find_package(cjson REQUIRED)
|
||||
find_package(llamacpp REQUIRED)
|
||||
|
||||
add_library(
|
||||
wasi_nn_llamacpp
|
||||
SHARED
|
||||
${WASI_NN_ROOT}/src/wasi_nn_llamacpp.c
|
||||
)
|
||||
|
||||
target_include_directories(
|
||||
wasi_nn_llamacpp
|
||||
PUBLIC
|
||||
${cjson_SOURCE_DIR}
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
wasi_nn_llamacpp
|
||||
PUBLIC
|
||||
libiwasm
|
||||
cjson
|
||||
common
|
||||
ggml
|
||||
llama
|
||||
)
|
||||
|
||||
install(TARGETS wasi_nn_llamacpp DESTINATION lib)
|
||||
endif()
|
||||
|
||||
Reference in New Issue
Block a user