christoph/wamr
1
Fork 0
Code Releases Activity

[wasi-nn] Add a new wasi-nn backend openvino (#3603)

Browse Source
This commit is contained in:
liang.he
2024-07-22 17:16:41 +08:00
committed by GitHub
parent 50f28495a1
commit 058bc47102
12 changed files with 1037 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c
View File

@ -88,7 +88,7 @@ graph_builder_array_app_native(wasm_module_inst_t instance,
}
NN_DBG_PRINTF("Graph builder %d contains %d elements", i,
builder->size);
builder[i].size);
}
builder_array->buf = builder;

70
core/iwasm/libraries/wasi-nn/src/wasi_nn.c
View File

@ -26,15 +26,17 @@
// so, just keep one `api_function` is enough
static api_function lookup = { 0 };
#define call_wasi_nn_func(wasi_error, func, ...) \
do { \
if (lookup.func) { \
wasi_error = lookup.func(__VA_ARGS__); \
} \
else { \
NN_ERR_PRINTF("Error: %s is not registered", #func); \
wasi_error = unsupported_operation; \
} \
#define call_wasi_nn_func(wasi_error, func, ...) \
do { \
if (lookup.func) { \
wasi_error = lookup.func(__VA_ARGS__); \
if (wasi_error != success) \
NN_ERR_PRINTF("Error %s: %d", #func, wasi_error); \
} \
else { \
NN_ERR_PRINTF("Error %s is not registered", #func); \
wasi_error = unsupported_operation; \
} \
} while (0)
static HashMap *hashmap;
@ -68,7 +70,9 @@ key_equal_func(void *key1, void *key2)
static void
key_destroy_func(void *key1)
{}
{
/* key type is wasm_module_inst_t*. do nothing */
}
static void
value_destroy_func(void *value)
@ -79,7 +83,8 @@ value_destroy_func(void *value)
static WASINNContext *
wasi_nn_initialize_context()
{
NN_DBG_PRINTF("Initializing wasi-nn context");
NN_DBG_PRINTF("[WASI NN] INIT...");
WASINNContext *wasi_nn_ctx =
(WASINNContext *)wasm_runtime_malloc(sizeof(WASINNContext));
if (wasi_nn_ctx == NULL) {
@ -92,7 +97,6 @@ wasi_nn_initialize_context()
wasi_nn_error res;
call_wasi_nn_func(res, init, &wasi_nn_ctx->backend_ctx);
if (res != success) {
NN_ERR_PRINTF("Error while initializing backend");
wasm_runtime_free(wasi_nn_ctx);
return NULL;
}
@ -103,7 +107,7 @@ wasi_nn_initialize_context()
static bool
wasi_nn_initialize()
{
NN_DBG_PRINTF("Initializing wasi-nn");
NN_DBG_PRINTF("[WASI NN General] Initializing wasi-nn");
// hashmap { instance: wasi_nn_ctx }
hashmap = bh_hash_map_create(HASHMAP_INITIAL_SIZE, true, hash_func,
key_equal_func, key_destroy_func,
@ -133,13 +137,15 @@ wasm_runtime_get_wasi_nn_ctx(wasm_module_inst_t instance)
return NULL;
}
}
NN_DBG_PRINTF("Returning ctx");
return wasi_nn_ctx;
}
static void
wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx)
{
NN_DBG_PRINTF("[WASI NN] DEINIT...");
if (wasi_nn_ctx == NULL) {
NN_ERR_PRINTF(
"Error when deallocating memory. WASI-NN context is NULL");
@ -152,9 +158,6 @@ wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx)
/* only one backend can be registered */
wasi_nn_error res;
call_wasi_nn_func(res, deinit, wasi_nn_ctx->backend_ctx);
if (res != success) {
NN_ERR_PRINTF("Error while destroyging backend");
}
wasm_runtime_free(wasi_nn_ctx);
}
@ -191,7 +194,7 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder,
graph_encoding encoding, execution_target target, graph *g)
#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */
{
NN_DBG_PRINTF("Running wasi_nn_load [encoding=%d, target=%d]...", encoding,
NN_DBG_PRINTF("[WASI NN] LOAD [encoding=%d, target=%d]...", encoding,
target);
wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
@ -222,7 +225,6 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder,
WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance);
call_wasi_nn_func(res, load, wasi_nn_ctx->backend_ctx, &builder_native,
encoding, target, g);
NN_DBG_PRINTF("wasi_nn_load finished with status %d [graph=%d]", res, *g);
if (res != success)
goto fail;
@ -241,7 +243,7 @@ wasi_nn_error
wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len,
graph *g)
{
NN_DBG_PRINTF("Running wasi_nn_load_by_name ...");
NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", name);
wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
if (!instance) {
@ -261,7 +263,6 @@ wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len,
wasi_nn_error res;
call_wasi_nn_func(res, load_by_name, wasi_nn_ctx->backend_ctx, name,
name_len, g);
NN_DBG_PRINTF("wasi_nn_load_by_name finished with status %d", *g);
if (res != success)
return res;
@ -274,7 +275,7 @@ wasi_nn_error
wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g,
graph_execution_context *ctx)
{
NN_DBG_PRINTF("Running wasi_nn_init_execution_context [graph=%d]...", g);
NN_DBG_PRINTF("[WASI NN] INIT_EXECUTION_CONTEXT...");
wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
if (!instance) {
@ -295,9 +296,6 @@ wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g,
call_wasi_nn_func(res, init_execution_context, wasi_nn_ctx->backend_ctx, g,
ctx);
NN_DBG_PRINTF(
"wasi_nn_init_execution_context finished with status %d [ctx=%d]", res,
*ctx);
return res;
}
@ -305,8 +303,7 @@ wasi_nn_error
wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx,
uint32_t index, tensor_wasm *input_tensor)
{
NN_DBG_PRINTF("Running wasi_nn_set_input [ctx=%d, index=%d]...", ctx,
index);
NN_DBG_PRINTF("[WASI NN] SET_INPUT [ctx=%d, index=%d]...", ctx, index);
wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
if (!instance) {
@ -331,14 +328,13 @@ wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx,
if (input_tensor_native.dimensions)
wasm_runtime_free(input_tensor_native.dimensions);
NN_DBG_PRINTF("wasi_nn_set_input finished with status %d", res);
return res;
}
wasi_nn_error
wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx)
{
NN_DBG_PRINTF("Running wasi_nn_compute [ctx=%d]...", ctx);
NN_DBG_PRINTF("[WASI NN] COMPUTE [ctx=%d]...", ctx);
wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
if (!instance) {
@ -352,7 +348,6 @@ wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx)
return res;
call_wasi_nn_func(res, compute, wasi_nn_ctx->backend_ctx, ctx);
NN_DBG_PRINTF("wasi_nn_compute finished with status %d", res);
return res;
}
@ -368,8 +363,7 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx,
uint32_t *output_tensor_size)
#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */
{
NN_DBG_PRINTF("Running wasi_nn_get_output [ctx=%d, index=%d]...", ctx,
index);
NN_DBG_PRINTF("[WASI NN] GET_OUTPUT [ctx=%d, index=%d]...", ctx, index);
wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
if (!instance) {
@ -396,8 +390,6 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx,
call_wasi_nn_func(res, get_output, wasi_nn_ctx->backend_ctx, ctx, index,
output_tensor, output_tensor_size);
#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */
NN_DBG_PRINTF("wasi_nn_get_output finished with status %d [data_size=%d]",
res, *output_tensor_size);
return res;
}
@ -435,7 +427,7 @@ get_wasi_nn_export_apis(NativeSymbol **p_native_symbols)
__attribute__((used)) uint32_t
get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols)
{
NN_DBG_PRINTF("--|> get_native_lib");
NN_DBG_PRINTF("[Native Register] get_native_lib");
#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0
*p_module_name = "wasi_ephemeral_nn";
@ -449,7 +441,7 @@ get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols)
__attribute__((used)) int
init_native_lib()
{
NN_DBG_PRINTF("--|> init_native_lib");
NN_DBG_PRINTF("[Native Register] init_native_lib");
if (!wasi_nn_initialize())
return 1;
@ -460,7 +452,7 @@ init_native_lib()
__attribute__((used)) void
deinit_native_lib()
{
NN_DBG_PRINTF("--|> deinit_native_lib");
NN_DBG_PRINTF("[Native Register] deinit_native_lib");
wasi_nn_destroy();
}
@ -468,7 +460,7 @@ deinit_native_lib()
__attribute__((used)) bool
wasi_nn_register_backend(api_function apis)
{
NN_DBG_PRINTF("--|> wasi_nn_register_backend");
NN_DBG_PRINTF("[Native Register] wasi_nn_register_backend");
lookup = apis;
return true;
}
}

559
core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c Normal file
View File

@ -0,0 +1,559 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "wasi_nn_types.h"
#include "wasi_nn_openvino.h"
#include "logger.h"
#include "bh_platform.h"
#include "openvino/c/openvino.h"
/*
* refer to
* https://docs.openvino.ai/2024/openvino-workflow/running-inference/integrate-openvino-with-your-application.html
*
* Steps about integrating OpenVINO are:
*
* 1. Create OpenVINO Runtime Core
* 2. Compile Model
* 3. Create Inference Request
* 4. Set Inputs
* 5. Start Inference
* 6. Process Inference Results
*
* from 4. to 6. is the Inference Loop
*/
typedef struct {
ov_core_t *core;
/* keep input model files */
void *weight_data;
ov_tensor_t *weights_tensor;
ov_model_t *model;
/* add prepostprocess */
ov_model_t *new_model;
ov_compiled_model_t *compiled_model;
ov_infer_request_t *infer_request;
ov_tensor_t *input_tensor;
} OpenVINOContext;
/*
* BE AWARE OF "goto fail"
*/
#define CHECK_OV_STATUS(status, error_code) \
do { \
ov_status_e s = status; \
if (s != OK) { \
NN_ERR_PRINTF("return status \"%s\", line %d", \
ov_get_error_info(s), __LINE__); \
error_code = runtime_error; \
goto fail; \
} \
} while (0)
static void
dump_ov_shape_t(const ov_shape_t *shape, int32_t output_len, char *output)
{
int ret = 0;
ret = snprintf(output, output_len, "%ld,[", shape->rank);
if (!ret)
return;
output_len -= ret;
output += ret;
for (unsigned i = 0; i < shape->rank && output_len; i++) {
ret = snprintf(output, output_len, " %ld", shape->dims[i]);
if (!ret)
return;
output_len -= ret;
output += ret;
}
snprintf(output, output_len, "]");
return;
}
#ifndef NDEBUG
static void
print_model_input_output_info(ov_model_t *model)
{
wasi_nn_error ov_error;
char *friendly_name = NULL;
size_t input_size = 0;
ov_output_const_port_t *input_port = NULL;
ov_shape_t input_shape = { 0 };
ov_element_type_e input_type;
char shape_info[64] = { 0 };
ov_output_const_port_t *output_port = NULL;
ov_shape_t output_shape = { 0 };
ov_element_type_e output_type;
CHECK_OV_STATUS(ov_model_get_friendly_name(model, &friendly_name),
ov_error);
NN_DBG_PRINTF("model name: %s", friendly_name);
ov_model_inputs_size(model, &input_size);
for (unsigned i = 0; i < input_size; i++) {
CHECK_OV_STATUS(ov_model_const_input_by_index(model, i, &input_port),
ov_error);
CHECK_OV_STATUS(ov_const_port_get_shape(input_port, &input_shape),
ov_error);
CHECK_OV_STATUS(ov_port_get_element_type(input_port, &input_type),
ov_error);
dump_ov_shape_t(&input_shape, 60, shape_info);
NN_DBG_PRINTF("model input[%u]. element_type: %d, shape: %s", i,
input_type, shape_info);
ov_shape_free(&input_shape);
memset(&input_shape, 0, sizeof(input_shape));
ov_output_const_port_free(input_port);
input_port = NULL;
}
size_t output_size = 0;
ov_model_outputs_size(model, &output_size);
for (unsigned i = 0; i < output_size; i++) {
CHECK_OV_STATUS(ov_model_const_output_by_index(model, i, &output_port),
ov_error);
CHECK_OV_STATUS(ov_const_port_get_shape(output_port, &output_shape),
ov_error);
CHECK_OV_STATUS(ov_port_get_element_type(output_port, &output_type),
ov_error);
dump_ov_shape_t(&output_shape, 60, shape_info);
NN_DBG_PRINTF("model output[%u]. element_type: %d, shape: %s", i,
output_type, shape_info);
ov_shape_free(&output_shape);
memset(&output_shape, 0, sizeof(output_shape));
ov_output_const_port_free(output_port);
output_port = NULL;
}
fail:
if (friendly_name)
ov_free(friendly_name);
ov_shape_free(&input_shape);
if (input_port)
ov_output_const_port_free(input_port);
ov_shape_free(&output_shape);
if (output_port)
ov_output_const_port_free(output_port);
return;
}
#endif
static ov_element_type_e
wasi_nn_tensor_type_to_openvino_element_type(tensor_type wasi_nn_type)
{
switch (wasi_nn_type) {
case fp16:
return F16;
case fp32:
return F32;
case fp64:
return F64;
case bf16:
return BF16;
case u8:
return U8;
case i32:
return I32;
case i64:
return I64;
default:
break;
}
NN_ERR_PRINTF("%d is an undefined tensor type", wasi_nn_type);
return UNDEFINED;
}
static wasi_nn_error
uint32_array_to_int64_array(uint32_t array_size, uint32_t *src, int64_t **dst)
{
*dst = malloc(array_size * sizeof(int64_t));
if (!(*dst))
return runtime_error;
for (unsigned i = 0; i < array_size; i++) {
(*dst)[i] = src[i];
}
return success;
}
wasi_nn_error
openvino_load(void *ctx, graph_builder_array *builder, graph_encoding encoding,
execution_target target, graph *g)
{
OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
wasi_nn_error ret = unsupported_operation;
if (encoding != openvino) {
NN_ERR_PRINTF("Unexpected encoding %d.", encoding);
return invalid_argument;
}
/*FIXME: unblock non-cpu device after supporting */
if (target != cpu) {
NN_ERR_PRINTF("Unexpected device %d.", target);
return invalid_argument;
}
if (builder->size != 2) {
NN_ERR_PRINTF("Unexpected builder format.");
return invalid_argument;
}
/*
* The first builder is the XML file.
* The second builder is the weight file.
*/
graph_builder xml = builder->buf[0];
graph_builder weight = builder->buf[1];
/* if xml is a String with a model in IR */
if (!(xml.buf[xml.size] == '\0' && xml.buf[xml.size - 1] != '\0')) {
NN_ERR_PRINTF("Invalid xml string.");
return invalid_argument;
}
/* transfer weight to an ov tensor */
{
ov_ctx->weight_data = malloc(weight.size);
if (!ov_ctx->weight_data)
goto fail;
memcpy(ov_ctx->weight_data, weight.buf, weight.size);
ov_element_type_e type = U8;
int64_t dims[1] = { weight.size };
ov_shape_t shape = { 1, dims };
CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(type, shape,
ov_ctx->weight_data,
&ov_ctx->weights_tensor),
ret);
}
/* load model from buffer */
CHECK_OV_STATUS(ov_core_read_model_from_memory_buffer(
ov_ctx->core, (char *)xml.buf, xml.size,
ov_ctx->weights_tensor, &ov_ctx->model),
ret);
#ifndef NDEBUG
print_model_input_output_info(ov_ctx->model);
#endif
ret = success;
fail:
return ret;
}
wasi_nn_error
openvino_load_by_name(void *ctx, const char *filename, uint32_t filename_len,
graph *g)
{
OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
wasi_nn_error ret = unsupported_operation;
CHECK_OV_STATUS(
ov_core_read_model(ov_ctx->core, filename, NULL, &ov_ctx->model), ret);
ret = success;
fail:
return ret;
}
wasi_nn_error
openvino_init_execution_context(void *ctx, graph g,
graph_execution_context *exec_ctx)
{
return success;
}
wasi_nn_error
openvino_set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index,
tensor *wasi_nn_tensor)
{
OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
wasi_nn_error ret = unsupported_operation;
ov_shape_t input_shape = { 0 };
int64_t *ov_dims = NULL;
ov_preprocess_prepostprocessor_t *ppp = NULL;
ov_preprocess_input_info_t *input_info = NULL;
ov_preprocess_input_tensor_info_t *input_tensor_info = NULL;
ov_layout_t *input_layout = NULL;
ov_preprocess_preprocess_steps_t *input_process = NULL;
ov_preprocess_input_model_info_t *p_input_model = NULL;
ov_layout_t *model_layout = NULL;
ov_preprocess_output_info_t *output_info = NULL;
ov_preprocess_output_tensor_info_t *output_tensor_info = NULL;
/* wasi_nn_tensor -> ov_tensor */
{
ret = uint32_array_to_int64_array(wasi_nn_tensor->dimensions->size,
wasi_nn_tensor->dimensions->buf,
&ov_dims);
if (ret != success)
goto fail;
/* NCHW -> NHWC */
if (wasi_nn_tensor->dimensions->size == 4 || ov_dims[1] == 3) {
/* N */
/* H */
ov_dims[1] = ov_dims[2];
/* W */
ov_dims[2] = ov_dims[3];
/* C */
ov_dims[3] = 3;
}
CHECK_OV_STATUS(ov_shape_create(wasi_nn_tensor->dimensions->size,
ov_dims, &input_shape),
ret);
ov_element_type_e input_type =
wasi_nn_tensor_type_to_openvino_element_type(wasi_nn_tensor->type);
if (input_type == UNDEFINED)
goto fail;
char shape_info[64] = { 0 };
dump_ov_shape_t(&input_shape, 60, shape_info);
NN_DBG_PRINTF("input tensor. element_type: %d, shape: %s", input_type,
shape_info);
CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(input_type, input_shape,
wasi_nn_tensor->data,
&ov_ctx->input_tensor),
ret);
}
/* set preprocess based on wasi_nn_tensor */
{
CHECK_OV_STATUS(
ov_preprocess_prepostprocessor_create(ov_ctx->model, &ppp), ret);
/* reuse user' created tensor's info */
CHECK_OV_STATUS(ov_preprocess_prepostprocessor_get_input_info_by_index(
ppp, index, &input_info),
ret);
CHECK_OV_STATUS(ov_preprocess_input_info_get_tensor_info(
input_info, &input_tensor_info),
ret);
CHECK_OV_STATUS(ov_preprocess_input_tensor_info_set_from(
input_tensor_info, ov_ctx->input_tensor),
ret);
/* ! HAS TO BE NHWC. Match previous layout conversion */
CHECK_OV_STATUS(ov_layout_create("NHWC", &input_layout), ret);
CHECK_OV_STATUS(ov_preprocess_input_tensor_info_set_layout(
input_tensor_info, input_layout),
ret);
/* add RESIZE */
CHECK_OV_STATUS(ov_preprocess_input_info_get_preprocess_steps(
input_info, &input_process),
ret);
CHECK_OV_STATUS(
ov_preprocess_preprocess_steps_resize(input_process, RESIZE_LINEAR),
ret);
/* input model */
CHECK_OV_STATUS(
ov_preprocess_input_info_get_model_info(input_info, &p_input_model),
ret);
// TODO: what if not?
CHECK_OV_STATUS(ov_layout_create("NCHW", &model_layout), ret);
CHECK_OV_STATUS(ov_preprocess_input_model_info_set_layout(p_input_model,
model_layout),
ret);
/* output -> F32(possibility) */
CHECK_OV_STATUS(ov_preprocess_prepostprocessor_get_output_info_by_index(
ppp, index, &output_info),
ret);
CHECK_OV_STATUS(ov_preprocess_output_info_get_tensor_info(
output_info, &output_tensor_info),
ret);
CHECK_OV_STATUS(
ov_preprocess_output_set_element_type(output_tensor_info, F32),
ret);
CHECK_OV_STATUS(
ov_preprocess_prepostprocessor_build(ppp, &ov_ctx->new_model), ret);
}
CHECK_OV_STATUS(ov_core_compile_model(ov_ctx->core, ov_ctx->new_model,
"CPU", 0, &ov_ctx->compiled_model),
ret);
CHECK_OV_STATUS(ov_compiled_model_create_infer_request(
ov_ctx->compiled_model, &ov_ctx->infer_request),
ret);
/* install ov_tensor -> infer_request */
CHECK_OV_STATUS(ov_infer_request_set_input_tensor_by_index(
ov_ctx->infer_request, index, ov_ctx->input_tensor),
ret);
ret = success;
fail:
if (ov_dims)
free(ov_dims);
ov_shape_free(&input_shape);
if (ppp)
ov_preprocess_prepostprocessor_free(ppp);
if (input_info)
ov_preprocess_input_info_free(input_info);
if (input_tensor_info)
ov_preprocess_input_tensor_info_free(input_tensor_info);
if (input_layout)
ov_layout_free(input_layout);
if (input_process)
ov_preprocess_preprocess_steps_free(input_process);
if (p_input_model)
ov_preprocess_input_model_info_free(p_input_model);
if (model_layout)
ov_layout_free(model_layout);
if (output_info)
ov_preprocess_output_info_free(output_info);
if (output_tensor_info)
ov_preprocess_output_tensor_info_free(output_tensor_info);
return ret;
}
wasi_nn_error
openvino_compute(void *ctx, graph_execution_context exec_ctx)
{
OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
wasi_nn_error ret = unsupported_operation;
CHECK_OV_STATUS(ov_infer_request_infer(ov_ctx->infer_request), ret);
ret = success;
fail:
return ret;
}
wasi_nn_error
openvino_get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index,
tensor_data output_tensor, uint32_t *output_tensor_size)
{
OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
wasi_nn_error ret = unsupported_operation;
ov_tensor_t *ov_tensor = NULL;
void *data = NULL;
size_t byte_size = 0;
CHECK_OV_STATUS(ov_infer_request_get_output_tensor_by_index(
ov_ctx->infer_request, index, &ov_tensor),
ret);
CHECK_OV_STATUS(ov_tensor_get_byte_size(ov_tensor, &byte_size), ret);
CHECK_OV_STATUS(ov_tensor_data(ov_tensor, &data), ret);
memcpy(output_tensor, data, byte_size);
*output_tensor_size = (uint32_t)byte_size;
ret = success;
fail:
if (ov_tensor)
ov_tensor_free(ov_tensor);
return ret;
}
wasi_nn_error
openvino_initialize(void **ctx)
{
ov_version_t version;
OpenVINOContext *ov_ctx = NULL;
wasi_nn_error ret = unsupported_operation;
if (!ctx) {
ret = invalid_argument;
goto fail;
}
/* Get OpenVINO runtime version */
CHECK_OV_STATUS(ov_get_openvino_version(&version), ret);
NN_INFO_PRINTF("OpenVINO INFO:");
NN_INFO_PRINTF(" Description : %s", version.description);
NN_INFO_PRINTF(" Build Number: %s", version.buildNumber);
ov_version_free(&version);
ov_ctx = (OpenVINOContext *)os_malloc(sizeof(OpenVINOContext));
if (!ov_ctx) {
NN_ERR_PRINTF("Allocate for OpenVINOContext failed");
ret = runtime_error;
goto fail;
}
memset(ov_ctx, 0, sizeof(OpenVINOContext));
/* Initialize OpenVINO Runtime Core */
CHECK_OV_STATUS(ov_core_create(&ov_ctx->core), ret);
*ctx = (void *)ov_ctx;
return success;
fail:
openvino_destroy((void *)ov_ctx);
return ret;
}
wasi_nn_error
openvino_destroy(void *ctx)
{
OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
if (!ov_ctx)
return invalid_argument;
if (ov_ctx->weight_data)
free(ov_ctx->weight_data);
if (ov_ctx->weights_tensor)
ov_tensor_free(ov_ctx->weights_tensor);
if (ov_ctx->input_tensor)
ov_tensor_free(ov_ctx->input_tensor);
if (ov_ctx->infer_request)
ov_infer_request_free(ov_ctx->infer_request);
if (ov_ctx->compiled_model)
ov_compiled_model_free(ov_ctx->compiled_model);
if (ov_ctx->model)
ov_model_free(ov_ctx->model);
if (ov_ctx->core)
ov_core_free(ov_ctx->core);
os_free(ov_ctx);
return success;
}
__attribute__((constructor(200))) void
openvino_register_backend()
{
api_function apis = {
.load = openvino_load,
.load_by_name = openvino_load_by_name,
.init_execution_context = openvino_init_execution_context,
.set_input = openvino_set_input,
.compute = openvino_compute,
.get_output = openvino_get_output,
.init = openvino_initialize,
.deinit = openvino_destroy,
};
wasi_nn_register_backend(apis);
}

36
core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef WASI_NN_OPENVINO_HPP
#define WASI_NN_OPENVINO_HPP
#include "wasi_nn_types.h"
wasi_nn_error
openvino_load(void *ctx, graph_builder_array *builder, graph_encoding encoding,
execution_target target, graph *g);
wasi_nn_error
openvino_init_execution_context(void *ctx, graph g,
graph_execution_context *exec_ctx);
wasi_nn_error
openvino_set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index,
tensor *input_tensor);
wasi_nn_error
openvino_compute(void *ctx, graph_execution_context exec_ctx);
wasi_nn_error
openvino_get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index,
tensor_data output_tensor, uint32_t *output_tensor_size);
wasi_nn_error
openvino_initialize(void **ctx);
wasi_nn_error
openvino_destroy(void *ctx);
#endif /* WASI_NN_OPENVINO_HPP */