From 7c7684819dc42f1104f835a187269dffc6d0ee15 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 10 Jan 2024 16:44:09 +0800 Subject: [PATCH] Register quick call entries to speedup the aot/jit func call process (#2978) In some scenarios there may be lots of callings to AOT/JIT functions from the host embedder, which expects good performance for the calling process, while in the current implementation, runtime calls the wasm_runtime_invoke_native to prepare the array of registers and stacks for the invokeNative assemble code, and the latter then puts the elements in the array to physical registers and native stacks and calls the AOT/JIT function, there may be many data copying and handlings which impact the performance. This PR registers some quick AOT/JIT entries for some simple wasm signatures, and let runtime call the entry to directly invoke the AOT/JIT function instead of calling wasm_runtime_invoke_native, which speedups the calling process. We may extend the mechanism next to allow the developer to register his quick AOT/JIT entries to speedup the calling process of invoking the AOT/JIT functions for some specific signatures. --- build-scripts/config_common.cmake | 13 +- core/config.h | 7 + core/iwasm/aot/aot_loader.c | 5 + core/iwasm/aot/aot_runtime.c | 39 +- core/iwasm/common/wasm_native.c | 1079 ++++++++++++++++-- core/iwasm/common/wasm_native.h | 5 + core/iwasm/interpreter/wasm.h | 4 + core/iwasm/interpreter/wasm_interp_classic.c | 27 +- core/iwasm/interpreter/wasm_loader.c | 4 + core/iwasm/interpreter/wasm_mini_loader.c | 4 + 10 files changed, 1044 insertions(+), 143 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index ca4935e3..212141d5 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -449,7 +449,18 @@ if (WAMR_CONFIGUABLE_BOUNDS_CHECKS EQUAL 1) endif () if (WAMR_BUILD_LINUX_PERF EQUAL 1) add_definitions (-DWASM_ENABLE_LINUX_PERF=1) - message (" Enable linux perf support") + message (" Linux perf support enabled") +endif () +if (NOT DEFINED WAMR_BUILD_QUICK_AOT_ENTRY) + # Enable quick aot/jit entries by default + set (WAMR_BUILD_QUICK_AOT_ENTRY 1) +endif () +if (WAMR_BUILD_QUICK_AOT_ENTRY EQUAL 1) + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=1) + message (" Quick AOT/JIT entries enabled") +else () + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=0) + message (" Quick AOT/JIT entries disabled") endif () if (APPLE) diff --git a/core/config.h b/core/config.h index c30d0ace..e067ea27 100644 --- a/core/config.h +++ b/core/config.h @@ -495,4 +495,11 @@ #define WASM_ENABLE_LINUX_PERF 0 #endif +/* Support registering quick AOT/JIT function entries of some func types + to speedup the calling process of invoking the AOT/JIT functions of + these types from the host embedder */ +#ifndef WASM_ENABLE_QUICK_AOT_ENTRY +#define WASM_ENABLE_QUICK_AOT_ENTRY 1 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 341f66ae..d0d0c369 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1202,6 +1202,11 @@ load_func_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, func_types[i]->param_cell_num = (uint16)param_cell_num; func_types[i]->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + func_types[i]->quick_aot_entry = + wasm_native_lookup_quick_aot_entry(func_types[i]); +#endif } *p_buf = buf; diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index d806e064..ac83d5c2 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1385,9 +1385,6 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; uint32 page_size = os_getpagesize(); uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - uint16 param_count = func_type->param_count; - uint16 result_count = func_type->result_count; - const uint8 *types = func_type->types; #ifdef BH_PLATFORM_WINDOWS int result; bool has_exception; @@ -1426,28 +1423,22 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { - /* Quick call with func_ptr if the function signature is simple */ - if (!signature && param_count == 1 && types[0] == VALUE_TYPE_I32) { - if (result_count == 0) { - void (*NativeFunc)(WASMExecEnv *, uint32) = - (void (*)(WASMExecEnv *, uint32))func_ptr; - NativeFunc(exec_env, argv[0]); - ret = aot_copy_exception(module_inst, NULL) ? false : true; - } - else if (result_count == 1 - && types[param_count] == VALUE_TYPE_I32) { - uint32 (*NativeFunc)(WASMExecEnv *, uint32) = - (uint32(*)(WASMExecEnv *, uint32))func_ptr; - argv_ret[0] = NativeFunc(exec_env, argv[0]); - ret = aot_copy_exception(module_inst, NULL) ? false : true; - } - else { - ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type, - signature, attachment, argv, - argc, argv_ret); - } +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick aot entry is registered */ + if (!signature && func_type->quick_aot_entry) { + void (*invoke_native)( + void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) = func_type->quick_aot_entry; + invoke_native(func_ptr, + func_type->result_count > 0 + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID, + exec_env, argv, argv_ret); + ret = !aot_copy_exception(module_inst, NULL); } - else { + else +#endif + { ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, attachment, argv, argc, argv_ret); diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index c42bf7b3..69ae6f2a 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -16,19 +16,6 @@ #include "../libraries/thread-mgr/thread_manager.h" #endif -#if !defined(BH_PLATFORM_ZEPHYR) && !defined(BH_PLATFORM_ALIOS_THINGS) \ - && !defined(BH_PLATFORM_OPENRTOS) && !defined(BH_PLATFORM_ESP_IDF) -#define ENABLE_QUICKSORT 1 -#else -#define ENABLE_QUICKSORT 0 -#endif - -#define ENABLE_SORT_DEBUG 0 - -#if ENABLE_SORT_DEBUG != 0 -#include -#endif - static NativeSymbolsList g_native_symbols_list = NULL; #if WASM_ENABLE_LIBC_WASI != 0 @@ -171,93 +158,26 @@ check_symbol_signature(const WASMType *type, const char *signature) return true; } -#if ENABLE_QUICKSORT == 0 -static void -sort_symbol_ptr(NativeSymbol *native_symbols, uint32 n_native_symbols) +static int +native_symbol_cmp(const void *native_symbol1, const void *native_symbol2) { - uint32 i, j; - NativeSymbol temp; - - for (i = 0; i < n_native_symbols - 1; i++) { - for (j = i + 1; j < n_native_symbols; j++) { - if (strcmp(native_symbols[i].symbol, native_symbols[j].symbol) - > 0) { - temp = native_symbols[i]; - native_symbols[i] = native_symbols[j]; - native_symbols[j] = temp; - } - } - } + return strcmp(((const NativeSymbol *)native_symbol1)->symbol, + ((const NativeSymbol *)native_symbol2)->symbol); } -#else -static void -swap_symbol(NativeSymbol *left, NativeSymbol *right) -{ - NativeSymbol temp = *left; - *left = *right; - *right = temp; -} - -static void -quick_sort_symbols(NativeSymbol *native_symbols, int left, int right) -{ - NativeSymbol base_symbol; - int pin_left = left; - int pin_right = right; - - if (left >= right) { - return; - } - - base_symbol = native_symbols[left]; - while (left < right) { - while (left < right - && strcmp(native_symbols[right].symbol, base_symbol.symbol) - > 0) { - right--; - } - - if (left < right) { - swap_symbol(&native_symbols[left], &native_symbols[right]); - left++; - } - - while (left < right - && strcmp(native_symbols[left].symbol, base_symbol.symbol) < 0) { - left++; - } - - if (left < right) { - swap_symbol(&native_symbols[left], &native_symbols[right]); - right--; - } - } - native_symbols[left] = base_symbol; - - quick_sort_symbols(native_symbols, pin_left, left - 1); - quick_sort_symbols(native_symbols, left + 1, pin_right); -} -#endif /* end of ENABLE_QUICKSORT */ static void * lookup_symbol(NativeSymbol *native_symbols, uint32 n_native_symbols, const char *symbol, const char **p_signature, void **p_attachment) { - int low = 0, mid, ret; - int high = (int32)n_native_symbols - 1; + NativeSymbol *native_symbol, key = { 0 }; - while (low <= high) { - mid = (low + high) / 2; - ret = strcmp(symbol, native_symbols[mid].symbol); - if (ret == 0) { - *p_signature = native_symbols[mid].signature; - *p_attachment = native_symbols[mid].attachment; - return native_symbols[mid].func_ptr; - } - else if (ret < 0) - high = mid - 1; - else - low = mid + 1; + key.symbol = symbol; + + if ((native_symbol = bsearch(&key, native_symbols, n_native_symbols, + sizeof(NativeSymbol), native_symbol_cmp))) { + *p_signature = native_symbol->signature; + *p_attachment = native_symbol->attachment; + return native_symbol->func_ptr; } return NULL; @@ -328,11 +248,6 @@ register_natives(const char *module_name, NativeSymbol *native_symbols, uint32 n_native_symbols, bool call_conv_raw) { NativeSymbolsNode *node; -#if ENABLE_SORT_DEBUG != 0 - struct timeval start; - struct timeval end; - unsigned long timer; -#endif if (!(node = wasm_runtime_malloc(sizeof(NativeSymbolsNode)))) return false; @@ -349,23 +264,9 @@ register_natives(const char *module_name, NativeSymbol *native_symbols, node->next = g_native_symbols_list; g_native_symbols_list = node; -#if ENABLE_SORT_DEBUG != 0 - gettimeofday(&start, NULL); -#endif + qsort(native_symbols, n_native_symbols, sizeof(NativeSymbol), + native_symbol_cmp); -#if ENABLE_QUICKSORT == 0 - sort_symbol_ptr(native_symbols, n_native_symbols); -#else - quick_sort_symbols(native_symbols, 0, (int)(n_native_symbols - 1)); -#endif - -#if ENABLE_SORT_DEBUG != 0 - gettimeofday(&end, NULL); - timer = - 1000000 * (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec); - LOG_ERROR("module_name: %s, nums: %d, sorted used: %ld us", module_name, - n_native_symbols, timer); -#endif return true; } @@ -555,6 +456,11 @@ wasi_context_dtor(WASMModuleInstanceCommon *inst, void *ctx) } #endif /* end of WASM_ENABLE_LIBC_WASI */ +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +static bool +quick_aot_entry_init(); +#endif + bool wasm_native_init() { @@ -655,6 +561,20 @@ wasm_native_init() goto fail; #endif +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + if (!quick_aot_entry_init()) { +#if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ + || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ + || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0 \ + || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \ + || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + goto fail; +#else + return false; +#endif + } +#endif + return true; #if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ @@ -695,3 +615,936 @@ wasm_native_destroy() g_native_symbols_list = NULL; } + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +static void +invoke_no_args_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env); +} +static void +invoke_no_args_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env); +} +static void +invoke_no_args_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_i_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0]); +} +static void +invoke_i_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0]); +} +static void +invoke_i_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_I_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv)); +} +static void +invoke_I_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv)); +} +static void +invoke_I_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_ii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1]); +} +static void +invoke_ii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1]); +} +static void +invoke_ii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1)); +} +static void +invoke_iI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1)); +} +static void +invoke_iI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Ii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2]); +} +static void +invoke_Ii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2]); +} +static void +invoke_Ii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_II_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_II_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_II_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2]); +} +static void +invoke_iii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2]); +} +static void +invoke_iii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], argv[2]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_iiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_iiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3]); +} +static void +invoke_iIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3]); +} +static void +invoke_iIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Iii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3]); +} +static void +invoke_Iii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3]); +} +static void +invoke_Iii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_IiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_IiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + argv[4]); +} +static void +invoke_IIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4]); +} +static void +invoke_IIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_III_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_III_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_III_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); +} +static void +invoke_iiii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); +} +static void +invoke_iiii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iiiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iiiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR(argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + argv[4]); +} +static void +invoke_iiIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR(argv + 2), argv[4]); +} +static void +invoke_iiIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR(argv + 2), argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iiII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iiII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3], + argv[4]); +} +static void +invoke_iIii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], argv[4]); +} +static void +invoke_iIii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3], + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iIiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iIiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_iIIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_iIIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_iIII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_iIII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Iiii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3], argv[4]); +} +static void +invoke_Iiii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], argv[4]); +} +static void +invoke_Iiii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3], + GET_I64_FROM_ADDR(argv + 4)); +} + +static void +invoke_IiiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], GET_I64_FROM_ADDR(argv + 4)); +} + +static void +invoke_IiiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_IiIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_IiIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IiII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IiII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + argv[4], argv[5]); +} +static void +invoke_IIii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], argv[5]); +} +static void +invoke_IIii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + argv[4], GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IIiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], + GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IIiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], + GET_I64_FROM_ADDR(argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), argv[6]); +} +static void +invoke_IIIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), argv[6]); +} +static void +invoke_IIIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), argv[6]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), GET_I64_FROM_ADDR(argv + 6)); +} +static void +invoke_IIII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code( + exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), GET_I64_FROM_ADDR(argv + 6)); +} +static void +invoke_IIII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code( + exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), GET_I64_FROM_ADDR(argv + 6)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiiii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); +} +static void +invoke_iiiii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); +} +static void +invoke_iiiii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +typedef struct QuickAOTEntry { + const char *signature; + void *func_ptr; +} QuickAOTEntry; + +/* clang-format off */ +static QuickAOTEntry quick_aot_entries[] = { + { "()v", invoke_no_args_v }, + { "()i", invoke_no_args_i }, + { "()I", invoke_no_args_I }, + + { "(i)v", invoke_i_v }, { "(i)i", invoke_i_i }, { "(i)I", invoke_i_I }, + { "(I)v", invoke_I_v }, { "(I)i", invoke_I_i }, { "(I)I", invoke_I_I }, + + { "(ii)v", invoke_ii_v }, { "(ii)i", invoke_ii_i }, { "(ii)I", invoke_ii_I }, + { "(iI)v", invoke_iI_v }, { "(iI)i", invoke_iI_i }, { "(iI)I", invoke_iI_I }, + { "(Ii)v", invoke_Ii_v }, { "(Ii)i", invoke_Ii_i }, { "(Ii)I", invoke_Ii_I }, + { "(II)v", invoke_II_v }, { "(II)i", invoke_II_i }, { "(II)I", invoke_II_I }, + + { "(iii)v", invoke_iii_v }, { "(iii)i", invoke_iii_i }, { "(iii)I", invoke_iii_I }, + { "(iiI)v", invoke_iiI_v }, { "(iiI)i", invoke_iiI_i }, { "(iiI)I", invoke_iiI_I }, + { "(iIi)v", invoke_iIi_v }, { "(iIi)i", invoke_iIi_i }, { "(iIi)I", invoke_iIi_I }, + { "(iII)v", invoke_iII_v }, { "(iII)i", invoke_iII_i }, { "(iII)I", invoke_iII_I }, + { "(Iii)v", invoke_Iii_v }, { "(Iii)i", invoke_Iii_i }, { "(Iii)I", invoke_Iii_I }, + { "(IiI)v", invoke_IiI_v }, { "(IiI)i", invoke_IiI_i }, { "(IiI)I", invoke_IiI_I }, + { "(IIi)v", invoke_IIi_v }, { "(IIi)i", invoke_IIi_i }, { "(IIi)I", invoke_IIi_I }, + { "(III)v", invoke_III_v }, { "(III)i", invoke_III_i }, { "(III)I", invoke_III_I }, + + { "(iiii)v", invoke_iiii_v }, { "(iiii)i", invoke_iiii_i }, { "(iiii)I", invoke_iiii_I }, + { "(iiiI)v", invoke_iiiI_v }, { "(iiiI)i", invoke_iiiI_i }, { "(iiiI)I", invoke_iiiI_I }, + { "(iiIi)v", invoke_iiIi_v }, { "(iiIi)i", invoke_iiIi_i }, { "(iiIi)I", invoke_iiIi_I }, + { "(iiII)v", invoke_iiII_v }, { "(iiII)i", invoke_iiII_i }, { "(iiII)I", invoke_iiII_I }, + { "(iIii)v", invoke_iIii_v }, { "(iIii)i", invoke_iIii_i }, { "(iIii)I", invoke_iIii_I }, + { "(iIiI)v", invoke_iIiI_v }, { "(iIiI)i", invoke_iIiI_i }, { "(iIiI)I", invoke_iIiI_I }, + { "(iIIi)v", invoke_iIIi_v }, { "(iIIi)i", invoke_iIIi_i }, { "(iIIi)I", invoke_iIIi_I }, + { "(iIII)v", invoke_iIII_v }, { "(iIII)i", invoke_iIII_i }, { "(iIII)I", invoke_iIII_I }, + { "(Iiii)v", invoke_Iiii_v }, { "(Iiii)i", invoke_Iiii_i }, { "(Iiii)I", invoke_Iiii_I }, + { "(IiiI)v", invoke_IiiI_v }, { "(IiiI)i", invoke_IiiI_i }, { "(IiiI)I", invoke_IiiI_I }, + { "(IiIi)v", invoke_IiIi_v }, { "(IiIi)i", invoke_IiIi_i }, { "(IiIi)I", invoke_IiIi_I }, + { "(IiII)v", invoke_IiII_v }, { "(IiII)i", invoke_IiII_i }, { "(IiII)I", invoke_IiII_I }, + { "(IIii)v", invoke_IIii_v }, { "(IIii)i", invoke_IIii_i }, { "(IIii)I", invoke_IIii_I }, + { "(IIiI)v", invoke_IIiI_v }, { "(IIiI)i", invoke_IIiI_i }, { "(IIiI)I", invoke_IIiI_I }, + { "(IIIi)v", invoke_IIIi_v }, { "(IIIi)i", invoke_IIIi_i }, { "(IIIi)I", invoke_IIIi_I }, + { "(IIII)v", invoke_IIII_v }, { "(IIII)i", invoke_IIII_i }, { "(IIII)I", invoke_IIII_I }, + + { "(iiiii)v", invoke_iiiii_v }, { "(iiiii)i", invoke_iiiii_i }, { "(iiiii)I", invoke_iiiii_I }, +}; +/* clang-format on */ + +static int +quick_aot_entry_cmp(const void *quick_aot_entry1, const void *quick_aot_entry2) +{ + return strcmp(((const QuickAOTEntry *)quick_aot_entry1)->signature, + ((const QuickAOTEntry *)quick_aot_entry2)->signature); +} + +static bool +quick_aot_entry_init() +{ + qsort(quick_aot_entries, sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), + sizeof(QuickAOTEntry), quick_aot_entry_cmp); + + return true; +} + +void * +wasm_native_lookup_quick_aot_entry(const WASMType *func_type) +{ + char signature[16] = { 0 }; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count, i, j = 0; + const uint8 *types = func_type->types; + QuickAOTEntry *quick_aot_entry, key = { 0 }; + + if (param_count > 5 || result_count > 1) + return NULL; + + signature[j++] = '('; + + for (i = 0; i < param_count; i++) { + if (types[i] == VALUE_TYPE_I32) + signature[j++] = 'i'; + else if (types[i] == VALUE_TYPE_I64) + signature[j++] = 'I'; + else + return NULL; + } + + signature[j++] = ')'; + + if (result_count == 0) { + signature[j++] = 'v'; + } + else { + if (types[i] == VALUE_TYPE_I32) + signature[j++] = 'i'; + else if (types[i] == VALUE_TYPE_I64) + signature[j++] = 'I'; + else + return NULL; + } + + key.signature = signature; + if ((quick_aot_entry = + bsearch(&key, quick_aot_entries, + sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), + sizeof(QuickAOTEntry), quick_aot_entry_cmp))) { + return quick_aot_entry->func_ptr; + } + + return NULL; +} +#endif /* end of WASM_ENABLE_QUICK_AOT_ENTRY != 0 */ diff --git a/core/iwasm/common/wasm_native.h b/core/iwasm/common/wasm_native.h index 9ca5265c..591bbe2f 100644 --- a/core/iwasm/common/wasm_native.h +++ b/core/iwasm/common/wasm_native.h @@ -104,6 +104,11 @@ wasm_native_init(); void wasm_native_destroy(); +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +void * +wasm_native_lookup_quick_aot_entry(const WASMType *func_type); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index ee537aa6..518e7664 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -129,6 +129,10 @@ typedef struct WASMType { /* Code block to call llvm jit functions of this kind of function type from fast jit jitted code */ void *call_to_llvm_jit_from_fast_jit; +#endif +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick AOT/JIT entry of this func type */ + void *quick_aot_entry; #endif /* types of params and results */ uint8 types[1]; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index cc908fb2..937f8525 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -4199,12 +4199,29 @@ llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst, ret = true; } else { - ret = wasm_runtime_invoke_native( - exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, NULL, - argv, argc, argv); - - if (ret) +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick jit entry is registered */ + if (func_type->quick_aot_entry) { + void (*invoke_native)( + void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) = func_type->quick_aot_entry; + invoke_native(module_inst->func_ptrs[func_idx], + func_type->result_count > 0 + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID, + exec_env, argv, argv); ret = !wasm_copy_exception(module_inst, NULL); + } + else +#endif + { + ret = wasm_runtime_invoke_native( + exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, + NULL, argv, argc, argv); + + if (ret) + ret = !wasm_copy_exception(module_inst, NULL); + } } fail: diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 03530702..85f402a0 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -641,6 +641,10 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, type->param_cell_num = (uint16)param_cell_num; type->ret_cell_num = (uint16)ret_cell_num; +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + /* If there is already a same type created, use it instead */ for (j = 0; j < i; j++) { if (wasm_type_equal(type, module->types[j])) { diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 4bbec3d1..eb77e0c5 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -418,6 +418,10 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, type->param_cell_num = (uint16)param_cell_num; type->ret_cell_num = (uint16)ret_cell_num; +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + /* If there is already a same type created, use it instead */ for (j = 0; j < i; ++j) { if (wasm_type_equal(type, module->types[j])) {