diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index ca4935e3..212141d5 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -449,7 +449,18 @@ if (WAMR_CONFIGUABLE_BOUNDS_CHECKS EQUAL 1) endif () if (WAMR_BUILD_LINUX_PERF EQUAL 1) add_definitions (-DWASM_ENABLE_LINUX_PERF=1) - message (" Enable linux perf support") + message (" Linux perf support enabled") +endif () +if (NOT DEFINED WAMR_BUILD_QUICK_AOT_ENTRY) + # Enable quick aot/jit entries by default + set (WAMR_BUILD_QUICK_AOT_ENTRY 1) +endif () +if (WAMR_BUILD_QUICK_AOT_ENTRY EQUAL 1) + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=1) + message (" Quick AOT/JIT entries enabled") +else () + add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=0) + message (" Quick AOT/JIT entries disabled") endif () if (APPLE) diff --git a/core/config.h b/core/config.h index c30d0ace..e067ea27 100644 --- a/core/config.h +++ b/core/config.h @@ -495,4 +495,11 @@ #define WASM_ENABLE_LINUX_PERF 0 #endif +/* Support registering quick AOT/JIT function entries of some func types + to speedup the calling process of invoking the AOT/JIT functions of + these types from the host embedder */ +#ifndef WASM_ENABLE_QUICK_AOT_ENTRY +#define WASM_ENABLE_QUICK_AOT_ENTRY 1 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 341f66ae..d0d0c369 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1202,6 +1202,11 @@ load_func_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, func_types[i]->param_cell_num = (uint16)param_cell_num; func_types[i]->ret_cell_num = (uint16)ret_cell_num; + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + func_types[i]->quick_aot_entry = + wasm_native_lookup_quick_aot_entry(func_types[i]); +#endif } *p_buf = buf; diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index d806e064..ac83d5c2 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1385,9 +1385,6 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; uint32 page_size = os_getpagesize(); uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - uint16 param_count = func_type->param_count; - uint16 result_count = func_type->result_count; - const uint8 *types = func_type->types; #ifdef BH_PLATFORM_WINDOWS int result; bool has_exception; @@ -1426,28 +1423,22 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { - /* Quick call with func_ptr if the function signature is simple */ - if (!signature && param_count == 1 && types[0] == VALUE_TYPE_I32) { - if (result_count == 0) { - void (*NativeFunc)(WASMExecEnv *, uint32) = - (void (*)(WASMExecEnv *, uint32))func_ptr; - NativeFunc(exec_env, argv[0]); - ret = aot_copy_exception(module_inst, NULL) ? false : true; - } - else if (result_count == 1 - && types[param_count] == VALUE_TYPE_I32) { - uint32 (*NativeFunc)(WASMExecEnv *, uint32) = - (uint32(*)(WASMExecEnv *, uint32))func_ptr; - argv_ret[0] = NativeFunc(exec_env, argv[0]); - ret = aot_copy_exception(module_inst, NULL) ? false : true; - } - else { - ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type, - signature, attachment, argv, - argc, argv_ret); - } +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick aot entry is registered */ + if (!signature && func_type->quick_aot_entry) { + void (*invoke_native)( + void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) = func_type->quick_aot_entry; + invoke_native(func_ptr, + func_type->result_count > 0 + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID, + exec_env, argv, argv_ret); + ret = !aot_copy_exception(module_inst, NULL); } - else { + else +#endif + { ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, attachment, argv, argc, argv_ret); diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index c42bf7b3..69ae6f2a 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -16,19 +16,6 @@ #include "../libraries/thread-mgr/thread_manager.h" #endif -#if !defined(BH_PLATFORM_ZEPHYR) && !defined(BH_PLATFORM_ALIOS_THINGS) \ - && !defined(BH_PLATFORM_OPENRTOS) && !defined(BH_PLATFORM_ESP_IDF) -#define ENABLE_QUICKSORT 1 -#else -#define ENABLE_QUICKSORT 0 -#endif - -#define ENABLE_SORT_DEBUG 0 - -#if ENABLE_SORT_DEBUG != 0 -#include -#endif - static NativeSymbolsList g_native_symbols_list = NULL; #if WASM_ENABLE_LIBC_WASI != 0 @@ -171,93 +158,26 @@ check_symbol_signature(const WASMType *type, const char *signature) return true; } -#if ENABLE_QUICKSORT == 0 -static void -sort_symbol_ptr(NativeSymbol *native_symbols, uint32 n_native_symbols) +static int +native_symbol_cmp(const void *native_symbol1, const void *native_symbol2) { - uint32 i, j; - NativeSymbol temp; - - for (i = 0; i < n_native_symbols - 1; i++) { - for (j = i + 1; j < n_native_symbols; j++) { - if (strcmp(native_symbols[i].symbol, native_symbols[j].symbol) - > 0) { - temp = native_symbols[i]; - native_symbols[i] = native_symbols[j]; - native_symbols[j] = temp; - } - } - } + return strcmp(((const NativeSymbol *)native_symbol1)->symbol, + ((const NativeSymbol *)native_symbol2)->symbol); } -#else -static void -swap_symbol(NativeSymbol *left, NativeSymbol *right) -{ - NativeSymbol temp = *left; - *left = *right; - *right = temp; -} - -static void -quick_sort_symbols(NativeSymbol *native_symbols, int left, int right) -{ - NativeSymbol base_symbol; - int pin_left = left; - int pin_right = right; - - if (left >= right) { - return; - } - - base_symbol = native_symbols[left]; - while (left < right) { - while (left < right - && strcmp(native_symbols[right].symbol, base_symbol.symbol) - > 0) { - right--; - } - - if (left < right) { - swap_symbol(&native_symbols[left], &native_symbols[right]); - left++; - } - - while (left < right - && strcmp(native_symbols[left].symbol, base_symbol.symbol) < 0) { - left++; - } - - if (left < right) { - swap_symbol(&native_symbols[left], &native_symbols[right]); - right--; - } - } - native_symbols[left] = base_symbol; - - quick_sort_symbols(native_symbols, pin_left, left - 1); - quick_sort_symbols(native_symbols, left + 1, pin_right); -} -#endif /* end of ENABLE_QUICKSORT */ static void * lookup_symbol(NativeSymbol *native_symbols, uint32 n_native_symbols, const char *symbol, const char **p_signature, void **p_attachment) { - int low = 0, mid, ret; - int high = (int32)n_native_symbols - 1; + NativeSymbol *native_symbol, key = { 0 }; - while (low <= high) { - mid = (low + high) / 2; - ret = strcmp(symbol, native_symbols[mid].symbol); - if (ret == 0) { - *p_signature = native_symbols[mid].signature; - *p_attachment = native_symbols[mid].attachment; - return native_symbols[mid].func_ptr; - } - else if (ret < 0) - high = mid - 1; - else - low = mid + 1; + key.symbol = symbol; + + if ((native_symbol = bsearch(&key, native_symbols, n_native_symbols, + sizeof(NativeSymbol), native_symbol_cmp))) { + *p_signature = native_symbol->signature; + *p_attachment = native_symbol->attachment; + return native_symbol->func_ptr; } return NULL; @@ -328,11 +248,6 @@ register_natives(const char *module_name, NativeSymbol *native_symbols, uint32 n_native_symbols, bool call_conv_raw) { NativeSymbolsNode *node; -#if ENABLE_SORT_DEBUG != 0 - struct timeval start; - struct timeval end; - unsigned long timer; -#endif if (!(node = wasm_runtime_malloc(sizeof(NativeSymbolsNode)))) return false; @@ -349,23 +264,9 @@ register_natives(const char *module_name, NativeSymbol *native_symbols, node->next = g_native_symbols_list; g_native_symbols_list = node; -#if ENABLE_SORT_DEBUG != 0 - gettimeofday(&start, NULL); -#endif + qsort(native_symbols, n_native_symbols, sizeof(NativeSymbol), + native_symbol_cmp); -#if ENABLE_QUICKSORT == 0 - sort_symbol_ptr(native_symbols, n_native_symbols); -#else - quick_sort_symbols(native_symbols, 0, (int)(n_native_symbols - 1)); -#endif - -#if ENABLE_SORT_DEBUG != 0 - gettimeofday(&end, NULL); - timer = - 1000000 * (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec); - LOG_ERROR("module_name: %s, nums: %d, sorted used: %ld us", module_name, - n_native_symbols, timer); -#endif return true; } @@ -555,6 +456,11 @@ wasi_context_dtor(WASMModuleInstanceCommon *inst, void *ctx) } #endif /* end of WASM_ENABLE_LIBC_WASI */ +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +static bool +quick_aot_entry_init(); +#endif + bool wasm_native_init() { @@ -655,6 +561,20 @@ wasm_native_init() goto fail; #endif +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + if (!quick_aot_entry_init()) { +#if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ + || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ + || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0 \ + || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \ + || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0 + goto fail; +#else + return false; +#endif + } +#endif + return true; #if WASM_ENABLE_SPEC_TEST != 0 || WASM_ENABLE_LIBC_BUILTIN != 0 \ || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0 \ @@ -695,3 +615,936 @@ wasm_native_destroy() g_native_symbols_list = NULL; } + +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +static void +invoke_no_args_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env); +} +static void +invoke_no_args_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env); +} +static void +invoke_no_args_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_i_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0]); +} +static void +invoke_i_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0]); +} +static void +invoke_i_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_I_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv)); +} +static void +invoke_I_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv)); +} +static void +invoke_I_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_ii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1]); +} +static void +invoke_ii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1]); +} +static void +invoke_ii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1)); +} +static void +invoke_iI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1)); +} +static void +invoke_iI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Ii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2]); +} +static void +invoke_Ii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2]); +} +static void +invoke_Ii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_II_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_II_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_II_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2]); +} +static void +invoke_iii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2]); +} +static void +invoke_iii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], argv[2]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_iiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2)); +} +static void +invoke_iiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3]); +} +static void +invoke_iIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3]); +} +static void +invoke_iIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Iii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3]); +} +static void +invoke_Iii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3]); +} +static void +invoke_Iii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_IiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_IiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + argv[4]); +} +static void +invoke_IIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4]); +} +static void +invoke_IIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_III_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_III_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_III_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); +} +static void +invoke_iiii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); +} +static void +invoke_iiii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], argv[2], argv[3]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iiiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR(argv + 3)); +} +static void +invoke_iiiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], argv[2], + GET_I64_FROM_ADDR(argv + 3)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + argv[4]); +} +static void +invoke_iiIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR(argv + 2), argv[4]); +} +static void +invoke_iiIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], argv[1], + GET_I64_FROM_ADDR(argv + 2), argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iiII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iiII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], argv[1], GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3], + argv[4]); +} +static void +invoke_iIii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], argv[4]); +} +static void +invoke_iIii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), argv[3], + GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iIiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], GET_I64_FROM_ADDR(argv + 4)); +} +static void +invoke_iIiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + argv[3], GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_iIIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_iIIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iIII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_iIII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_iIII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], GET_I64_FROM_ADDR(argv + 1), + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_Iiii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3], argv[4]); +} +static void +invoke_Iiii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], argv[4]); +} +static void +invoke_Iiii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], argv[3], + GET_I64_FROM_ADDR(argv + 4)); +} + +static void +invoke_IiiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], GET_I64_FROM_ADDR(argv + 4)); +} + +static void +invoke_IiiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + argv[3], GET_I64_FROM_ADDR(argv + 4)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_IiIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), argv[5]); +} +static void +invoke_IiIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IiII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IiII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IiII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, GET_I64_FROM_ADDR(argv), argv[2], + GET_I64_FROM_ADDR(argv + 3), GET_I64_FROM_ADDR(argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + argv[4], argv[5]); +} +static void +invoke_IIii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], argv[5]); +} +static void +invoke_IIii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], argv[5]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIiI_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + argv[4], GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IIiI_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], + GET_I64_FROM_ADDR(argv + 5)); +} +static void +invoke_IIiI_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), argv[4], + GET_I64_FROM_ADDR(argv + 5)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIIi_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), argv[6]); +} +static void +invoke_IIIi_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), argv[6]); +} +static void +invoke_IIIi_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code(exec_env, GET_I64_FROM_ADDR(argv), + GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), argv[6]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_IIII_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), GET_I64_FROM_ADDR(argv + 6)); +} +static void +invoke_IIII_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = native_code( + exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), GET_I64_FROM_ADDR(argv + 6)); +} +static void +invoke_IIII_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = native_code( + exec_env, GET_I64_FROM_ADDR(argv), GET_I64_FROM_ADDR(argv + 2), + GET_I64_FROM_ADDR(argv + 4), GET_I64_FROM_ADDR(argv + 6)); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +static void +invoke_iiiii_v(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + void (*native_code)() = func_ptr; + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); +} +static void +invoke_iiiii_i(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint32 (*native_code)() = func_ptr; + argv_ret[0] = + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); +} +static void +invoke_iiiii_I(void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) +{ + uint64 (*native_code)() = func_ptr; + uint64 ret = + native_code(exec_env, argv[0], argv[1], argv[2], argv[3], argv[4]); + PUT_I64_TO_ADDR(argv_ret, ret); +} + +typedef struct QuickAOTEntry { + const char *signature; + void *func_ptr; +} QuickAOTEntry; + +/* clang-format off */ +static QuickAOTEntry quick_aot_entries[] = { + { "()v", invoke_no_args_v }, + { "()i", invoke_no_args_i }, + { "()I", invoke_no_args_I }, + + { "(i)v", invoke_i_v }, { "(i)i", invoke_i_i }, { "(i)I", invoke_i_I }, + { "(I)v", invoke_I_v }, { "(I)i", invoke_I_i }, { "(I)I", invoke_I_I }, + + { "(ii)v", invoke_ii_v }, { "(ii)i", invoke_ii_i }, { "(ii)I", invoke_ii_I }, + { "(iI)v", invoke_iI_v }, { "(iI)i", invoke_iI_i }, { "(iI)I", invoke_iI_I }, + { "(Ii)v", invoke_Ii_v }, { "(Ii)i", invoke_Ii_i }, { "(Ii)I", invoke_Ii_I }, + { "(II)v", invoke_II_v }, { "(II)i", invoke_II_i }, { "(II)I", invoke_II_I }, + + { "(iii)v", invoke_iii_v }, { "(iii)i", invoke_iii_i }, { "(iii)I", invoke_iii_I }, + { "(iiI)v", invoke_iiI_v }, { "(iiI)i", invoke_iiI_i }, { "(iiI)I", invoke_iiI_I }, + { "(iIi)v", invoke_iIi_v }, { "(iIi)i", invoke_iIi_i }, { "(iIi)I", invoke_iIi_I }, + { "(iII)v", invoke_iII_v }, { "(iII)i", invoke_iII_i }, { "(iII)I", invoke_iII_I }, + { "(Iii)v", invoke_Iii_v }, { "(Iii)i", invoke_Iii_i }, { "(Iii)I", invoke_Iii_I }, + { "(IiI)v", invoke_IiI_v }, { "(IiI)i", invoke_IiI_i }, { "(IiI)I", invoke_IiI_I }, + { "(IIi)v", invoke_IIi_v }, { "(IIi)i", invoke_IIi_i }, { "(IIi)I", invoke_IIi_I }, + { "(III)v", invoke_III_v }, { "(III)i", invoke_III_i }, { "(III)I", invoke_III_I }, + + { "(iiii)v", invoke_iiii_v }, { "(iiii)i", invoke_iiii_i }, { "(iiii)I", invoke_iiii_I }, + { "(iiiI)v", invoke_iiiI_v }, { "(iiiI)i", invoke_iiiI_i }, { "(iiiI)I", invoke_iiiI_I }, + { "(iiIi)v", invoke_iiIi_v }, { "(iiIi)i", invoke_iiIi_i }, { "(iiIi)I", invoke_iiIi_I }, + { "(iiII)v", invoke_iiII_v }, { "(iiII)i", invoke_iiII_i }, { "(iiII)I", invoke_iiII_I }, + { "(iIii)v", invoke_iIii_v }, { "(iIii)i", invoke_iIii_i }, { "(iIii)I", invoke_iIii_I }, + { "(iIiI)v", invoke_iIiI_v }, { "(iIiI)i", invoke_iIiI_i }, { "(iIiI)I", invoke_iIiI_I }, + { "(iIIi)v", invoke_iIIi_v }, { "(iIIi)i", invoke_iIIi_i }, { "(iIIi)I", invoke_iIIi_I }, + { "(iIII)v", invoke_iIII_v }, { "(iIII)i", invoke_iIII_i }, { "(iIII)I", invoke_iIII_I }, + { "(Iiii)v", invoke_Iiii_v }, { "(Iiii)i", invoke_Iiii_i }, { "(Iiii)I", invoke_Iiii_I }, + { "(IiiI)v", invoke_IiiI_v }, { "(IiiI)i", invoke_IiiI_i }, { "(IiiI)I", invoke_IiiI_I }, + { "(IiIi)v", invoke_IiIi_v }, { "(IiIi)i", invoke_IiIi_i }, { "(IiIi)I", invoke_IiIi_I }, + { "(IiII)v", invoke_IiII_v }, { "(IiII)i", invoke_IiII_i }, { "(IiII)I", invoke_IiII_I }, + { "(IIii)v", invoke_IIii_v }, { "(IIii)i", invoke_IIii_i }, { "(IIii)I", invoke_IIii_I }, + { "(IIiI)v", invoke_IIiI_v }, { "(IIiI)i", invoke_IIiI_i }, { "(IIiI)I", invoke_IIiI_I }, + { "(IIIi)v", invoke_IIIi_v }, { "(IIIi)i", invoke_IIIi_i }, { "(IIIi)I", invoke_IIIi_I }, + { "(IIII)v", invoke_IIII_v }, { "(IIII)i", invoke_IIII_i }, { "(IIII)I", invoke_IIII_I }, + + { "(iiiii)v", invoke_iiiii_v }, { "(iiiii)i", invoke_iiiii_i }, { "(iiiii)I", invoke_iiiii_I }, +}; +/* clang-format on */ + +static int +quick_aot_entry_cmp(const void *quick_aot_entry1, const void *quick_aot_entry2) +{ + return strcmp(((const QuickAOTEntry *)quick_aot_entry1)->signature, + ((const QuickAOTEntry *)quick_aot_entry2)->signature); +} + +static bool +quick_aot_entry_init() +{ + qsort(quick_aot_entries, sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), + sizeof(QuickAOTEntry), quick_aot_entry_cmp); + + return true; +} + +void * +wasm_native_lookup_quick_aot_entry(const WASMType *func_type) +{ + char signature[16] = { 0 }; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count, i, j = 0; + const uint8 *types = func_type->types; + QuickAOTEntry *quick_aot_entry, key = { 0 }; + + if (param_count > 5 || result_count > 1) + return NULL; + + signature[j++] = '('; + + for (i = 0; i < param_count; i++) { + if (types[i] == VALUE_TYPE_I32) + signature[j++] = 'i'; + else if (types[i] == VALUE_TYPE_I64) + signature[j++] = 'I'; + else + return NULL; + } + + signature[j++] = ')'; + + if (result_count == 0) { + signature[j++] = 'v'; + } + else { + if (types[i] == VALUE_TYPE_I32) + signature[j++] = 'i'; + else if (types[i] == VALUE_TYPE_I64) + signature[j++] = 'I'; + else + return NULL; + } + + key.signature = signature; + if ((quick_aot_entry = + bsearch(&key, quick_aot_entries, + sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), + sizeof(QuickAOTEntry), quick_aot_entry_cmp))) { + return quick_aot_entry->func_ptr; + } + + return NULL; +} +#endif /* end of WASM_ENABLE_QUICK_AOT_ENTRY != 0 */ diff --git a/core/iwasm/common/wasm_native.h b/core/iwasm/common/wasm_native.h index 9ca5265c..591bbe2f 100644 --- a/core/iwasm/common/wasm_native.h +++ b/core/iwasm/common/wasm_native.h @@ -104,6 +104,11 @@ wasm_native_init(); void wasm_native_destroy(); +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 +void * +wasm_native_lookup_quick_aot_entry(const WASMType *func_type); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index ee537aa6..518e7664 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -129,6 +129,10 @@ typedef struct WASMType { /* Code block to call llvm jit functions of this kind of function type from fast jit jitted code */ void *call_to_llvm_jit_from_fast_jit; +#endif +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick AOT/JIT entry of this func type */ + void *quick_aot_entry; #endif /* types of params and results */ uint8 types[1]; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index cc908fb2..937f8525 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -4199,12 +4199,29 @@ llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst, ret = true; } else { - ret = wasm_runtime_invoke_native( - exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, NULL, - argv, argc, argv); - - if (ret) +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + /* Quick call if the quick jit entry is registered */ + if (func_type->quick_aot_entry) { + void (*invoke_native)( + void *func_ptr, uint8 ret_type, void *exec_env, uint32 *argv, + uint32 *argv_ret) = func_type->quick_aot_entry; + invoke_native(module_inst->func_ptrs[func_idx], + func_type->result_count > 0 + ? func_type->types[func_type->param_count] + : VALUE_TYPE_VOID, + exec_env, argv, argv); ret = !wasm_copy_exception(module_inst, NULL); + } + else +#endif + { + ret = wasm_runtime_invoke_native( + exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, + NULL, argv, argc, argv); + + if (ret) + ret = !wasm_copy_exception(module_inst, NULL); + } } fail: diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 03530702..85f402a0 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -641,6 +641,10 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, type->param_cell_num = (uint16)param_cell_num; type->ret_cell_num = (uint16)ret_cell_num; +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + /* If there is already a same type created, use it instead */ for (j = 0; j < i; j++) { if (wasm_type_equal(type, module->types[j])) { diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 4bbec3d1..eb77e0c5 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -418,6 +418,10 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, type->param_cell_num = (uint16)param_cell_num; type->ret_cell_num = (uint16)ret_cell_num; +#if WASM_ENABLE_QUICK_AOT_ENTRY != 0 + type->quick_aot_entry = wasm_native_lookup_quick_aot_entry(type); +#endif + /* If there is already a same type created, use it instead */ for (j = 0; j < i; ++j) { if (wasm_type_equal(type, module->types[j])) {