christoph/wamr
1
Fork 0
Code Releases Activity

Implement Multi-tier JIT (#1774)

Browse Source 
Implement 2-level Multi-tier JIT engine: tier-up from Fast JIT to LLVM JIT to
get quick cold startup by Fast JIT and better performance by gradually
switching to LLVM JIT when the LLVM JIT functions are compiled by the
backend threads.

Refer to:
https://github.com/bytecodealliance/wasm-micro-runtime/issues/1302
This commit is contained in:
Wenyong Huang
2022-12-19 11:24:46 +08:00
parent 7db49db777
commit e8ce4c542e
21 changed files with 2180 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
core/iwasm/interpreter/wasm.h
View File

@ -124,6 +124,12 @@ typedef struct WASMType {
uint16 param_cell_num;
uint16 ret_cell_num;
uint16 ref_count;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* Code block to call llvm jit functions of this
kind of function type from fast jit jitted code */
void *call_to_llvm_jit_from_fast_jit;
#endif
/* types of params and results */
uint8 types[1];
} WASMType;
@ -255,13 +261,6 @@ struct WASMFunction {
uint32 const_cell_num;
#endif
#if WASM_ENABLE_FAST_JIT != 0
void *fast_jit_jitted_code;
#endif
#if WASM_ENABLE_JIT != 0
void *llvm_jit_func_ptr;
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
|| WASM_ENABLE_WAMR_COMPILER != 0
/* Whether function has opcode memory.grow */
@ -277,6 +276,13 @@ struct WASMFunction {
/* Whether function has opcode set_global_aux_stack */
bool has_op_set_global_aux_stack;
#endif
#if WASM_ENABLE_FAST_JIT != 0
void *fast_jit_jitted_code;
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
void *llvm_jit_func_ptr;
#endif
#endif
};
struct WASMGlobal {
@ -377,18 +383,22 @@ typedef struct WASMCustomSection {
} WASMCustomSection;
#endif
#if WASM_ENABLE_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
struct AOTCompData;
struct AOTCompContext;
/* Orc JIT thread arguments */
typedef struct OrcJitThreadArg {
#if WASM_ENABLE_JIT != 0
struct AOTCompContext *comp_ctx;
#endif
struct WASMModule *module;
uint32 group_idx;
} OrcJitThreadArg;
#endif
struct WASMModuleInstance;
struct WASMModule {
/* Module type, for module loaded from WASM bytecode binary,
this field is Wasm_Module_Bytecode;
@ -495,18 +505,22 @@ struct WASMModule {
uint64 load_size;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
&& WASM_ENABLE_LAZY_JIT != 0)
/**
* Count how many instances reference this module. When source
* debugging feature enabled, the debugger may modify the code
* section of the module, so we need to report a warning if user
* create several instances based on the same module
* List of instances referred to this module. When source debugging
* feature is enabled, the debugger may modify the code section of
* the module, so we need to report a warning if user create several
* instances based on the same module. Sub instances created by
* lib-pthread or spawn API won't be added into the list.
*
* Sub_instances created by lib-pthread or spawn API will not
* influence or check the ref count
* Also add the instance to the list for Fast JIT to LLVM JIT
* tier-up, since we need to lazily update the LLVM func pointers
* in the instance.
*/
uint32 ref_count;
korp_mutex ref_count_lock;
struct WASMModuleInstance *instance_list;
korp_mutex instance_list_lock;
#endif
#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
@ -521,6 +535,9 @@ struct WASMModule {
#if WASM_ENABLE_FAST_JIT != 0
/* func pointers of Fast JITed (un-imported) functions */
void **fast_jit_func_ptrs;
/* locks for Fast JIT lazy compilation */
korp_mutex fast_jit_thread_locks[WASM_ORC_JIT_BACKEND_THREAD_NUM];
bool fast_jit_thread_locks_inited[WASM_ORC_JIT_BACKEND_THREAD_NUM];
#endif
#if WASM_ENABLE_JIT != 0
@ -530,9 +547,27 @@ struct WASMModule {
void **func_ptrs;
/* whether the func pointers are compiled */
bool *func_ptrs_compiled;
bool orcjit_stop_compiling;
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* backend compilation threads */
korp_tid orcjit_threads[WASM_ORC_JIT_BACKEND_THREAD_NUM];
/* backend thread arguments */
OrcJitThreadArg orcjit_thread_args[WASM_ORC_JIT_BACKEND_THREAD_NUM];
/* whether to stop the compilation of backend threads */
bool orcjit_stop_compiling;
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* wait lock/cond for the synchronization of
the llvm jit initialization */
korp_mutex tierup_wait_lock;
korp_cond tierup_wait_cond;
bool tierup_wait_lock_inited;
korp_tid llvm_jit_init_thread;
/* whether the llvm jit is initialized */
bool llvm_jit_inited;
#endif
};

93
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -3947,28 +3947,48 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_FAST_JIT != 0
static void
fast_jit_call_func_bytecode(WASMExecEnv *exec_env,
fast_jit_call_func_bytecode(WASMModuleInstance *module_inst,
WASMExecEnv *exec_env,
WASMFunctionInstance *function,
WASMInterpFrame *frame)
{
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
JitInterpSwitchInfo info;
WASMModule *module = module_inst->module;
WASMType *func_type = function->u.func->func_type;
uint8 type = func_type->result_count
? func_type->types[func_type->param_count]
: VALUE_TYPE_VOID;
uint32 func_idx = (uint32)(function - module_inst->e->functions);
uint32 func_idx_non_import = func_idx - module->import_function_count;
int32 action;
#if WASM_ENABLE_REF_TYPES != 0
if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
type = VALUE_TYPE_I32;
#endif
#if WASM_ENABLE_LAZY_JIT != 0
if (!jit_compiler_compile(module, func_idx)) {
wasm_set_exception(module_inst, "failed to compile fast jit function");
return;
}
#endif
bh_assert(jit_compiler_is_compiled(module, func_idx));
/* Switch to jitted code to call the jit function */
info.out.ret.last_return_type = type;
info.frame = frame;
frame->jitted_return_addr =
(uint8 *)jit_globals->return_to_interp_from_jitted;
jit_interp_switch_to_jitted(exec_env, &info,
function->u.func->fast_jit_jitted_code);
action = jit_interp_switch_to_jitted(
exec_env, &info, func_idx,
module_inst->fast_jit_func_ptrs[func_idx_non_import]);
bh_assert(action == JIT_INTERP_ACTION_NORMAL
|| (action == JIT_INTERP_ACTION_THROWN
&& wasm_runtime_get_exception(exec_env->module_inst)));
/* Get the return values form info.out.ret */
if (func_type->result_count) {
switch (type) {
case VALUE_TYPE_I32:
@ -3992,8 +4012,10 @@ fast_jit_call_func_bytecode(WASMExecEnv *exec_env,
break;
}
}
(void)action;
(void)func_idx;
}
#endif
#endif /* end of WASM_ENABLE_FAST_JIT != 0 */
#if WASM_ENABLE_JIT != 0
static bool
@ -4023,11 +4045,13 @@ llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
WASMType *func_type = function->u.func->func_type;
uint32 result_count = func_type->result_count;
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
uint32 func_idx = (uint32)(function - module_inst->e->functions);
bool ret;
#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) {
wasm_set_exception(module_inst, "wasm operand stack overflow");
/* wasm operand stack overflow has been thrown,
no need to throw again */
return false;
}
#endif
@ -4069,8 +4093,8 @@ llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
}
ret = wasm_runtime_invoke_native(
exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL,
NULL, argv1, argc, argv);
exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, NULL,
argv1, argc, argv);
if (!ret || wasm_get_exception(module_inst)) {
if (clear_wasi_proc_exit_exception(module_inst))
@ -4120,8 +4144,8 @@ llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
}
else {
ret = wasm_runtime_invoke_native(
exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL,
NULL, argv, argc, argv);
exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, NULL,
argv, argc, argv);
if (clear_wasi_proc_exit_exception(module_inst))
ret = true;
@ -4129,7 +4153,7 @@ llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
return ret && !wasm_get_exception(module_inst) ? true : false;
}
}
#endif
#endif /* end of WASM_ENABLE_JIT != 0 */
void
wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
@ -4202,18 +4226,63 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
}
else {
#if WASM_ENABLE_JIT != 0
#if WASM_ENABLE_LAZY_JIT != 0
/* Fast JIT to LLVM JIT tier-up is enabled */
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0
/* Fast JIT and LLVM JIT are both enabled, call llvm jit function
if it is compiled, else call fast jit function */
uint32 func_idx = (uint32)(function - module_inst->e->functions);
if (module_inst->module->func_ptrs_compiled
[func_idx - module_inst->module->import_function_count]) {
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
}
else {
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
}
#elif WASM_ENABLE_JIT != 0
/* Only LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
fast_jit_call_func_bytecode(exec_env, function, frame);
/* Only Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#else
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
#else /* else of WASM_ENABLE_LAZY_JIT != 0 */
/* Fast JIT to LLVM JIT tier-up is enabled */
#if WASM_ENABLE_JIT != 0
/* LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
/* Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#else
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
(void)wasm_interp_call_func_bytecode;
#if WASM_ENABLE_FAST_JIT != 0
(void)fast_jit_call_func_bytecode;
#endif
}
/* Output the return value to the caller */

514
core/iwasm/interpreter/wasm_loader.c
View File

@ -428,6 +428,12 @@ destroy_wasm_type(WASMType *type)
return;
}
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
if (type->call_to_llvm_jit_from_fast_jit)
jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit);
#endif
wasm_runtime_free(type);
}
@ -2925,93 +2931,78 @@ calculate_global_data_offset(WASMModule *module)
module->global_data_size = data_offset;
}
#if WASM_ENABLE_JIT != 0
static void *
orcjit_thread_callback(void *arg)
{
LLVMOrcJITTargetAddress func_addr = 0;
OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg;
AOTCompContext *comp_ctx = thread_arg->comp_ctx;
WASMModule *module = thread_arg->module;
uint32 group_idx = thread_arg->group_idx;
uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
uint32 func_count = module->function_count;
uint32 i, j;
typedef void (*F)(void);
LLVMErrorRef error;
char func_name[48];
union {
F f;
void *v;
} u;
/* Compile jit functions of this group */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i,
"_wrapper");
LOG_DEBUG("compile func %s", func_name);
error =
LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile orc jit function: %s\n", err_msg);
LLVMDisposeErrorMessage(err_msg);
continue;
}
/* Call the jit wrapper function to trigger its compilation, so as
to compile the actual jit functions, since we add the latter to
function list in the PartitionFunction callback */
u.v = (void *)func_addr;
u.f();
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count)
module->func_ptrs_compiled[i + j * group_stride] = true;
}
if (module->orcjit_stop_compiling) {
break;
}
}
return NULL;
}
static void
orcjit_stop_compile_threads(WASMModule *module)
{
uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args)
/ sizeof(OrcJitThreadArg));
module->orcjit_stop_compiling = true;
for (i = 0; i < thread_num; i++) {
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
}
#if WASM_ENABLE_FAST_JIT != 0
static bool
compile_llvm_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
init_fast_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
#if WASM_ENABLE_LAZY_JIT != 0
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
#endif
uint32 i;
if (!module->function_count)
return true;
if (!(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count, error_buf,
error_buf_size))) {
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
for (i = 0; i < module->function_count; i++) {
module->fast_jit_func_ptrs[i] =
jit_globals->compile_fast_jit_and_then_call;
}
#endif
for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) {
if (os_mutex_init(&module->fast_jit_thread_locks[i]) != 0) {
set_error_buf(error_buf, error_buf_size,
"init fast jit thread lock failed");
return false;
}
module->fast_jit_thread_locks_inited[i] = true;
}
return true;
}
#endif /* end of WASM_ENABLE_FAST_JIT != 0 */
#if WASM_ENABLE_JIT != 0
static bool
init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
AOTCompOption option = { 0 };
char *aot_last_error;
uint64 size;
uint32 thread_num, i;
if (module->function_count > 0) {
size = sizeof(void *) * (uint64)module->function_count
+ sizeof(bool) * (uint64)module->function_count;
if (!(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->func_ptrs_compiled =
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
if (module->function_count == 0)
return true;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LLVM_JIT != 0
if (os_mutex_init(&module->tierup_wait_lock) != 0) {
set_error_buf(error_buf, error_buf_size, "init jit tierup lock failed");
return false;
}
if (os_cond_init(&module->tierup_wait_cond) != 0) {
set_error_buf(error_buf, error_buf_size, "init jit tierup cond failed");
os_mutex_destroy(&module->tierup_wait_lock);
return false;
}
module->tierup_wait_lock_inited = true;
#endif
size = sizeof(void *) * (uint64)module->function_count
+ sizeof(bool) * (uint64)module->function_count;
if (!(module->func_ptrs = loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->func_ptrs_compiled =
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
module->comp_data = aot_create_comp_data(module);
if (!module->comp_data) {
@ -3052,6 +3043,19 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
return false;
}
return true;
}
static bool
init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
char *aot_last_error;
uint32 i;
if (module->function_count == 0)
return true;
if (!aot_compile_wasm(module->comp_ctx)) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
@ -3059,7 +3063,12 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
return false;
}
bh_print_time("Begin to lookup jit functions");
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->orcjit_stop_compiling)
return false;
#endif
bh_print_time("Begin to lookup llvm jit functions");
for (i = 0; i < module->function_count; i++) {
LLVMOrcJITTargetAddress func_addr = 0;
@ -3072,7 +3081,7 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
set_error_buf_v(error_buf, error_buf_size,
"failed to compile orc jit function: %s", err_msg);
"failed to compile llvm jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
return false;
}
@ -3084,17 +3093,211 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
* loading/storing at the same time.
*/
module->func_ptrs[i] = (void *)func_addr;
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->orcjit_stop_compiling)
return false;
#endif
}
bh_print_time("End lookup llvm jit functions");
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
static void *
init_llvm_jit_functions_stage2_callback(void *arg)
{
WASMModule *module = (WASMModule *)arg;
char error_buf[128];
uint32 error_buf_size = (uint32)sizeof(error_buf);
if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) {
module->orcjit_stop_compiling = true;
return NULL;
}
os_mutex_lock(&module->tierup_wait_lock);
module->llvm_jit_inited = true;
os_cond_broadcast(&module->tierup_wait_cond);
os_mutex_unlock(&module->tierup_wait_lock);
return NULL;
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* The callback function to compile jit functions */
static void *
orcjit_thread_callback(void *arg)
{
OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg;
#if WASM_ENABLE_JIT != 0
AOTCompContext *comp_ctx = thread_arg->comp_ctx;
#endif
WASMModule *module = thread_arg->module;
uint32 group_idx = thread_arg->group_idx;
uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
uint32 func_count = module->function_count;
uint32 i;
#if WASM_ENABLE_FAST_JIT != 0
/* Compile fast jit funcitons of this group */
for (i = group_idx; i < func_count; i += group_stride) {
if (!jit_compiler_compile(module, i + module->import_function_count)) {
os_printf("failed to compile fast jit function %u\n", i);
break;
}
if (module->orcjit_stop_compiling) {
return NULL;
}
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* For JIT tier-up, set each llvm jit func to call_to_fast_jit */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
uint32 j;
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count) {
if (!jit_compiler_set_call_to_fast_jit(
module,
i + j * group_stride + module->import_function_count)) {
os_printf(
"failed to compile call_to_fast_jit for func %u\n",
i + j * group_stride + module->import_function_count);
module->orcjit_stop_compiling = true;
return NULL;
}
}
if (module->orcjit_stop_compiling) {
return NULL;
}
}
}
/* Wait until init_llvm_jit_functions_stage2 finishes */
os_mutex_lock(&module->tierup_wait_lock);
while (!module->llvm_jit_inited) {
os_cond_reltimedwait(&module->tierup_wait_cond,
&module->tierup_wait_lock, 10);
if (module->orcjit_stop_compiling) {
/* init_llvm_jit_functions_stage2 failed */
os_mutex_unlock(&module->tierup_wait_lock);
return NULL;
}
}
os_mutex_unlock(&module->tierup_wait_lock);
#endif
#if WASM_ENABLE_JIT != 0
/* Compile llvm jit functions of this group */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
LLVMOrcJITTargetAddress func_addr = 0;
LLVMErrorRef error;
char func_name[48];
typedef void (*F)(void);
union {
F f;
void *v;
} u;
uint32 j;
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i,
"_wrapper");
LOG_DEBUG("compile llvm jit func %s", func_name);
error =
LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile llvm jit function %u: %s", i, err_msg);
LLVMDisposeErrorMessage(err_msg);
break;
}
/* Call the jit wrapper function to trigger its compilation, so as
to compile the actual jit functions, since we add the latter to
function list in the PartitionFunction callback */
u.v = (void *)func_addr;
u.f();
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count) {
module->func_ptrs_compiled[i + j * group_stride] = true;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX,
i + j * group_stride);
error = LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr,
func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile llvm jit function %u: %s", i,
err_msg);
LLVMDisposeErrorMessage(err_msg);
/* Ignore current llvm jit func, as its func ptr is
previous set to call_to_fast_jit, which also works */
continue;
}
jit_compiler_set_llvm_jit_func_ptr(
module,
i + j * group_stride + module->import_function_count,
(void *)func_addr);
/* Try to switch to call this llvm jit funtion instead of
fast jit function from fast jit jitted code */
jit_compiler_set_call_to_llvm_jit(
module,
i + j * group_stride + module->import_function_count);
#endif
}
}
if (module->orcjit_stop_compiling) {
break;
}
}
#endif
return NULL;
}
static void
orcjit_stop_compile_threads(WASMModule *module)
{
uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args)
/ sizeof(OrcJitThreadArg));
module->orcjit_stop_compiling = true;
for (i = 0; i < thread_num; i++) {
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
}
static bool
compile_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
uint32 thread_num =
(uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg));
uint32 i, j;
bh_print_time("Begin to compile jit functions");
thread_num =
(uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg));
/* Create threads to compile the jit functions */
for (i = 0; i < thread_num; i++) {
for (i = 0; i < thread_num && i < module->function_count; i++) {
#if WASM_ENABLE_JIT != 0
module->orcjit_thread_args[i].comp_ctx = module->comp_ctx;
#endif
module->orcjit_thread_args[i].module = module;
module->orcjit_thread_args[i].group_idx = i;
@ -3102,8 +3305,6 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
(void *)&module->orcjit_thread_args[i],
APP_THREAD_STACK_SIZE_DEFAULT)
!= 0) {
uint32 j;
set_error_buf(error_buf, error_buf_size,
"create orcjit compile thread failed");
/* Terminate the threads created */
@ -3118,15 +3319,39 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
#if WASM_ENABLE_LAZY_JIT == 0
/* Wait until all jit functions are compiled for eager mode */
for (i = 0; i < thread_num; i++) {
os_thread_join(module->orcjit_threads[i], NULL);
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
#if WASM_ENABLE_FAST_JIT != 0
/* Ensure all the fast-jit functions are compiled */
for (i = 0; i < module->function_count; i++) {
if (!jit_compiler_is_compiled(module,
i + module->import_function_count)) {
set_error_buf(error_buf, error_buf_size,
"failed to compile fast jit function");
return false;
}
}
#endif
#if WASM_ENABLE_JIT != 0
/* Ensure all the llvm-jit functions are compiled */
for (i = 0; i < module->function_count; i++) {
if (!module->func_ptrs_compiled[i]) {
set_error_buf(error_buf, error_buf_size,
"failed to compile llvm jit function");
return false;
}
}
#endif
#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */
bh_print_time("End compile jit functions");
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */
static bool
wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
@ -3538,23 +3763,41 @@ load_from_sections(WASMModule *module, WASMSection *sections,
calculate_global_data_offset(module);
#if WASM_ENABLE_FAST_JIT != 0
if (module->function_count
&& !(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count,
error_buf, error_buf_size))) {
return false;
}
if (!jit_compiler_compile_all(module)) {
set_error_buf(error_buf, error_buf_size, "fast jit compilation failed");
if (!init_fast_jit_functions(module, error_buf, error_buf_size)) {
return false;
}
#endif
#if WASM_ENABLE_JIT != 0
if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) {
if (!init_llvm_jit_functions_stage1(module, error_buf, error_buf_size)) {
return false;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0)
if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) {
return false;
}
#else
/* Run aot_compile_wasm in a backend thread, so as not to block the main
thread fast jit execution, since applying llvm optimizations in
aot_compile_wasm may cost a lot of time.
Create thread with enough native stack to apply llvm optimizations */
if (os_thread_create(&module->llvm_jit_init_thread,
init_llvm_jit_functions_stage2_callback,
(void *)module, APP_THREAD_STACK_SIZE_DEFAULT * 8)
!= 0) {
set_error_buf(error_buf, error_buf_size,
"create orcjit compile thread failed");
return false;
}
#endif
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* Create threads to compile the jit functions */
if (!compile_jit_functions(module, error_buf, error_buf_size)) {
return false;
}
#endif
#if WASM_ENABLE_MEMORY_TRACING != 0
wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module);
@ -3567,9 +3810,7 @@ create_module(char *error_buf, uint32 error_buf_size)
{
WASMModule *module =
loader_malloc(sizeof(WASMModule), error_buf, error_buf_size);
#if WASM_ENABLE_FAST_INTERP == 0
bh_list_status ret;
#endif
if (!module) {
return NULL;
@ -3584,19 +3825,31 @@ create_module(char *error_buf, uint32 error_buf_size)
module->br_table_cache_list = &module->br_table_cache_list_head;
ret = bh_list_init(module->br_table_cache_list);
bh_assert(ret == BH_LIST_SUCCESS);
(void)ret;
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
module->import_module_list = &module->import_module_list_head;
ret = bh_list_init(module->import_module_list);
bh_assert(ret == BH_LIST_SUCCESS);
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
bh_list_init(&module->fast_opcode_list);
if (os_mutex_init(&module->ref_count_lock) != 0) {
ret = bh_list_init(&module->fast_opcode_list);
bh_assert(ret == BH_LIST_SUCCESS);
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
&& WASM_ENABLE_LAZY_JIT != 0)
if (os_mutex_init(&module->instance_list_lock) != 0) {
set_error_buf(error_buf, error_buf_size,
"init instance list lock failed");
wasm_runtime_free(module);
return NULL;
}
#endif
(void)ret;
return module;
}
@ -3964,10 +4217,19 @@ wasm_loader_unload(WASMModule *module)
if (!module)
return;
#if WASM_ENABLE_JIT != 0
/* Stop LLVM JIT compilation firstly to avoid accessing
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
module->orcjit_stop_compiling = true;
if (module->llvm_jit_init_thread)
os_thread_join(module->llvm_jit_init_thread, NULL);
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* Stop Fast/LLVM JIT compilation firstly to avoid accessing
module internal data after they were freed */
orcjit_stop_compile_threads(module);
#endif
#if WASM_ENABLE_JIT != 0
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
@ -3976,6 +4238,13 @@ wasm_loader_unload(WASMModule *module)
aot_destroy_comp_data(module->comp_data);
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
if (module->tierup_wait_lock_inited) {
os_mutex_destroy(&module->tierup_wait_lock);
os_cond_destroy(&module->tierup_wait_cond);
}
#endif
if (module->types) {
for (i = 0; i < module->type_count; i++) {
if (module->types[i])
@ -3997,6 +4266,18 @@ wasm_loader_unload(WASMModule *module)
wasm_runtime_free(module->functions[i]->code_compiled);
if (module->functions[i]->consts)
wasm_runtime_free(module->functions[i]->consts);
#endif
#if WASM_ENABLE_FAST_JIT != 0
if (module->functions[i]->fast_jit_jitted_code) {
jit_code_cache_free(
module->functions[i]->fast_jit_jitted_code);
}
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->functions[i]->llvm_jit_func_ptr) {
jit_code_cache_free(
module->functions[i]->llvm_jit_func_ptr);
}
#endif
#endif
wasm_runtime_free(module->functions[i]);
}
@ -4084,7 +4365,12 @@ wasm_loader_unload(WASMModule *module)
wasm_runtime_free(fast_opcode);
fast_opcode = next;
}
os_mutex_destroy(&module->ref_count_lock);
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
&& WASM_ENABLE_LAZY_JIT != 0)
os_mutex_destroy(&module->instance_list_lock);
#endif
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
@ -4093,12 +4379,14 @@ wasm_loader_unload(WASMModule *module)
#if WASM_ENABLE_FAST_JIT != 0
if (module->fast_jit_func_ptrs) {
for (i = 0; i < module->function_count; i++) {
if (module->fast_jit_func_ptrs[i])
jit_code_cache_free(module->fast_jit_func_ptrs[i]);
}
wasm_runtime_free(module->fast_jit_func_ptrs);
}
for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) {
if (module->fast_jit_thread_locks_inited[i]) {
os_mutex_destroy(&module->fast_jit_thread_locks[i]);
}
}
#endif
wasm_runtime_free(module);

511
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -261,6 +261,12 @@ destroy_wasm_type(WASMType *type)
return;
}
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
if (type->call_to_llvm_jit_from_fast_jit)
jit_code_cache_free(type->call_to_llvm_jit_from_fast_jit);
#endif
wasm_runtime_free(type);
}
@ -1783,93 +1789,78 @@ calculate_global_data_offset(WASMModule *module)
module->global_data_size = data_offset;
}
#if WASM_ENABLE_JIT != 0
static void *
orcjit_thread_callback(void *arg)
{
LLVMOrcJITTargetAddress func_addr = 0;
OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg;
AOTCompContext *comp_ctx = thread_arg->comp_ctx;
WASMModule *module = thread_arg->module;
uint32 group_idx = thread_arg->group_idx;
uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
uint32 func_count = module->function_count;
uint32 i, j;
typedef void (*F)(void);
LLVMErrorRef error;
char func_name[48];
union {
F f;
void *v;
} u;
/* Compile jit functions of this group */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i,
"_wrapper");
LOG_DEBUG("compile func %s", func_name);
error =
LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile orc jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
continue;
}
/* Call the jit wrapper function to trigger its compilation, so as
to compile the actual jit functions, since we add the latter to
function list in the PartitionFunction callback */
u.v = (void *)func_addr;
u.f();
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count)
module->func_ptrs_compiled[i + j * group_stride] = true;
}
if (module->orcjit_stop_compiling) {
break;
}
}
return NULL;
}
static void
orcjit_stop_compile_threads(WASMModule *module)
{
uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args)
/ sizeof(OrcJitThreadArg));
module->orcjit_stop_compiling = true;
for (i = 0; i < thread_num; i++) {
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
}
#if WASM_ENABLE_FAST_JIT != 0
static bool
compile_llvm_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
init_fast_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
#if WASM_ENABLE_LAZY_JIT != 0
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
#endif
uint32 i;
if (!module->function_count)
return true;
if (!(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count, error_buf,
error_buf_size))) {
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
for (i = 0; i < module->function_count; i++) {
module->fast_jit_func_ptrs[i] =
jit_globals->compile_fast_jit_and_then_call;
}
#endif
for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) {
if (os_mutex_init(&module->fast_jit_thread_locks[i]) != 0) {
set_error_buf(error_buf, error_buf_size,
"init fast jit thread lock failed");
return false;
}
module->fast_jit_thread_locks_inited[i] = true;
}
return true;
}
#endif /* end of WASM_ENABLE_FAST_JIT != 0 */
#if WASM_ENABLE_JIT != 0
static bool
init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
AOTCompOption option = { 0 };
char *aot_last_error;
uint64 size;
uint32 thread_num, i;
if (module->function_count > 0) {
size = sizeof(void *) * (uint64)module->function_count
+ sizeof(bool) * (uint64)module->function_count;
if (!(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->func_ptrs_compiled =
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
if (module->function_count == 0)
return true;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LLVM_JIT != 0
if (os_mutex_init(&module->tierup_wait_lock) != 0) {
set_error_buf(error_buf, error_buf_size, "init jit tierup lock failed");
return false;
}
if (os_cond_init(&module->tierup_wait_cond) != 0) {
set_error_buf(error_buf, error_buf_size, "init jit tierup cond failed");
os_mutex_destroy(&module->tierup_wait_lock);
return false;
}
module->tierup_wait_lock_inited = true;
#endif
size = sizeof(void *) * (uint64)module->function_count
+ sizeof(bool) * (uint64)module->function_count;
if (!(module->func_ptrs = loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->func_ptrs_compiled =
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
module->comp_data = aot_create_comp_data(module);
if (!module->comp_data) {
@ -1910,6 +1901,19 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
return false;
}
return true;
}
static bool
init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
char *aot_last_error;
uint32 i;
if (module->function_count == 0)
return true;
if (!aot_compile_wasm(module->comp_ctx)) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
@ -1917,7 +1921,12 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
return false;
}
bh_print_time("Begin to lookup jit functions");
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->orcjit_stop_compiling)
return false;
#endif
bh_print_time("Begin to lookup llvm jit functions");
for (i = 0; i < module->function_count; i++) {
LLVMOrcJITTargetAddress func_addr = 0;
@ -1929,9 +1938,9 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
char buf[128];
char buf[96];
snprintf(buf, sizeof(buf),
"failed to compile orc jit function: %s\n", err_msg);
"failed to compile llvm jit function: %s", err_msg);
set_error_buf(error_buf, error_buf_size, buf);
LLVMDisposeErrorMessage(err_msg);
return false;
@ -1944,17 +1953,211 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
* loading/storing at the same time.
*/
module->func_ptrs[i] = (void *)func_addr;
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->orcjit_stop_compiling)
return false;
#endif
}
bh_print_time("End lookup llvm jit functions");
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
static void *
init_llvm_jit_functions_stage2_callback(void *arg)
{
WASMModule *module = (WASMModule *)arg;
char error_buf[128];
uint32 error_buf_size = (uint32)sizeof(error_buf);
if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) {
module->orcjit_stop_compiling = true;
return NULL;
}
os_mutex_lock(&module->tierup_wait_lock);
module->llvm_jit_inited = true;
os_cond_broadcast(&module->tierup_wait_cond);
os_mutex_unlock(&module->tierup_wait_lock);
return NULL;
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* The callback function to compile jit functions */
static void *
orcjit_thread_callback(void *arg)
{
OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg;
#if WASM_ENABLE_JIT != 0
AOTCompContext *comp_ctx = thread_arg->comp_ctx;
#endif
WASMModule *module = thread_arg->module;
uint32 group_idx = thread_arg->group_idx;
uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
uint32 func_count = module->function_count;
uint32 i;
#if WASM_ENABLE_FAST_JIT != 0
/* Compile fast jit funcitons of this group */
for (i = group_idx; i < func_count; i += group_stride) {
if (!jit_compiler_compile(module, i + module->import_function_count)) {
os_printf("failed to compile fast jit function %u\n", i);
break;
}
if (module->orcjit_stop_compiling) {
return NULL;
}
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* For JIT tier-up, set each llvm jit func to call_to_fast_jit */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
uint32 j;
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count) {
if (!jit_compiler_set_call_to_fast_jit(
module,
i + j * group_stride + module->import_function_count)) {
os_printf(
"failed to compile call_to_fast_jit for func %u\n",
i + j * group_stride + module->import_function_count);
module->orcjit_stop_compiling = true;
return NULL;
}
}
if (module->orcjit_stop_compiling) {
return NULL;
}
}
}
/* Wait until init_llvm_jit_functions_stage2 finishes */
os_mutex_lock(&module->tierup_wait_lock);
while (!module->llvm_jit_inited) {
os_cond_reltimedwait(&module->tierup_wait_cond,
&module->tierup_wait_lock, 10);
if (module->orcjit_stop_compiling) {
/* init_llvm_jit_functions_stage2 failed */
os_mutex_unlock(&module->tierup_wait_lock);
return NULL;
}
}
os_mutex_unlock(&module->tierup_wait_lock);
#endif
#if WASM_ENABLE_JIT != 0
/* Compile llvm jit functions of this group */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
LLVMOrcJITTargetAddress func_addr = 0;
LLVMErrorRef error;
char func_name[48];
typedef void (*F)(void);
union {
F f;
void *v;
} u;
uint32 j;
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i,
"_wrapper");
LOG_DEBUG("compile llvm jit func %s", func_name);
error =
LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile llvm jit function %u: %s", i, err_msg);
LLVMDisposeErrorMessage(err_msg);
break;
}
/* Call the jit wrapper function to trigger its compilation, so as
to compile the actual jit functions, since we add the latter to
function list in the PartitionFunction callback */
u.v = (void *)func_addr;
u.f();
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count) {
module->func_ptrs_compiled[i + j * group_stride] = true;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX,
i + j * group_stride);
error = LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr,
func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile llvm jit function %u: %s", i,
err_msg);
LLVMDisposeErrorMessage(err_msg);
/* Ignore current llvm jit func, as its func ptr is
previous set to call_to_fast_jit, which also works */
continue;
}
jit_compiler_set_llvm_jit_func_ptr(
module,
i + j * group_stride + module->import_function_count,
(void *)func_addr);
/* Try to switch to call this llvm jit funtion instead of
fast jit function from fast jit jitted code */
jit_compiler_set_call_to_llvm_jit(
module,
i + j * group_stride + module->import_function_count);
#endif
}
}
if (module->orcjit_stop_compiling) {
break;
}
}
#endif
return NULL;
}
static void
orcjit_stop_compile_threads(WASMModule *module)
{
uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args)
/ sizeof(OrcJitThreadArg));
module->orcjit_stop_compiling = true;
for (i = 0; i < thread_num; i++) {
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
}
static bool
compile_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
uint32 thread_num =
(uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg));
uint32 i, j;
bh_print_time("Begin to compile jit functions");
thread_num =
(uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg));
/* Create threads to compile the jit functions */
for (i = 0; i < thread_num; i++) {
for (i = 0; i < thread_num && i < module->function_count; i++) {
#if WASM_ENABLE_JIT != 0
module->orcjit_thread_args[i].comp_ctx = module->comp_ctx;
#endif
module->orcjit_thread_args[i].module = module;
module->orcjit_thread_args[i].group_idx = i;
@ -1962,8 +2165,6 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
(void *)&module->orcjit_thread_args[i],
APP_THREAD_STACK_SIZE_DEFAULT)
!= 0) {
uint32 j;
set_error_buf(error_buf, error_buf_size,
"create orcjit compile thread failed");
/* Terminate the threads created */
@ -1978,15 +2179,39 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
#if WASM_ENABLE_LAZY_JIT == 0
/* Wait until all jit functions are compiled for eager mode */
for (i = 0; i < thread_num; i++) {
os_thread_join(module->orcjit_threads[i], NULL);
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
#if WASM_ENABLE_FAST_JIT != 0
/* Ensure all the fast-jit functions are compiled */
for (i = 0; i < module->function_count; i++) {
if (!jit_compiler_is_compiled(module,
i + module->import_function_count)) {
set_error_buf(error_buf, error_buf_size,
"failed to compile fast jit function");
return false;
}
}
#endif
#if WASM_ENABLE_JIT != 0
/* Ensure all the llvm-jit functions are compiled */
for (i = 0; i < module->function_count; i++) {
if (!module->func_ptrs_compiled[i]) {
set_error_buf(error_buf, error_buf_size,
"failed to compile llvm jit function");
return false;
}
}
#endif
#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */
bh_print_time("End compile jit functions");
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_REF_TYPES != 0
static bool
@ -2427,22 +2652,41 @@ load_from_sections(WASMModule *module, WASMSection *sections,
calculate_global_data_offset(module);
#if WASM_ENABLE_FAST_JIT != 0
if (!(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count, error_buf,
error_buf_size))) {
return false;
}
if (!jit_compiler_compile_all(module)) {
set_error_buf(error_buf, error_buf_size, "fast jit compilation failed");
if (!init_fast_jit_functions(module, error_buf, error_buf_size)) {
return false;
}
#endif
#if WASM_ENABLE_JIT != 0
if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) {
if (!init_llvm_jit_functions_stage1(module, error_buf, error_buf_size)) {
return false;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0)
if (!init_llvm_jit_functions_stage2(module, error_buf, error_buf_size)) {
return false;
}
#else
/* Run aot_compile_wasm in a backend thread, so as not to block the main
thread fast jit execution, since applying llvm optimizations in
aot_compile_wasm may cost a lot of time.
Create thread with enough native stack to apply llvm optimizations */
if (os_thread_create(&module->llvm_jit_init_thread,
init_llvm_jit_functions_stage2_callback,
(void *)module, APP_THREAD_STACK_SIZE_DEFAULT * 8)
!= 0) {
set_error_buf(error_buf, error_buf_size,
"create orcjit compile thread failed");
return false;
}
#endif
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* Create threads to compile the jit functions */
if (!compile_jit_functions(module, error_buf, error_buf_size)) {
return false;
}
#endif
#if WASM_ENABLE_MEMORY_TRACING != 0
wasm_runtime_dump_module_mem_consumption(module);
@ -2455,9 +2699,7 @@ create_module(char *error_buf, uint32 error_buf_size)
{
WASMModule *module =
loader_malloc(sizeof(WASMModule), error_buf, error_buf_size);
#if WASM_ENABLE_FAST_INTERP == 0
bh_list_status ret;
#endif
if (!module) {
return NULL;
@ -2472,9 +2714,18 @@ create_module(char *error_buf, uint32 error_buf_size)
module->br_table_cache_list = &module->br_table_cache_list_head;
ret = bh_list_init(module->br_table_cache_list);
bh_assert(ret == BH_LIST_SUCCESS);
(void)ret;
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
if (os_mutex_init(&module->instance_list_lock) != 0) {
set_error_buf(error_buf, error_buf_size,
"init instance list lock failed");
wasm_runtime_free(module);
return NULL;
}
#endif
(void)ret;
return module;
}
@ -2686,10 +2937,19 @@ wasm_loader_unload(WASMModule *module)
if (!module)
return;
#if WASM_ENABLE_JIT != 0
/* Stop LLVM JIT compilation firstly to avoid accessing
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
module->orcjit_stop_compiling = true;
if (module->llvm_jit_init_thread)
os_thread_join(module->llvm_jit_init_thread, NULL);
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
/* Stop Fast/LLVM JIT compilation firstly to avoid accessing
module internal data after they were freed */
orcjit_stop_compile_threads(module);
#endif
#if WASM_ENABLE_JIT != 0
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
@ -2698,6 +2958,13 @@ wasm_loader_unload(WASMModule *module)
aot_destroy_comp_data(module->comp_data);
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
if (module->tierup_wait_lock_inited) {
os_mutex_destroy(&module->tierup_wait_lock);
os_cond_destroy(&module->tierup_wait_cond);
}
#endif
if (module->types) {
for (i = 0; i < module->type_count; i++) {
if (module->types[i])
@ -2719,6 +2986,18 @@ wasm_loader_unload(WASMModule *module)
wasm_runtime_free(module->functions[i]->code_compiled);
if (module->functions[i]->consts)
wasm_runtime_free(module->functions[i]->consts);
#endif
#if WASM_ENABLE_FAST_JIT != 0
if (module->functions[i]->fast_jit_jitted_code) {
jit_code_cache_free(
module->functions[i]->fast_jit_jitted_code);
}
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->functions[i]->llvm_jit_func_ptr) {
jit_code_cache_free(
module->functions[i]->llvm_jit_func_ptr);
}
#endif
#endif
wasm_runtime_free(module->functions[i]);
}
@ -2775,14 +3054,20 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
os_mutex_destroy(&module->instance_list_lock);
#endif
#if WASM_ENABLE_FAST_JIT != 0
if (module->fast_jit_func_ptrs) {
for (i = 0; i < module->function_count; i++) {
if (module->fast_jit_func_ptrs[i])
jit_code_cache_free(module->fast_jit_func_ptrs[i]);
}
wasm_runtime_free(module->fast_jit_func_ptrs);
}
for (i = 0; i < WASM_ORC_JIT_BACKEND_THREAD_NUM; i++) {
if (module->fast_jit_thread_locks_inited[i]) {
os_mutex_destroy(&module->fast_jit_thread_locks[i]);
}
}
#endif
wasm_runtime_free(module);
@ -5572,12 +5857,6 @@ re_scan:
if (depth > 255) {
/* The depth cannot be stored in one byte,
create br_table cache to store each depth */
#if WASM_ENABLE_DEBUG_INTERP != 0
if (!record_fast_op(module, p_org, *p_org,
error_buf, error_buf_size)) {
goto fail;
}
#endif
if (!(br_table_cache = loader_malloc(
offsetof(BrTableCache, br_depths)
+ sizeof(uint32)

79
core/iwasm/interpreter/wasm_runtime.c
View File

@ -19,6 +19,9 @@
#if WASM_ENABLE_DEBUG_INTERP != 0
#include "../libraries/debug-engine/debug_engine.h"
#endif
#if WASM_ENABLE_FAST_JIT != 0
#include "../fast-jit/jit_compiler.h"
#endif
#if WASM_ENABLE_JIT != 0
#include "../aot/aot_runtime.h"
#endif
@ -1414,30 +1417,10 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
extra_info_offset = (uint32)total_size;
total_size += sizeof(WASMModuleInstanceExtra);
#if WASM_ENABLE_DEBUG_INTERP != 0
if (!is_sub_inst) {
os_mutex_lock(&module->ref_count_lock);
if (module->ref_count != 0) {
LOG_WARNING(
"warning: multiple instances referencing the same module may "
"cause unexpected behaviour during debugging");
}
module->ref_count++;
os_mutex_unlock(&module->ref_count_lock);
}
#endif
/* Allocate the memory for module instance with memory instances,
global data, table data appended at the end */
if (!(module_inst =
runtime_malloc(total_size, error_buf, error_buf_size))) {
#if WASM_ENABLE_DEBUG_INTERP != 0
if (!is_sub_inst) {
os_mutex_lock(&module->ref_count_lock);
module->ref_count--;
os_mutex_unlock(&module->ref_count_lock);
}
#endif
return NULL;
}
@ -1827,6 +1810,33 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
}
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
if (!is_sub_inst) {
/* Add module instance into module's instance list */
os_mutex_lock(&module->instance_list_lock);
#if WASM_ENABLE_DEBUG_INTERP != 0
if (module->instance_list) {
LOG_WARNING(
"warning: multiple instances referencing to the same module "
"may cause unexpected behaviour during debugging");
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* Copy llvm func ptrs again in case that they were updated
after the module instance was created */
bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
sizeof(void *) * module->function_count, module->func_ptrs,
sizeof(void *) * module->function_count);
#endif
module_inst->e->next = module->instance_list;
module->instance_list = module_inst;
os_mutex_unlock(&module->instance_list_lock);
}
#endif
if (module->start_function != (uint32)-1) {
/* TODO: fix start function can be import function issue */
if (module->start_function >= module->import_function_count)
@ -1864,8 +1874,8 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
wasm_runtime_dump_module_inst_mem_consumption(
(WASMModuleInstanceCommon *)module_inst);
#endif
(void)global_data_end;
(void)global_data_end;
return module_inst;
fail:
@ -1935,11 +1945,28 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
}
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
if (!is_sub_inst) {
os_mutex_lock(&module_inst->module->ref_count_lock);
module_inst->module->ref_count--;
os_mutex_unlock(&module_inst->module->ref_count_lock);
WASMModule *module = module_inst->module;
WASMModuleInstance *instance_prev = NULL, *instance;
os_mutex_lock(&module->instance_list_lock);
instance = module->instance_list;
while (instance) {
if (instance == module_inst) {
if (!instance_prev)
module->instance_list = instance->e->next;
else
instance_prev->e->next = instance->e->next;
break;
}
instance_prev = instance;
instance = instance->e->next;
}
os_mutex_unlock(&module->instance_list_lock);
}
#endif
@ -2819,7 +2846,7 @@ fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true,
type_idx);
}
#endif
#endif /* end of WASM_ENABLE_FAST_JIT != 0 */
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0

9
core/iwasm/interpreter/wasm_runtime.h
View File

@ -59,6 +59,9 @@ typedef enum WASMExceptionID {
EXCE_AUX_STACK_UNDERFLOW,
EXCE_OUT_OF_BOUNDS_TABLE_ACCESS,
EXCE_OPERAND_STACK_OVERFLOW,
#if WASM_ENABLE_FAST_JIT != 0
EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC,
#endif
EXCE_ALREADY_THROWN,
EXCE_NUM,
} WASMExceptionID;
@ -233,6 +236,12 @@ typedef struct WASMModuleInstanceExtra {
#if WASM_ENABLE_MEMORY_PROFILING != 0
uint32 max_aux_stack_used;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
&& WASM_ENABLE_LAZY_JIT != 0)
WASMModuleInstance *next;
#endif
} WASMModuleInstanceExtra;
struct AOTFuncPerfProfInfo;