christoph/wamr
1
Fork 0
Code Releases Activity

Merge branch main into dev/wasi_threads

Browse Source
This commit is contained in:
Wenyong Huang
2023-02-17 08:46:12 +08:00
parent ee1871d3f8 50650e4634
commit e170c355a2
163 changed files with 7153 additions and 1857 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
core/iwasm/interpreter/wasm.h
View File

@ -278,9 +278,14 @@ struct WASMFunction {
#endif
#if WASM_ENABLE_FAST_JIT != 0
/* The compiled fast jit jitted code block of this function */
void *fast_jit_jitted_code;
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
/* The compiled llvm jit func ptr of this function */
void *llvm_jit_func_ptr;
/* Code block to call fast jit jitted code of this function
from the llvm jit jitted code */
void *call_to_fast_jit_from_llvm_jit;
#endif
#endif
};
@ -505,15 +510,14 @@ struct WASMModule {
uint64 load_size;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
/**
* List of instances referred to this module. When source debugging
* feature is enabled, the debugger may modify the code section of
* the module, so we need to report a warning if user create several
* instances based on the same module. Sub instances created by
* lib-pthread or spawn API won't be added into the list.
* instances based on the same module.
*
* Also add the instance to the list for Fast JIT to LLVM JIT
* tier-up, since we need to lazily update the LLVM func pointers
@ -533,7 +537,22 @@ struct WASMModule {
#endif
#if WASM_ENABLE_FAST_JIT != 0
/* func pointers of Fast JITed (un-imported) functions */
/**
* func pointers of Fast JITed (un-imported) functions
* for non Multi-Tier JIT mode:
* (1) when lazy jit is disabled, each pointer is set to the compiled
* fast jit jitted code
* (2) when lazy jit is enabled, each pointer is firstly inited as
* jit_global->compile_fast_jit_and_then_call, and then set to the
* compiled fast jit jitted code when it is called (the stub will
* compile the jit function and then update itself)
* for Multi-Tier JIT mode:
* each pointer is firstly inited as compile_fast_jit_and_then_call,
* and then set to the compiled fast jit jitted code when it is called,
* and when the llvm jit func ptr of the same function is compiled, it
* will be set to call_to_llvm_jit_from_fast_jit of this function type
* (tier-up from fast-jit to llvm-jit)
*/
void **fast_jit_func_ptrs;
/* locks for Fast JIT lazy compilation */
korp_mutex fast_jit_thread_locks[WASM_ORC_JIT_BACKEND_THREAD_NUM];
@ -543,7 +562,16 @@ struct WASMModule {
#if WASM_ENABLE_JIT != 0
struct AOTCompData *comp_data;
struct AOTCompContext *comp_ctx;
/* func pointers of LLVM JITed (un-imported) functions */
/**
* func pointers of LLVM JITed (un-imported) functions
* for non Multi-Tier JIT mode:
* each pointer is set to the lookuped llvm jit func ptr, note that it
* is a stub and will trigger the actual compilation when it is called
* for Multi-Tier JIT mode:
* each pointer is inited as call_to_fast_jit code block, when the llvm
* jit func ptr is actually compiled, it is set to the compiled llvm jit
* func ptr
*/
void **func_ptrs;
/* whether the func pointers are compiled */
bool *func_ptrs_compiled;
@ -568,6 +596,12 @@ struct WASMModule {
korp_tid llvm_jit_init_thread;
/* whether the llvm jit is initialized */
bool llvm_jit_inited;
/* Whether to enable llvm jit compilation:
it is set to true only when there is a module instance starts to
run with running mode Mode_LLVM_JIT or Mode_Multi_Tier_JIT,
since no need to enable llvm jit compilation for Mode_Interp and
Mode_Fast_JIT, so as to improve performance for them */
bool enable_llvm_jit_compilation;
#endif
};

197
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -696,28 +696,28 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint32)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint32)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = LOAD_I32(maddr); \
STORE_U32(maddr, readv op sval); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
PUSH_I32(readv); \
break; \
@ -736,39 +736,39 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)LOAD_U32(maddr); \
STORE_U32(maddr, (uint32)(readv op sval)); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else { \
uint64 op_result; \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)LOAD_I64(maddr); \
op_result = readv op sval; \
STORE_I64(maddr, op_result); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
PUSH_I64(readv); \
break; \
@ -1151,6 +1151,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 cache_index, type_index, param_cell_num, cell_num;
uint8 value_type;
#if WASM_ENABLE_SHARED_MEMORY != 0
WASMSharedMemNode *node =
wasm_module_get_shared_memory((WASMModuleCommon *)module->module);
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
uint8 *frame_ip_orig = NULL;
WASMDebugInstance *debug_instance = wasm_exec_env_get_instance(exec_env);
@ -3458,23 +3463,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)(*(uint8 *)maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)LOAD_U16(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = LOAD_I32(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I32(readv);
@ -3493,30 +3498,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)(*(uint8 *)maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U16(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U32(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = LOAD_I64(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I64(readv);
@ -3535,23 +3540,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U32(maddr, frame_sp[1]);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
break;
}
@ -3569,31 +3574,31 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE32) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U32(maddr, (uint32)sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
PUT_I64_TO_ADDR((uint32 *)maddr,
GET_I64_FROM_ADDR(frame_sp + 1));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
break;
}
@ -3613,32 +3618,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint8)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint16)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = LOAD_I32(maddr);
if (readv == expect)
STORE_U32(maddr, sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I32(readv);
break;
@ -3659,44 +3664,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint8)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint16)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint32)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U32(maddr);
if (readv == expect)
STORE_U32(maddr, (uint32)(sval));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_I64(maddr);
if (readv == expect) {
STORE_I64(maddr, sval);
}
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I64(readv);
break;
@ -4150,6 +4155,7 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
argc = function->param_cell_num;
RECORD_STACK_USAGE(exec_env, (uint8 *)&prev_frame);
#if !(defined(OS_ENABLE_HW_BOUND_CHECK) \
&& WASM_DISABLE_STACK_HW_BOUND_CHECK == 0)
if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) {
@ -4194,58 +4200,51 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
}
else {
#if WASM_ENABLE_LAZY_JIT != 0
RunningMode running_mode =
wasm_runtime_get_running_mode((wasm_module_inst_t)module_inst);
/* Fast JIT to LLVM JIT tier-up is enabled */
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0
/* Fast JIT and LLVM JIT are both enabled, call llvm jit function
if it is compiled, else call fast jit function */
uint32 func_idx = (uint32)(function - module_inst->e->functions);
if (module_inst->module->func_ptrs_compiled
[func_idx - module_inst->module->import_function_count]) {
if (running_mode == Mode_Interp) {
wasm_interp_call_func_bytecode(module_inst, exec_env, function,
frame);
}
#if WASM_ENABLE_FAST_JIT != 0
else if (running_mode == Mode_Fast_JIT) {
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
}
#endif
#if WASM_ENABLE_JIT != 0
else if (running_mode == Mode_LLVM_JIT) {
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
}
else {
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 \
&& WASM_ENABLE_JIT != 0
else if (running_mode == Mode_Multi_Tier_JIT) {
/* Tier-up from Fast JIT to LLVM JIT, call llvm jit function
if it is compiled, else call fast jit function */
uint32 func_idx = (uint32)(function - module_inst->e->functions);
if (module_inst->module->func_ptrs_compiled
[func_idx - module_inst->module->import_function_count]) {
llvm_jit_call_func_bytecode(module_inst, exec_env, function,
argc, argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
}
else {
fast_jit_call_func_bytecode(module_inst, exec_env, function,
frame);
}
}
#elif WASM_ENABLE_JIT != 0
/* Only LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
/* Only Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#else
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
#else /* else of WASM_ENABLE_LAZY_JIT != 0 */
/* Fast JIT to LLVM JIT tier-up is enabled */
#if WASM_ENABLE_JIT != 0
/* LLVM JIT is enabled */
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
/* Fast JIT is enabled */
fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
#else
/* Both Fast JIT and LLVM JIT are disabled */
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
else {
/* There should always be a supported running mode selected */
bh_assert(0);
}
(void)wasm_interp_call_func_bytecode;
#if WASM_ENABLE_FAST_JIT != 0

118
core/iwasm/interpreter/wasm_interp_fast.c
View File

@ -469,28 +469,28 @@ LOAD_PTR(void *addr)
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(1); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint32)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(2); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint32)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(4); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = LOAD_I32(maddr); \
STORE_U32(maddr, readv op sval); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
PUSH_I32(readv); \
break; \
@ -509,39 +509,39 @@ LOAD_PTR(void *addr)
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(1); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(2); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(4); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)LOAD_U32(maddr); \
STORE_U32(maddr, (uint32)(readv op sval)); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
else { \
uint64 op_result; \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(8); \
\
os_mutex_lock(&module->e->mem_lock); \
os_mutex_lock(&node->shared_mem_lock); \
readv = (uint64)LOAD_I64(maddr); \
op_result = readv op sval; \
STORE_I64(maddr, op_result); \
os_mutex_unlock(&module->e->mem_lock); \
os_mutex_unlock(&node->shared_mem_lock); \
} \
PUSH_I64(readv); \
break; \
@ -1183,6 +1183,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 local_idx, local_offset, global_idx;
uint8 opcode, local_type, *global_addr;
#if WASM_ENABLE_SHARED_MEMORY != 0
WASMSharedMemNode *node =
wasm_module_get_shared_memory((WASMModuleCommon *)module->module);
#endif
#if WASM_ENABLE_LABELS_AS_VALUES != 0
#define HANDLE_OPCODE(op) &&HANDLE_##op
DEFINE_GOTO_TABLE(const void *, handle_table);
@ -3296,23 +3301,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)(*(uint8 *)maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)LOAD_U16(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = LOAD_I32(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I32(readv);
@ -3331,30 +3336,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)(*(uint8 *)maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U16(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U32(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(8);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = LOAD_I64(maddr);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I64(readv);
@ -3372,23 +3377,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U32(maddr, sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
break;
}
@ -3406,30 +3411,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE32) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_U32(maddr, (uint32)sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(8);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
STORE_I64(maddr, sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
break;
}
@ -3449,32 +3454,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS(1);
expect = (uint8)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
expect = (uint16)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint32)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = LOAD_I32(maddr);
if (readv == expect)
STORE_U32(maddr, sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I32(readv);
break;
@ -3495,44 +3500,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS(1);
expect = (uint8)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
expect = (uint16)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
expect = (uint32)expect;
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_U32(maddr);
if (readv == expect)
STORE_U32(maddr, (uint32)(sval));
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(8);
os_mutex_lock(&module->e->mem_lock);
os_mutex_lock(&node->shared_mem_lock);
readv = (uint64)LOAD_I64(maddr);
if (readv == expect) {
STORE_I64(maddr, sval);
}
os_mutex_unlock(&module->e->mem_lock);
os_mutex_unlock(&node->shared_mem_lock);
}
PUSH_I64(readv);
break;
@ -3901,6 +3906,7 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
argc = function->param_cell_num;
RECORD_STACK_USAGE(exec_env, (uint8 *)&prev_frame);
#if !(defined(OS_ENABLE_HW_BOUND_CHECK) \
&& WASM_DISABLE_STACK_HW_BOUND_CHECK == 0)
if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) {

50
core/iwasm/interpreter/wasm_loader.c
View File

@ -1399,6 +1399,7 @@ load_global_import(const uint8 **p_buf, const uint8 *buf_end,
WASMModule *sub_module = NULL;
WASMGlobal *linked_global = NULL;
#endif
bool ret = false;
CHECK_BUF(p, p_end, 2);
declare_type = read_uint8(p);
@ -1411,15 +1412,16 @@ load_global_import(const uint8 **p_buf, const uint8 *buf_end,
}
#if WASM_ENABLE_LIBC_BUILTIN != 0
global->is_linked = wasm_native_lookup_libc_builtin_global(
sub_module_name, global_name, global);
if (global->is_linked) {
ret = wasm_native_lookup_libc_builtin_global(sub_module_name, global_name,
global);
if (ret) {
if (global->type != declare_type
|| global->is_mutable != declare_mutable) {
set_error_buf(error_buf, error_buf_size,
"incompatible import type");
return false;
}
global->is_linked = true;
}
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
@ -1449,6 +1451,7 @@ load_global_import(const uint8 **p_buf, const uint8 *buf_end,
global->is_mutable = (declare_mutable == 1);
(void)parent_module;
(void)ret;
return true;
fail:
return false;
@ -2989,6 +2992,7 @@ static bool
init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
LLVMJITOptions llvm_jit_options = wasm_runtime_get_llvm_jit_options();
AOTCompOption option = { 0 };
char *aot_last_error;
uint64 size;
@ -3027,8 +3031,11 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
}
option.is_jit_mode = true;
option.opt_level = 3;
option.size_level = 3;
llvm_jit_options = wasm_runtime_get_llvm_jit_options();
option.opt_level = llvm_jit_options.opt_level;
option.size_level = llvm_jit_options.size_level;
#if WASM_ENABLE_BULK_MEMORY != 0
option.enable_bulk_memory = true;
#endif
@ -3048,6 +3055,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0)
option.enable_aux_stack_frame = true;
#endif
#if WASM_ENABLE_MEMORY_PROFILING != 0
option.enable_stack_estimation = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {
@ -3109,6 +3119,8 @@ init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf,
module->func_ptrs[i] = (void *)func_addr;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
if (module->orcjit_stop_compiling)
return false;
#endif
@ -3199,9 +3211,9 @@ orcjit_thread_callback(void *arg)
/* Wait until init_llvm_jit_functions_stage2 finishes */
os_mutex_lock(&module->tierup_wait_lock);
while (!module->llvm_jit_inited) {
while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation)) {
os_cond_reltimedwait(&module->tierup_wait_cond,
&module->tierup_wait_lock, 10);
&module->tierup_wait_lock, 10000);
if (module->orcjit_stop_compiling) {
/* init_llvm_jit_functions_stage2 failed */
os_mutex_unlock(&module->tierup_wait_lock);
@ -3852,8 +3864,8 @@ create_module(char *error_buf, uint32 error_buf_size)
bh_assert(ret == BH_LIST_SUCCESS);
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
if (os_mutex_init(&module->instance_list_lock) != 0) {
set_error_buf(error_buf, error_buf_size,
@ -4148,10 +4160,8 @@ check_wasi_abi_compatibility(const WASMModule *module,
/* should have one at least */
if (module->import_wasi_api && !start && !initialize) {
set_error_buf(
error_buf, error_buf_size,
"a module with WASI apis must be either a command or a reactor");
return false;
LOG_WARNING("warning: a module with WASI apis should be either "
"a command or a reactor");
}
/*
@ -4256,7 +4266,8 @@ wasm_loader_unload(WASMModule *module)
if (!module)
return;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
module->orcjit_stop_compiling = true;
if (module->llvm_jit_init_thread)
os_thread_join(module->llvm_jit_init_thread, NULL);
@ -4277,7 +4288,8 @@ wasm_loader_unload(WASMModule *module)
aot_destroy_comp_data(module->comp_data);
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
if (module->tierup_wait_lock_inited) {
os_mutex_destroy(&module->tierup_wait_lock);
os_cond_destroy(&module->tierup_wait_cond);
@ -4312,9 +4324,9 @@ wasm_loader_unload(WASMModule *module)
module->functions[i]->fast_jit_jitted_code);
}
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->functions[i]->llvm_jit_func_ptr) {
if (module->functions[i]->call_to_fast_jit_from_llvm_jit) {
jit_code_cache_free(
module->functions[i]->llvm_jit_func_ptr);
module->functions[i]->call_to_fast_jit_from_llvm_jit);
}
#endif
#endif
@ -4406,8 +4418,8 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
os_mutex_destroy(&module->instance_list_lock);
#endif

31
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -1835,6 +1835,7 @@ static bool
init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
LLVMJITOptions llvm_jit_options = wasm_runtime_get_llvm_jit_options();
AOTCompOption option = { 0 };
char *aot_last_error;
uint64 size;
@ -1873,8 +1874,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
}
option.is_jit_mode = true;
option.opt_level = 3;
option.size_level = 3;
option.opt_level = llvm_jit_options.opt_level;
option.size_level = llvm_jit_options.size_level;
#if WASM_ENABLE_BULK_MEMORY != 0
option.enable_bulk_memory = true;
#endif
@ -1894,6 +1896,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0)
option.enable_aux_stack_frame = true;
#endif
#if WASM_ENABLE_MEMORY_PROFILING != 0
option.enable_stack_estimation = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {
@ -1957,6 +1962,8 @@ init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf,
module->func_ptrs[i] = (void *)func_addr;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
if (module->orcjit_stop_compiling)
return false;
#endif
@ -2047,9 +2054,9 @@ orcjit_thread_callback(void *arg)
/* Wait until init_llvm_jit_functions_stage2 finishes */
os_mutex_lock(&module->tierup_wait_lock);
while (!module->llvm_jit_inited) {
while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation)) {
os_cond_reltimedwait(&module->tierup_wait_cond,
&module->tierup_wait_lock, 10);
&module->tierup_wait_lock, 10000);
if (module->orcjit_stop_compiling) {
/* init_llvm_jit_functions_stage2 failed */
os_mutex_unlock(&module->tierup_wait_lock);
@ -2718,7 +2725,8 @@ create_module(char *error_buf, uint32 error_buf_size)
bh_assert(ret == BH_LIST_SUCCESS);
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
if (os_mutex_init(&module->instance_list_lock) != 0) {
set_error_buf(error_buf, error_buf_size,
"init instance list lock failed");
@ -2939,7 +2947,8 @@ wasm_loader_unload(WASMModule *module)
if (!module)
return;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
module->orcjit_stop_compiling = true;
if (module->llvm_jit_init_thread)
os_thread_join(module->llvm_jit_init_thread, NULL);
@ -2960,7 +2969,8 @@ wasm_loader_unload(WASMModule *module)
aot_destroy_comp_data(module->comp_data);
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
if (module->tierup_wait_lock_inited) {
os_mutex_destroy(&module->tierup_wait_lock);
os_cond_destroy(&module->tierup_wait_cond);
@ -2995,9 +3005,9 @@ wasm_loader_unload(WASMModule *module)
module->functions[i]->fast_jit_jitted_code);
}
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
if (module->functions[i]->llvm_jit_func_ptr) {
if (module->functions[i]->call_to_fast_jit_from_llvm_jit) {
jit_code_cache_free(
module->functions[i]->llvm_jit_func_ptr);
module->functions[i]->call_to_fast_jit_from_llvm_jit);
}
#endif
#endif
@ -3056,7 +3066,8 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT && WASM_ENABLE_LAZY_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
os_mutex_destroy(&module->instance_list_lock);
#endif

292
core/iwasm/interpreter/wasm_runtime.c
View File

@ -737,13 +737,12 @@ functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
function++;
}
bh_assert((uint32)(function - functions) == function_count);
#if WASM_ENABLE_FAST_JIT != 0
module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
#endif
bh_assert((uint32)(function - functions) == function_count);
(void)module_inst;
return functions;
}
@ -1288,9 +1287,8 @@ init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module,
*func_ptrs = import_func->func_ptr_linked;
}
/* Set defined function pointers */
bh_memcpy_s(func_ptrs, sizeof(void *) * module->function_count,
module->func_ptrs, sizeof(void *) * module->function_count);
/* The defined function pointers will be set in
wasm_runtime_set_running_mode, no need to set them here */
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
@ -1336,6 +1334,173 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf,
}
#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */
static bool
set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode,
bool first_time_set)
{
WASMModule *module = module_inst->module;
if (running_mode == Mode_Default) {
#if WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT == 0
running_mode = Mode_Interp;
#elif WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT == 0
running_mode = Mode_Fast_JIT;
#elif WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT != 0
running_mode = Mode_LLVM_JIT;
#else /* WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_LAZY_JIT == 0
running_mode = Mode_LLVM_JIT;
#else
running_mode = Mode_Multi_Tier_JIT;
#endif
#endif
}
if (!wasm_runtime_is_running_mode_supported(running_mode))
return false;
#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */
module_inst->e->running_mode = running_mode;
if (running_mode == Mode_Interp) {
/* Do nothing for Mode_Interp */
}
else if (running_mode == Mode_Fast_JIT) {
/* Do nothing for Mode_Fast_JIT since
module_inst->fast_jit_func_ptrs is same as
module->fast_jit_func_ptrs */
}
#if WASM_ENABLE_JIT != 0
else if (running_mode == Mode_LLVM_JIT) {
/* Set defined function pointers */
bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
sizeof(void *) * module->function_count, module->func_ptrs,
sizeof(void *) * module->function_count);
}
#endif
else {
bh_assert(0);
}
#else /* Possible multi-tier JIT */
os_mutex_lock(&module->instance_list_lock);
module_inst->e->running_mode = running_mode;
if (running_mode == Mode_Interp) {
/* Do nothing for Mode_Interp */
}
#if WASM_ENABLE_FAST_JIT != 0
else if (running_mode == Mode_Fast_JIT) {
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
uint32 i;
/* Allocate memory for fast_jit_func_ptrs if needed */
if (!module_inst->fast_jit_func_ptrs
|| module_inst->fast_jit_func_ptrs == module->fast_jit_func_ptrs) {
uint64 total_size = (uint64)sizeof(void *) * module->function_count;
if (!(module_inst->fast_jit_func_ptrs =
runtime_malloc(total_size, NULL, 0))) {
os_mutex_unlock(&module->instance_list_lock);
return false;
}
}
for (i = 0; i < module->function_count; i++) {
if (module->functions[i]->fast_jit_jitted_code) {
/* current fast jit function has been compiled */
module_inst->fast_jit_func_ptrs[i] =
module->functions[i]->fast_jit_jitted_code;
}
else {
module_inst->fast_jit_func_ptrs[i] =
jit_globals->compile_fast_jit_and_then_call;
}
}
}
#endif
#if WASM_ENABLE_JIT != 0
else if (running_mode == Mode_LLVM_JIT) {
void **llvm_jit_func_ptrs;
uint32 i;
/* Notify backend threads to start llvm jit compilation */
module->enable_llvm_jit_compilation = true;
/* Wait until llvm jit finishes initialization */
os_mutex_lock(&module->tierup_wait_lock);
while (!module->llvm_jit_inited) {
os_cond_reltimedwait(&module->tierup_wait_cond,
&module->tierup_wait_lock, 10);
if (module->orcjit_stop_compiling) {
/* init_llvm_jit_functions_stage2 failed */
os_mutex_unlock(&module->tierup_wait_lock);
os_mutex_unlock(&module->instance_list_lock);
return false;
}
}
os_mutex_unlock(&module->tierup_wait_lock);
llvm_jit_func_ptrs =
module_inst->func_ptrs + module->import_function_count;
for (i = 0; i < module->function_count; i++) {
llvm_jit_func_ptrs[i] = module->functions[i]->llvm_jit_func_ptr;
}
}
#endif
else if (running_mode == Mode_Multi_Tier_JIT) {
/* Notify backend threads to start llvm jit compilation */
module->enable_llvm_jit_compilation = true;
/* Free fast_jit_func_ptrs if it is allocated before */
if (module_inst->fast_jit_func_ptrs
&& module_inst->fast_jit_func_ptrs != module->fast_jit_func_ptrs) {
wasm_runtime_free(module_inst->fast_jit_func_ptrs);
}
module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
/* Copy all llvm jit func ptrs from the module */
bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
sizeof(void *) * module->function_count, module->func_ptrs,
sizeof(void *) * module->function_count);
}
else {
bh_assert(0);
}
/* Add module instance into module's instance list if not added */
if (first_time_set) {
bool found = false;
WASMModuleInstance *node = module->instance_list;
while (node) {
if (node == module_inst) {
found = true;
break;
}
node = node->e->next;
}
if (!found) {
module_inst->e->next = module->instance_list;
module->instance_list = module_inst;
}
}
os_mutex_unlock(&module->instance_list_lock);
#endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0) */
(void)module;
return true;
}
bool
wasm_set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode)
{
return set_running_mode(module_inst, running_mode, false);
}
/**
* Instantiate module
*/
@ -1422,15 +1587,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
module_inst->e =
(WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset);
#if WASM_ENABLE_SHARED_MEMORY != 0
if (os_mutex_init(&module_inst->e->mem_lock) != 0) {
set_error_buf(error_buf, error_buf_size,
"create shared memory lock failed");
goto fail;
}
module_inst->e->mem_lock_inited = true;
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
module_inst->e->sub_module_inst_list =
&module_inst->e->sub_module_inst_list_head;
@ -1803,33 +1959,39 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
}
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
#if WASM_ENABLE_WASI_NN != 0
if (!is_sub_inst) {
if (!(module_inst->e->wasi_nn_ctx = wasi_nn_initialize())) {
set_error_buf(error_buf, error_buf_size,
"wasi nn initialization failed");
goto fail;
}
}
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
if (!is_sub_inst) {
/* Add module instance into module's instance list */
os_mutex_lock(&module->instance_list_lock);
#if WASM_ENABLE_DEBUG_INTERP != 0
if (module->instance_list) {
LOG_WARNING(
"warning: multiple instances referencing to the same module "
"may cause unexpected behaviour during debugging");
}
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* Copy llvm func ptrs again in case that they were updated
after the module instance was created */
bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
sizeof(void *) * module->function_count, module->func_ptrs,
sizeof(void *) * module->function_count);
#endif
module_inst->e->next = module->instance_list;
module->instance_list = module_inst;
os_mutex_unlock(&module->instance_list_lock);
}
#endif
/* Set running mode before executing wasm functions */
if (!set_running_mode(module_inst, wasm_runtime_get_default_running_mode(),
true)) {
set_error_buf(error_buf, error_buf_size,
"set instance running mode failed");
goto fail;
}
if (module->start_function != (uint32)-1) {
/* TODO: fix start function can be import function issue */
if (module->start_function >= module->import_function_count)
@ -1895,11 +2057,48 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
if (!module_inst)
return;
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
/* Remove instance from module's instance list before freeing
func_ptrs and fast_jit_func_ptrs of the instance, to avoid
accessing the freed memory in the jit backend compilation
threads */
if (!is_sub_inst) {
WASMModule *module = module_inst->module;
WASMModuleInstance *instance_prev = NULL, *instance;
os_mutex_lock(&module->instance_list_lock);
instance = module->instance_list;
while (instance) {
if (instance == module_inst) {
if (!instance_prev)
module->instance_list = instance->e->next;
else
instance_prev->e->next = instance->e->next;
break;
}
instance_prev = instance;
instance = instance->e->next;
}
os_mutex_unlock(&module->instance_list_lock);
}
#endif
#if WASM_ENABLE_JIT != 0
if (module_inst->func_ptrs)
wasm_runtime_free(module_inst->func_ptrs);
#endif
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
if (module_inst->fast_jit_func_ptrs
&& module_inst->fast_jit_func_ptrs
!= module_inst->module->fast_jit_func_ptrs)
wasm_runtime_free(module_inst->fast_jit_func_ptrs);
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
if (module_inst->func_type_indexes)
wasm_runtime_free(module_inst->func_type_indexes);
@ -1951,39 +2150,17 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
}
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
if (!is_sub_inst) {
WASMModule *module = module_inst->module;
WASMModuleInstance *instance_prev = NULL, *instance;
os_mutex_lock(&module->instance_list_lock);
instance = module->instance_list;
while (instance) {
if (instance == module_inst) {
if (!instance_prev)
module->instance_list = instance->e->next;
else
instance_prev->e->next = instance->e->next;
break;
}
instance_prev = instance;
instance = instance->e->next;
}
os_mutex_unlock(&module->instance_list_lock);
}
#endif
#if WASM_ENABLE_SHARED_MEMORY != 0
if (module_inst->e->mem_lock_inited)
os_mutex_destroy(&module_inst->e->mem_lock);
#endif
if (module_inst->e->c_api_func_imports)
wasm_runtime_free(module_inst->e->c_api_func_imports);
#if WASM_ENABLE_WASI_NN != 0
if (!is_sub_inst) {
WASINNContext *wasi_nn_ctx = module_inst->e->wasi_nn_ctx;
if (wasi_nn_ctx)
wasi_nn_destroy(wasi_nn_ctx);
}
#endif
wasm_runtime_free(module_inst);
}
@ -2056,6 +2233,7 @@ call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst,
/* Check native stack overflow firstly to ensure we have enough
native stack to run the following codes before actually calling
the aot function in invokeNative function. */
RECORD_STACK_USAGE(exec_env, (uint8 *)&exec_env_tls);
if ((uint8 *)&exec_env_tls < exec_env->native_stack_boundary
+ page_size * (guard_page_count + 1)) {
wasm_set_exception(module_inst, "native stack overflow");

31
core/iwasm/interpreter/wasm_runtime.h
View File

@ -11,6 +11,10 @@
#include "../common/wasm_runtime_common.h"
#include "../common/wasm_exec_env.h"
#if WASM_ENABLE_WASI_NN != 0
#include "../libraries/wasi-nn/src/wasi_nn_private.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
@ -59,9 +63,7 @@ typedef enum WASMExceptionID {
EXCE_AUX_STACK_UNDERFLOW,
EXCE_OUT_OF_BOUNDS_TABLE_ACCESS,
EXCE_OPERAND_STACK_OVERFLOW,
#if WASM_ENABLE_FAST_JIT != 0
EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC,
#endif
EXCE_ALREADY_THROWN,
EXCE_NUM,
} WASMExceptionID;
@ -219,12 +221,7 @@ typedef struct WASMModuleInstanceExtra {
WASMFunctionInstance *retain_function;
CApiFuncImport *c_api_func_imports;
#if WASM_ENABLE_SHARED_MEMORY != 0
/* lock for shared memory atomic operations */
korp_mutex mem_lock;
bool mem_lock_inited;
#endif
RunningMode running_mode;
#if WASM_ENABLE_MULTI_MODULE != 0
bh_list sub_module_inst_list_head;
@ -237,11 +234,15 @@ typedef struct WASMModuleInstanceExtra {
uint32 max_aux_stack_used;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT \
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)
WASMModuleInstance *next;
#endif
#if WASM_ENABLE_WASI_NN != 0
WASINNContext *wasi_nn_ctx;
#endif
} WASMModuleInstanceExtra;
struct AOTFuncPerfProfInfo;
@ -298,7 +299,11 @@ struct WASMModuleInstance {
not available in AOTModuleInstance */
DefPointer(void **, import_func_ptrs);
/* Array of function pointers to fast jit functions,
not available in AOTModuleInstance */
not available in AOTModuleInstance:
Only when the multi-tier JIT macros are all enabled and the running
mode of current module instance is set to Mode_Fast_JIT, runtime
will allocate new memory for it, otherwise it always points to the
module->fast_jit_func_ptrs */
DefPointer(void **, fast_jit_func_ptrs);
/* The custom data that can be set/get by wasm_{get|set}_custom_data */
DefPointer(void *, custom_data);
@ -402,6 +407,10 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst);
void
wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst);
bool
wasm_set_running_mode(WASMModuleInstance *module_inst,
RunningMode running_mode);
WASMFunctionInstance *
wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name,
const char *signature);