christoph/wamr
1
Fork 0
Code Releases Activity

Refactor interpreter/AOT module instance layout (#1559)

Browse Source 
Refactor the layout of interpreter and AOT module instance:
- Unify the interp/AOT module instance, use the same WASMModuleInstance/
  WASMMemoryInstance/WASMTableInstance data structures for both interpreter
  and AOT
- Make the offset of most fields the same in module instance for both interpreter
  and AOT, append memory instance structure, global data and table instances to
  the end of module instance for interpreter mode (like AOT mode)
- For extra fields in WASM module instance, use WASMModuleInstanceExtra to
  create a field `e` for interpreter
- Change the LLVM JIT module instance creating process, LLVM JIT uses the WASM
  module and module instance same as interpreter/Fast-JIT mode. So that Fast JIT
  and LLVM JIT can access the same data structures, and make it possible to
  implement the Multi-tier JIT (tier-up from Fast JIT to LLVM JIT) in the future
- Unify some APIs: merge some APIs for module instance and memory instance's
  related operations (only implement one copy)

Note that the AOT ABI is same, the AOT file format, AOT relocation types, how AOT
code accesses the AOT module instance and so on are kept unchanged.

Refer to:
https://github.com/bytecodealliance/wasm-micro-runtime/issues/1384
This commit is contained in:
Wenyong Huang
2022-10-18 10:59:28 +08:00
committed by GitHub
parent dc4dcc3d6f
commit a182926a73
49 changed files with 3790 additions and 3274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

444
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -641,28 +641,28 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint32)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint32)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = LOAD_I32(maddr); \
STORE_U32(maddr, readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
PUSH_I32(readv); \
break; \
@ -681,39 +681,39 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_U32(maddr); \
STORE_U32(maddr, (uint32)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else { \
uint64 op_result; \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_I64(maddr); \
op_result = readv op sval; \
STORE_I64(maddr, op_result); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
PUSH_I64(readv); \
break; \
@ -850,7 +850,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst,
wasm_exec_env_set_cur_frame(exec_env, frame);
cur_func_index = (uint32)(cur_func - module_inst->functions);
cur_func_index = (uint32)(cur_func - module_inst->e->functions);
bh_assert(cur_func_index < module_inst->module->import_function_count);
native_func_pointer = module_inst->import_func_ptrs[cur_func_index];
@ -904,12 +904,12 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst,
#if WASM_ENABLE_FAST_JIT != 0
bool
jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
WASMInterpFrame *prev_frame)
fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
WASMInterpFrame *prev_frame)
{
WASMModuleInstance *module_inst =
(WASMModuleInstance *)exec_env->module_inst;
WASMFunctionInstance *cur_func = module_inst->functions + func_idx;
WASMFunctionInstance *cur_func = module_inst->e->functions + func_idx;
wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame);
return wasm_get_exception(module_inst) ? false : true;
@ -1081,7 +1081,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
WASMFunctionInstance *cur_func,
WASMInterpFrame *prev_frame)
{
WASMMemoryInstance *memory = module->default_memory;
WASMMemoryInstance *memory = wasm_get_default_memory(module);
uint8 *global_data = module->global_data;
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
@ -1091,7 +1091,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
memory ? num_bytes_per_page * memory->cur_page_count : 0;
#endif
WASMType **wasm_types = module->module->types;
WASMGlobalInstance *globals = module->globals, *global;
WASMGlobalInstance *globals = module->e->globals, *global;
uint8 opcode_IMPDEP = WASM_OP_IMPDEP;
WASMInterpFrame *frame = NULL;
/* Points to this special opcode so as to jump to the
@ -1355,13 +1355,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
read_leb_uint32(frame_ip, frame_ip_end, fidx);
#if WASM_ENABLE_MULTI_MODULE != 0
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
#endif
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
goto call_func_from_interp;
}
@ -1373,12 +1373,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
read_leb_uint32(frame_ip, frame_ip_end, fidx);
#if WASM_ENABLE_MULTI_MODULE != 0
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
#endif
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
goto call_func_from_return_call;
}
@ -1420,8 +1420,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
fidx = ((uint32 *)tbl_inst->base_addr)[val];
if (fidx == (uint32)-1) {
fidx = tbl_inst->elems[val];
if (fidx == NULL_REF) {
wasm_set_exception(module, "uninitialized element");
goto got_exception;
}
@ -1431,13 +1431,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
* another module. In that case, we don't validate
* the elem value while loading
*/
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
/* always call module own functions */
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
if (cur_func->is_import_func)
cur_func_type = cur_func->u.func_import->func_type;
@ -1531,7 +1531,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]);
PUSH_I32(tbl_inst->elems[elem_idx]);
HANDLE_OP_END();
}
@ -1552,7 +1552,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
((uint32 *)(tbl_inst->base_addr))[elem_idx] = elem_val;
tbl_inst->elems[elem_idx] = elem_val;
HANDLE_OP_END();
}
@ -1700,7 +1700,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_GET_GLOBAL)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
PUSH_I32(*(uint32 *)global_addr);
@ -1710,7 +1710,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_GET_GLOBAL_64)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
PUSH_I64(GET_I64_FROM_ADDR((uint32 *)global_addr));
@ -1720,7 +1720,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_SET_GLOBAL)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
*(int32 *)global_addr = POP_I32();
@ -1732,7 +1732,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 aux_stack_top;
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
aux_stack_top = *(uint32 *)(frame_sp - 1);
@ -1751,8 +1751,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (module->module->aux_stack_top_global_index != (uint32)-1) {
uint32 aux_stack_used = module->module->aux_stack_bottom
- *(uint32 *)global_addr;
if (aux_stack_used > module->max_aux_stack_used)
module->max_aux_stack_used = aux_stack_used;
if (aux_stack_used > module->e->max_aux_stack_used)
module->e->max_aux_stack_used = aux_stack_used;
}
#endif
HANDLE_OP_END();
@ -1761,7 +1761,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_SET_GLOBAL_64)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64());
@ -2039,8 +2039,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
else {
/* success, return previous page count */
PUSH_I32(prev_page_count);
/* update memory instance ptr and memory size */
memory = module->default_memory;
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
@ -3195,8 +3195,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
bh_memcpy_s(
(uint8 *)(tbl_inst)
+ offsetof(WASMTableInstance, base_addr)
(uint8 *)tbl_inst
+ offsetof(WASMTableInstance, elems)
+ d * sizeof(uint32),
(uint32)((tbl_inst->cur_size - d) * sizeof(uint32)),
module->module->table_segments[elem_idx]
@ -3246,16 +3246,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
/* if s >= d, copy from front to back */
/* if s < d, copy from back to front */
/* merge all together */
bh_memmove_s(
(uint8 *)(dst_tbl_inst)
+ offsetof(WASMTableInstance, base_addr)
+ d * sizeof(uint32),
(uint32)((dst_tbl_inst->cur_size - d)
* sizeof(uint32)),
(uint8 *)(src_tbl_inst)
+ offsetof(WASMTableInstance, base_addr)
+ s * sizeof(uint32),
(uint32)(n * sizeof(uint32)));
bh_memmove_s((uint8 *)dst_tbl_inst
+ offsetof(WASMTableInstance, elems)
+ d * sizeof(uint32),
(uint32)((dst_tbl_inst->cur_size - d)
* sizeof(uint32)),
(uint8 *)src_tbl_inst
+ offsetof(WASMTableInstance, elems)
+ s * sizeof(uint32),
(uint32)(n * sizeof(uint32)));
break;
}
case WASM_OP_TABLE_GROW:
@ -3319,7 +3318,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
for (; n != 0; i++, n--) {
((uint32 *)(tbl_inst->base_addr))[i] = fill_val;
tbl_inst->elems[i] = fill_val;
}
break;
@ -3420,23 +3419,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)(*(uint8 *)maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)LOAD_U16(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I32(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I32(readv);
@ -3455,30 +3454,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)(*(uint8 *)maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U16(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U32(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I64(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I64(readv);
@ -3497,23 +3496,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U32(maddr, frame_sp[1]);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
break;
}
@ -3531,31 +3530,31 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE32) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U32(maddr, (uint32)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
PUT_I64_TO_ADDR((uint32 *)maddr,
GET_I64_FROM_ADDR(frame_sp + 1));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
break;
}
@ -3575,32 +3574,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint8)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint16)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I32(maddr);
if (readv == expect)
STORE_U32(maddr, sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I32(readv);
break;
@ -3621,44 +3620,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint8)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint16)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint32)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U32(maddr);
if (readv == expect)
STORE_U32(maddr, (uint32)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_I64(maddr);
if (readv == expect) {
STORE_I64(maddr, sval);
}
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I64(readv);
break;
@ -3794,8 +3793,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
cur_func = frame->function;
UPDATE_ALL_FROM_FRAME();
/* update memory instance ptr and memory size */
memory = module->default_memory;
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
@ -3905,6 +3904,191 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
}
#if WASM_ENABLE_FAST_JIT != 0
static void
fast_jit_call_func_bytecode(WASMExecEnv *exec_env,
WASMFunctionInstance *function,
WASMInterpFrame *frame)
{
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
JitInterpSwitchInfo info;
WASMType *func_type = function->u.func->func_type;
uint8 type = func_type->result_count
? func_type->types[func_type->param_count]
: VALUE_TYPE_VOID;
#if WASM_ENABLE_REF_TYPES != 0
if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
type = VALUE_TYPE_I32;
#endif
info.out.ret.last_return_type = type;
info.frame = frame;
frame->jitted_return_addr =
(uint8 *)jit_globals->return_to_interp_from_jitted;
jit_interp_switch_to_jitted(exec_env, &info,
function->u.func->fast_jit_jitted_code);
if (func_type->result_count) {
switch (type) {
case VALUE_TYPE_I32:
*(frame->sp - function->ret_cell_num) = info.out.ret.ival[0];
break;
case VALUE_TYPE_I64:
*(frame->sp - function->ret_cell_num) = info.out.ret.ival[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.ival[1];
break;
case VALUE_TYPE_F32:
*(frame->sp - function->ret_cell_num) = info.out.ret.fval[0];
break;
case VALUE_TYPE_F64:
*(frame->sp - function->ret_cell_num) = info.out.ret.fval[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.fval[1];
break;
default:
bh_assert(0);
break;
}
}
}
#endif
#if WASM_ENABLE_JIT != 0
static bool
clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst)
{
#if WASM_ENABLE_LIBC_WASI != 0
const char *exception = wasm_get_exception(module_inst);
if (exception && !strcmp(exception, "Exception: wasi proc exit")) {
/* The "wasi proc exit" exception is thrown by native lib to
let wasm app exit, which is a normal behavior, we clear
the exception here. */
wasm_set_exception(module_inst, NULL);
return true;
}
return false;
#else
return false;
#endif
}
static bool
llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
WASMExecEnv *exec_env,
WASMFunctionInstance *function, uint32 argc,
uint32 argv[])
{
WASMType *func_type = function->u.func->func_type;
uint32 result_count = func_type->result_count;
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
bool ret;
#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) {
wasm_set_exception(module_inst, "wasm operand stack overflow");
}
#endif
if (ext_ret_count > 0) {
uint32 cell_num = 0, i;
uint8 *ext_ret_types = func_type->types + func_type->param_count + 1;
uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL;
uint32 *argv_ret = argv;
uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count);
uint64 size;
/* Allocate memory all arguments */
size =
sizeof(uint32) * (uint64)argc /* original arguments */
+ sizeof(void *)
* (uint64)ext_ret_count /* extra result values' addr */
+ sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */
if (size > sizeof(argv1_buf)) {
if (size > UINT32_MAX
|| !(argv1 = wasm_runtime_malloc((uint32)size))) {
wasm_set_exception(module_inst, "allocate memory failed");
return false;
}
}
/* Copy original arguments */
bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc);
/* Get the extra result value's address */
ext_rets =
argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
/* Append each extra result value's address to original arguments */
for (i = 0; i < ext_ret_count; i++) {
*(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) =
(uintptr_t)(ext_rets + cell_num);
cell_num += wasm_value_type_cell_num(ext_ret_types[i]);
}
ret = wasm_runtime_invoke_native(
exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL,
NULL, argv1, argc, argv);
if (!ret || wasm_get_exception(module_inst)) {
if (clear_wasi_proc_exit_exception(module_inst))
ret = true;
else
ret = false;
}
if (!ret) {
if (argv1 != argv1_buf)
wasm_runtime_free(argv1);
return ret;
}
/* Get extra result values */
switch (func_type->types[func_type->param_count]) {
case VALUE_TYPE_I32:
case VALUE_TYPE_F32:
#if WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
#endif
argv_ret++;
break;
case VALUE_TYPE_I64:
case VALUE_TYPE_F64:
argv_ret += 2;
break;
#if WASM_ENABLE_SIMD != 0
case VALUE_TYPE_V128:
argv_ret += 4;
break;
#endif
default:
bh_assert(0);
break;
}
ext_rets =
argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets,
sizeof(uint32) * cell_num);
if (argv1 != argv1_buf)
wasm_runtime_free(argv1);
return true;
}
else {
ret = wasm_runtime_invoke_native(
exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL,
NULL, argv, argc, argv);
if (clear_wasi_proc_exit_exception(module_inst))
ret = true;
return ret && !wasm_get_exception(module_inst) ? true : false;
}
}
#endif
void
wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
WASMFunctionInstance *function, uint32 argc,
@ -3912,14 +4096,14 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
{
WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env);
WASMInterpFrame *frame, *outs_area;
/* Allocate sufficient cells for all kinds of return values. */
unsigned all_cell_num =
function->ret_cell_num > 2 ? function->ret_cell_num : 2,
i;
function->ret_cell_num > 2 ? function->ret_cell_num : 2;
/* This frame won't be used by JITed code, so only allocate interp
frame here. */
unsigned frame_size = wasm_interp_interp_frame_size(all_cell_num);
unsigned i;
bool copy_argv_from_frame = true;
if (argc < function->param_cell_num) {
char buf[128];
@ -3969,62 +4153,26 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
}
else {
#if WASM_ENABLE_FAST_JIT == 0
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#if WASM_ENABLE_JIT != 0
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
fast_jit_call_func_bytecode(exec_env, function, frame);
#else
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
JitInterpSwitchInfo info;
WASMType *func_type = function->u.func->func_type;
uint8 type = func_type->result_count
? func_type->types[func_type->param_count]
: VALUE_TYPE_VOID;
#if WASM_ENABLE_REF_TYPES != 0
if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
type = VALUE_TYPE_I32;
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
info.out.ret.last_return_type = type;
info.frame = frame;
frame->jitted_return_addr =
(uint8 *)jit_globals->return_to_interp_from_jitted;
jit_interp_switch_to_jitted(exec_env, &info,
function->u.func->fast_jit_jitted_code);
if (func_type->result_count) {
switch (type) {
case VALUE_TYPE_I32:
*(frame->sp - function->ret_cell_num) =
info.out.ret.ival[0];
break;
case VALUE_TYPE_I64:
*(frame->sp - function->ret_cell_num) =
info.out.ret.ival[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.ival[1];
break;
case VALUE_TYPE_F32:
*(frame->sp - function->ret_cell_num) =
info.out.ret.fval[0];
break;
case VALUE_TYPE_F64:
*(frame->sp - function->ret_cell_num) =
info.out.ret.fval[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.fval[1];
break;
default:
bh_assert(0);
break;
}
}
(void)wasm_interp_call_func_bytecode;
#endif
}
/* Output the return value to the caller */
if (!wasm_get_exception(module_inst)) {
for (i = 0; i < function->ret_cell_num; i++) {
argv[i] = *(frame->sp + i - function->ret_cell_num);
if (copy_argv_from_frame) {
for (i = 0; i < function->ret_cell_num; i++) {
argv[i] = *(frame->sp + i - function->ret_cell_num);
}
}
}
else {