christoph/wamr
1
Fork 0
Code Releases Activity

Refactor interpreter/AOT module instance layout (#1559)

Browse Source 
Refactor the layout of interpreter and AOT module instance:
- Unify the interp/AOT module instance, use the same WASMModuleInstance/
  WASMMemoryInstance/WASMTableInstance data structures for both interpreter
  and AOT
- Make the offset of most fields the same in module instance for both interpreter
  and AOT, append memory instance structure, global data and table instances to
  the end of module instance for interpreter mode (like AOT mode)
- For extra fields in WASM module instance, use WASMModuleInstanceExtra to
  create a field `e` for interpreter
- Change the LLVM JIT module instance creating process, LLVM JIT uses the WASM
  module and module instance same as interpreter/Fast-JIT mode. So that Fast JIT
  and LLVM JIT can access the same data structures, and make it possible to
  implement the Multi-tier JIT (tier-up from Fast JIT to LLVM JIT) in the future
- Unify some APIs: merge some APIs for module instance and memory instance's
  related operations (only implement one copy)

Note that the AOT ABI is same, the AOT file format, AOT relocation types, how AOT
code accesses the AOT module instance and so on are kept unchanged.

Refer to:
https://github.com/bytecodealliance/wasm-micro-runtime/issues/1384
This commit is contained in:
Wenyong Huang
2022-10-18 10:59:28 +08:00
committed by GitHub
parent dc4dcc3d6f
commit a182926a73
49 changed files with 3790 additions and 3274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
core/iwasm/interpreter/wasm.h
View File

@ -263,6 +263,9 @@ struct WASMFunction {
#if WASM_ENABLE_FAST_JIT != 0
void *fast_jit_jitted_code;
#endif
#if WASM_ENABLE_JIT != 0
void *llvm_jit_func_ptr;
#endif
};
struct WASMGlobal {
@ -363,6 +366,11 @@ typedef struct WASMCustomSection {
} WASMCustomSection;
#endif
#if WASM_ENABLE_JIT != 0
struct AOTCompData;
struct AOTCompContext;
#endif
struct WASMModule {
/* Module type, for module loaded from WASM bytecode binary,
this field is Wasm_Module_Bytecode;
@ -407,6 +415,9 @@ struct WASMModule {
WASMDataSeg **data_segments;
uint32 start_function;
/* total global variable size */
uint32 global_data_size;
/* the index of auxiliary __data_end global,
-1 means unexported */
uint32 aux_data_end_global_index;
@ -493,6 +504,12 @@ struct WASMModule {
/* point to JITed functions */
void **fast_jit_func_ptrs;
#endif
#if WASM_ENABLE_JIT != 0
struct AOTCompData *comp_data;
struct AOTCompContext *comp_ctx;
void **func_ptrs;
#endif
};
typedef struct BlockType {

444
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -641,28 +641,28 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint32)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint32)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = LOAD_I32(maddr); \
STORE_U32(maddr, readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
PUSH_I32(readv); \
break; \
@ -681,39 +681,39 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_U32(maddr); \
STORE_U32(maddr, (uint32)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else { \
uint64 op_result; \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_I64(maddr); \
op_result = readv op sval; \
STORE_I64(maddr, op_result); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
PUSH_I64(readv); \
break; \
@ -850,7 +850,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst,
wasm_exec_env_set_cur_frame(exec_env, frame);
cur_func_index = (uint32)(cur_func - module_inst->functions);
cur_func_index = (uint32)(cur_func - module_inst->e->functions);
bh_assert(cur_func_index < module_inst->module->import_function_count);
native_func_pointer = module_inst->import_func_ptrs[cur_func_index];
@ -904,12 +904,12 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst,
#if WASM_ENABLE_FAST_JIT != 0
bool
jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
WASMInterpFrame *prev_frame)
fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
WASMInterpFrame *prev_frame)
{
WASMModuleInstance *module_inst =
(WASMModuleInstance *)exec_env->module_inst;
WASMFunctionInstance *cur_func = module_inst->functions + func_idx;
WASMFunctionInstance *cur_func = module_inst->e->functions + func_idx;
wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame);
return wasm_get_exception(module_inst) ? false : true;
@ -1081,7 +1081,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
WASMFunctionInstance *cur_func,
WASMInterpFrame *prev_frame)
{
WASMMemoryInstance *memory = module->default_memory;
WASMMemoryInstance *memory = wasm_get_default_memory(module);
uint8 *global_data = module->global_data;
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
@ -1091,7 +1091,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
memory ? num_bytes_per_page * memory->cur_page_count : 0;
#endif
WASMType **wasm_types = module->module->types;
WASMGlobalInstance *globals = module->globals, *global;
WASMGlobalInstance *globals = module->e->globals, *global;
uint8 opcode_IMPDEP = WASM_OP_IMPDEP;
WASMInterpFrame *frame = NULL;
/* Points to this special opcode so as to jump to the
@ -1355,13 +1355,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
read_leb_uint32(frame_ip, frame_ip_end, fidx);
#if WASM_ENABLE_MULTI_MODULE != 0
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
#endif
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
goto call_func_from_interp;
}
@ -1373,12 +1373,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
read_leb_uint32(frame_ip, frame_ip_end, fidx);
#if WASM_ENABLE_MULTI_MODULE != 0
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
#endif
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
goto call_func_from_return_call;
}
@ -1420,8 +1420,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
fidx = ((uint32 *)tbl_inst->base_addr)[val];
if (fidx == (uint32)-1) {
fidx = tbl_inst->elems[val];
if (fidx == NULL_REF) {
wasm_set_exception(module, "uninitialized element");
goto got_exception;
}
@ -1431,13 +1431,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
* another module. In that case, we don't validate
* the elem value while loading
*/
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
/* always call module own functions */
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
if (cur_func->is_import_func)
cur_func_type = cur_func->u.func_import->func_type;
@ -1531,7 +1531,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]);
PUSH_I32(tbl_inst->elems[elem_idx]);
HANDLE_OP_END();
}
@ -1552,7 +1552,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
((uint32 *)(tbl_inst->base_addr))[elem_idx] = elem_val;
tbl_inst->elems[elem_idx] = elem_val;
HANDLE_OP_END();
}
@ -1700,7 +1700,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_GET_GLOBAL)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
PUSH_I32(*(uint32 *)global_addr);
@ -1710,7 +1710,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_GET_GLOBAL_64)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
PUSH_I64(GET_I64_FROM_ADDR((uint32 *)global_addr));
@ -1720,7 +1720,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_SET_GLOBAL)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
*(int32 *)global_addr = POP_I32();
@ -1732,7 +1732,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 aux_stack_top;
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
aux_stack_top = *(uint32 *)(frame_sp - 1);
@ -1751,8 +1751,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (module->module->aux_stack_top_global_index != (uint32)-1) {
uint32 aux_stack_used = module->module->aux_stack_bottom
- *(uint32 *)global_addr;
if (aux_stack_used > module->max_aux_stack_used)
module->max_aux_stack_used = aux_stack_used;
if (aux_stack_used > module->e->max_aux_stack_used)
module->e->max_aux_stack_used = aux_stack_used;
}
#endif
HANDLE_OP_END();
@ -1761,7 +1761,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_SET_GLOBAL_64)
{
read_leb_uint32(frame_ip, frame_ip_end, global_idx);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64());
@ -2039,8 +2039,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
else {
/* success, return previous page count */
PUSH_I32(prev_page_count);
/* update memory instance ptr and memory size */
memory = module->default_memory;
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
@ -3195,8 +3195,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
bh_memcpy_s(
(uint8 *)(tbl_inst)
+ offsetof(WASMTableInstance, base_addr)
(uint8 *)tbl_inst
+ offsetof(WASMTableInstance, elems)
+ d * sizeof(uint32),
(uint32)((tbl_inst->cur_size - d) * sizeof(uint32)),
module->module->table_segments[elem_idx]
@ -3246,16 +3246,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
/* if s >= d, copy from front to back */
/* if s < d, copy from back to front */
/* merge all together */
bh_memmove_s(
(uint8 *)(dst_tbl_inst)
+ offsetof(WASMTableInstance, base_addr)
+ d * sizeof(uint32),
(uint32)((dst_tbl_inst->cur_size - d)
* sizeof(uint32)),
(uint8 *)(src_tbl_inst)
+ offsetof(WASMTableInstance, base_addr)
+ s * sizeof(uint32),
(uint32)(n * sizeof(uint32)));
bh_memmove_s((uint8 *)dst_tbl_inst
+ offsetof(WASMTableInstance, elems)
+ d * sizeof(uint32),
(uint32)((dst_tbl_inst->cur_size - d)
* sizeof(uint32)),
(uint8 *)src_tbl_inst
+ offsetof(WASMTableInstance, elems)
+ s * sizeof(uint32),
(uint32)(n * sizeof(uint32)));
break;
}
case WASM_OP_TABLE_GROW:
@ -3319,7 +3318,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
for (; n != 0; i++, n--) {
((uint32 *)(tbl_inst->base_addr))[i] = fill_val;
tbl_inst->elems[i] = fill_val;
}
break;
@ -3420,23 +3419,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)(*(uint8 *)maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)LOAD_U16(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I32(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I32(readv);
@ -3455,30 +3454,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)(*(uint8 *)maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U16(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U32(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I64(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I64(readv);
@ -3497,23 +3496,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U32(maddr, frame_sp[1]);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
break;
}
@ -3531,31 +3530,31 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE32) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U32(maddr, (uint32)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
PUT_I64_TO_ADDR((uint32 *)maddr,
GET_I64_FROM_ADDR(frame_sp + 1));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
break;
}
@ -3575,32 +3574,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint8)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint16)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I32(maddr);
if (readv == expect)
STORE_U32(maddr, sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I32(readv);
break;
@ -3621,44 +3620,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint8)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint16)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
expect = (uint32)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U32(maddr);
if (readv == expect)
STORE_U32(maddr, (uint32)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS();
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_I64(maddr);
if (readv == expect) {
STORE_I64(maddr, sval);
}
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I64(readv);
break;
@ -3794,8 +3793,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
cur_func = frame->function;
UPDATE_ALL_FROM_FRAME();
/* update memory instance ptr and memory size */
memory = module->default_memory;
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
@ -3905,6 +3904,191 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
}
#if WASM_ENABLE_FAST_JIT != 0
static void
fast_jit_call_func_bytecode(WASMExecEnv *exec_env,
WASMFunctionInstance *function,
WASMInterpFrame *frame)
{
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
JitInterpSwitchInfo info;
WASMType *func_type = function->u.func->func_type;
uint8 type = func_type->result_count
? func_type->types[func_type->param_count]
: VALUE_TYPE_VOID;
#if WASM_ENABLE_REF_TYPES != 0
if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
type = VALUE_TYPE_I32;
#endif
info.out.ret.last_return_type = type;
info.frame = frame;
frame->jitted_return_addr =
(uint8 *)jit_globals->return_to_interp_from_jitted;
jit_interp_switch_to_jitted(exec_env, &info,
function->u.func->fast_jit_jitted_code);
if (func_type->result_count) {
switch (type) {
case VALUE_TYPE_I32:
*(frame->sp - function->ret_cell_num) = info.out.ret.ival[0];
break;
case VALUE_TYPE_I64:
*(frame->sp - function->ret_cell_num) = info.out.ret.ival[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.ival[1];
break;
case VALUE_TYPE_F32:
*(frame->sp - function->ret_cell_num) = info.out.ret.fval[0];
break;
case VALUE_TYPE_F64:
*(frame->sp - function->ret_cell_num) = info.out.ret.fval[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.fval[1];
break;
default:
bh_assert(0);
break;
}
}
}
#endif
#if WASM_ENABLE_JIT != 0
static bool
clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst)
{
#if WASM_ENABLE_LIBC_WASI != 0
const char *exception = wasm_get_exception(module_inst);
if (exception && !strcmp(exception, "Exception: wasi proc exit")) {
/* The "wasi proc exit" exception is thrown by native lib to
let wasm app exit, which is a normal behavior, we clear
the exception here. */
wasm_set_exception(module_inst, NULL);
return true;
}
return false;
#else
return false;
#endif
}
static bool
llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
WASMExecEnv *exec_env,
WASMFunctionInstance *function, uint32 argc,
uint32 argv[])
{
WASMType *func_type = function->u.func->func_type;
uint32 result_count = func_type->result_count;
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
bool ret;
#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) {
wasm_set_exception(module_inst, "wasm operand stack overflow");
}
#endif
if (ext_ret_count > 0) {
uint32 cell_num = 0, i;
uint8 *ext_ret_types = func_type->types + func_type->param_count + 1;
uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL;
uint32 *argv_ret = argv;
uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count);
uint64 size;
/* Allocate memory all arguments */
size =
sizeof(uint32) * (uint64)argc /* original arguments */
+ sizeof(void *)
* (uint64)ext_ret_count /* extra result values' addr */
+ sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */
if (size > sizeof(argv1_buf)) {
if (size > UINT32_MAX
|| !(argv1 = wasm_runtime_malloc((uint32)size))) {
wasm_set_exception(module_inst, "allocate memory failed");
return false;
}
}
/* Copy original arguments */
bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc);
/* Get the extra result value's address */
ext_rets =
argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
/* Append each extra result value's address to original arguments */
for (i = 0; i < ext_ret_count; i++) {
*(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) =
(uintptr_t)(ext_rets + cell_num);
cell_num += wasm_value_type_cell_num(ext_ret_types[i]);
}
ret = wasm_runtime_invoke_native(
exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL,
NULL, argv1, argc, argv);
if (!ret || wasm_get_exception(module_inst)) {
if (clear_wasi_proc_exit_exception(module_inst))
ret = true;
else
ret = false;
}
if (!ret) {
if (argv1 != argv1_buf)
wasm_runtime_free(argv1);
return ret;
}
/* Get extra result values */
switch (func_type->types[func_type->param_count]) {
case VALUE_TYPE_I32:
case VALUE_TYPE_F32:
#if WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
#endif
argv_ret++;
break;
case VALUE_TYPE_I64:
case VALUE_TYPE_F64:
argv_ret += 2;
break;
#if WASM_ENABLE_SIMD != 0
case VALUE_TYPE_V128:
argv_ret += 4;
break;
#endif
default:
bh_assert(0);
break;
}
ext_rets =
argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets,
sizeof(uint32) * cell_num);
if (argv1 != argv1_buf)
wasm_runtime_free(argv1);
return true;
}
else {
ret = wasm_runtime_invoke_native(
exec_env, function->u.func->llvm_jit_func_ptr, func_type, NULL,
NULL, argv, argc, argv);
if (clear_wasi_proc_exit_exception(module_inst))
ret = true;
return ret && !wasm_get_exception(module_inst) ? true : false;
}
}
#endif
void
wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
WASMFunctionInstance *function, uint32 argc,
@ -3912,14 +4096,14 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
{
WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env);
WASMInterpFrame *frame, *outs_area;
/* Allocate sufficient cells for all kinds of return values. */
unsigned all_cell_num =
function->ret_cell_num > 2 ? function->ret_cell_num : 2,
i;
function->ret_cell_num > 2 ? function->ret_cell_num : 2;
/* This frame won't be used by JITed code, so only allocate interp
frame here. */
unsigned frame_size = wasm_interp_interp_frame_size(all_cell_num);
unsigned i;
bool copy_argv_from_frame = true;
if (argc < function->param_cell_num) {
char buf[128];
@ -3969,62 +4153,26 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
}
else {
#if WASM_ENABLE_FAST_JIT == 0
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#if WASM_ENABLE_JIT != 0
llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
argv);
/* For llvm jit, the results have been stored in argv,
no need to copy them from stack frame again */
copy_argv_from_frame = false;
#elif WASM_ENABLE_FAST_JIT != 0
fast_jit_call_func_bytecode(exec_env, function, frame);
#else
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
JitInterpSwitchInfo info;
WASMType *func_type = function->u.func->func_type;
uint8 type = func_type->result_count
? func_type->types[func_type->param_count]
: VALUE_TYPE_VOID;
#if WASM_ENABLE_REF_TYPES != 0
if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
type = VALUE_TYPE_I32;
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#endif
info.out.ret.last_return_type = type;
info.frame = frame;
frame->jitted_return_addr =
(uint8 *)jit_globals->return_to_interp_from_jitted;
jit_interp_switch_to_jitted(exec_env, &info,
function->u.func->fast_jit_jitted_code);
if (func_type->result_count) {
switch (type) {
case VALUE_TYPE_I32:
*(frame->sp - function->ret_cell_num) =
info.out.ret.ival[0];
break;
case VALUE_TYPE_I64:
*(frame->sp - function->ret_cell_num) =
info.out.ret.ival[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.ival[1];
break;
case VALUE_TYPE_F32:
*(frame->sp - function->ret_cell_num) =
info.out.ret.fval[0];
break;
case VALUE_TYPE_F64:
*(frame->sp - function->ret_cell_num) =
info.out.ret.fval[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.fval[1];
break;
default:
bh_assert(0);
break;
}
}
(void)wasm_interp_call_func_bytecode;
#endif
}
/* Output the return value to the caller */
if (!wasm_get_exception(module_inst)) {
for (i = 0; i < function->ret_cell_num; i++) {
argv[i] = *(frame->sp + i - function->ret_cell_num);
if (copy_argv_from_frame) {
for (i = 0; i < function->ret_cell_num; i++) {
argv[i] = *(frame->sp + i - function->ret_cell_num);
}
}
}
else {

184
core/iwasm/interpreter/wasm_interp_fast.c
View File

@ -443,28 +443,28 @@ LOAD_PTR(void *addr)
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(1); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint32)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(2); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint32)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(4); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = LOAD_I32(maddr); \
STORE_U32(maddr, readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
PUSH_I32(readv); \
break; \
@ -483,39 +483,39 @@ LOAD_PTR(void *addr)
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(1); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)(*(uint8 *)maddr); \
*(uint8 *)maddr = (uint8)(readv op sval); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(2); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_U16(maddr); \
STORE_U16(maddr, (uint16)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(4); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_U32(maddr); \
STORE_U32(maddr, (uint32)(readv op sval)); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
else { \
uint64 op_result; \
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \
CHECK_ATOMIC_MEMORY_ACCESS(8); \
\
os_mutex_lock(&memory->mem_lock); \
os_mutex_lock(&module->e->mem_lock); \
readv = (uint64)LOAD_I64(maddr); \
op_result = readv op sval; \
STORE_I64(maddr, op_result); \
os_mutex_unlock(&memory->mem_lock); \
os_mutex_unlock(&module->e->mem_lock); \
} \
PUSH_I64(readv); \
break; \
@ -913,7 +913,7 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst,
wasm_exec_env_set_cur_frame(exec_env, frame);
cur_func_index = (uint32)(cur_func - module_inst->functions);
cur_func_index = (uint32)(cur_func - module_inst->e->functions);
bh_assert(cur_func_index < module_inst->module->import_function_count);
native_func_pointer = module_inst->import_func_ptrs[cur_func_index];
@ -1138,7 +1138,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
WASMFunctionInstance *cur_func,
WASMInterpFrame *prev_frame)
{
WASMMemoryInstance *memory = module->default_memory;
WASMMemoryInstance *memory = wasm_get_default_memory(module);
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
@ -1147,7 +1147,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
memory ? num_bytes_per_page * memory->cur_page_count : 0;
#endif
uint8 *global_data = module->global_data;
WASMGlobalInstance *globals = module->globals, *global;
WASMGlobalInstance *globals = module->e ? module->e->globals : NULL;
WASMGlobalInstance *global;
uint8 opcode_IMPDEP = WASM_OP_IMPDEP;
WASMInterpFrame *frame = NULL;
/* Points to this special opcode so as to jump to the
@ -1345,8 +1346,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
fidx = ((uint32 *)tbl_inst->base_addr)[val];
if (fidx == (uint32)-1) {
fidx = tbl_inst->elems[val];
if (fidx == NULL_REF) {
wasm_set_exception(module, "uninitialized element");
goto got_exception;
}
@ -1356,13 +1357,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
* another module. in that case, we don't validate
* the elem value while loading
*/
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
/* always call module own functions */
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
if (cur_func->is_import_func)
cur_func_type = cur_func->u.func_import->func_type;
@ -1437,7 +1438,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
PUSH_I32(((uint32 *)tbl_inst->base_addr)[elem_idx]);
PUSH_I32(tbl_inst->elems[elem_idx]);
HANDLE_OP_END();
}
@ -1458,7 +1459,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
((uint32 *)tbl_inst->base_addr)[elem_idx] = elem_val;
tbl_inst->elems[elem_idx] = elem_val;
HANDLE_OP_END();
}
@ -1522,7 +1523,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_GET_GLOBAL)
{
global_idx = read_uint32(frame_ip);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
addr_ret = GET_OFFSET();
@ -1533,7 +1534,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_GET_GLOBAL_64)
{
global_idx = read_uint32(frame_ip);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
addr_ret = GET_OFFSET();
@ -1545,7 +1546,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_SET_GLOBAL)
{
global_idx = read_uint32(frame_ip);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
addr1 = GET_OFFSET();
@ -1558,7 +1559,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 aux_stack_top;
global_idx = read_uint32(frame_ip);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
aux_stack_top = frame_lp[GET_OFFSET()];
@ -1576,8 +1577,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (module->module->aux_stack_top_global_index != (uint32)-1) {
uint32 aux_stack_used = module->module->aux_stack_bottom
- *(uint32 *)global_addr;
if (aux_stack_used > module->max_aux_stack_used)
module->max_aux_stack_used = aux_stack_used;
if (aux_stack_used > module->e->max_aux_stack_used)
module->e->max_aux_stack_used = aux_stack_used;
}
#endif
HANDLE_OP_END();
@ -1586,7 +1587,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_SET_GLOBAL_64)
{
global_idx = read_uint32(frame_ip);
bh_assert(global_idx < module->global_count);
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
addr1 = GET_OFFSET();
@ -1860,8 +1861,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
else {
/* success, return previous page count */
frame_lp[addr_ret] = prev_page_count;
/* update memory instance ptr and memory size */
memory = module->default_memory;
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
@ -3093,7 +3094,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
bh_memcpy_s(
(uint8 *)tbl_inst
+ offsetof(WASMTableInstance, base_addr)
+ offsetof(WASMTableInstance, elems)
+ d * sizeof(uint32),
(uint32)((tbl_inst->cur_size - d) * sizeof(uint32)),
module->module->table_segments[elem_idx]
@ -3141,16 +3142,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
/* if s >= d, copy from front to back */
/* if s < d, copy from back to front */
/* merge all together */
bh_memmove_s(
(uint8 *)dst_tbl_inst
+ offsetof(WASMTableInstance, base_addr)
+ d * sizeof(uint32),
(uint32)((dst_tbl_inst->cur_size - d)
* sizeof(uint32)),
(uint8 *)src_tbl_inst
+ offsetof(WASMTableInstance, base_addr)
+ s * sizeof(uint32),
(uint32)(n * sizeof(uint32)));
bh_memmove_s((uint8 *)dst_tbl_inst
+ offsetof(WASMTableInstance, elems)
+ d * sizeof(uint32),
(uint32)((dst_tbl_inst->cur_size - d)
* sizeof(uint32)),
(uint8 *)src_tbl_inst
+ offsetof(WASMTableInstance, elems)
+ s * sizeof(uint32),
(uint32)(n * sizeof(uint32)));
break;
}
case WASM_OP_TABLE_GROW:
@ -3211,7 +3211,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
for (; n != 0; i++, n--) {
((uint32 *)(tbl_inst->base_addr))[i] = fill_val;
tbl_inst->elems[i] = fill_val;
}
break;
@ -3305,23 +3305,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)(*(uint8 *)maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)LOAD_U16(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I32(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I32(readv);
@ -3340,30 +3340,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)(*(uint8 *)maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U16(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U32(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(8);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I64(maddr);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I64(readv);
@ -3381,23 +3381,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I32_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U32(maddr, sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
break;
}
@ -3415,30 +3415,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(1);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
*(uint8 *)maddr = (uint8)sval;
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE16) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U16(maddr, (uint16)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_I64_STORE32) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_U32(maddr, (uint32)sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(8);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
STORE_I64(maddr, sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
break;
}
@ -3458,32 +3458,32 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS(1);
expect = (uint8)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
expect = (uint16)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint32)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = LOAD_I32(maddr);
if (readv == expect)
STORE_U32(maddr, sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I32(readv);
break;
@ -3504,44 +3504,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_ATOMIC_MEMORY_ACCESS(1);
expect = (uint8)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)(*(uint8 *)maddr);
if (readv == expect)
*(uint8 *)maddr = (uint8)(sval);
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(2);
expect = (uint16)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U16(maddr);
if (readv == expect)
STORE_U16(maddr, (uint16)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(4);
expect = (uint32)expect;
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_U32(maddr);
if (readv == expect)
STORE_U32(maddr, (uint32)(sval));
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
else {
CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr);
CHECK_ATOMIC_MEMORY_ACCESS(8);
os_mutex_lock(&memory->mem_lock);
os_mutex_lock(&module->e->mem_lock);
readv = (uint64)LOAD_I64(maddr);
if (readv == expect) {
STORE_I64(maddr, sval);
}
os_mutex_unlock(&memory->mem_lock);
os_mutex_unlock(&module->e->mem_lock);
}
PUSH_I64(readv);
break;
@ -3575,12 +3575,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
fidx = read_uint32(frame_ip);
#if WASM_ENABLE_MULTI_MODULE != 0
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
#endif
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
goto call_func_from_interp;
}
@ -3592,12 +3592,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#endif
fidx = read_uint32(frame_ip);
#if WASM_ENABLE_MULTI_MODULE != 0
if (fidx >= module->function_count) {
if (fidx >= module->e->function_count) {
wasm_set_exception(module, "unknown function");
goto got_exception;
}
#endif
cur_func = module->functions + fidx;
cur_func = module->e->functions + fidx;
goto call_func_from_return_call;
}
#endif /* WASM_ENABLE_TAIL_CALL */
@ -3782,8 +3782,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
cur_func = frame->function;
UPDATE_ALL_FROM_FRAME();
/* update memory instance ptr and memory size */
memory = module->default_memory;
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0

134
core/iwasm/interpreter/wasm_loader.c
View File

@ -18,6 +18,9 @@
#include "../fast-jit/jit_compiler.h"
#include "../fast-jit/jit_codecache.h"
#endif
#if WASM_ENABLE_JIT != 0
#include "../compilation/aot_llvm.h"
#endif
/* Read a value of given type from the address pointed to by the given
pointer and increase the pointer to the position just after the
@ -2923,7 +2926,6 @@ fail:
return false;
}
#if WASM_ENABLE_FAST_JIT != 0
static void
calculate_global_data_offset(WASMModule *module)
{
@ -2933,17 +2935,127 @@ calculate_global_data_offset(WASMModule *module)
for (i = 0; i < module->import_global_count; i++) {
WASMGlobalImport *import_global =
&((module->import_globals + i)->u.global);
#if WASM_ENABLE_FAST_JIT != 0
import_global->data_offset = data_offset;
#endif
data_offset += wasm_value_type_size(import_global->type);
}
for (i = 0; i < module->global_count; i++) {
WASMGlobal *global = module->globals + i;
#if WASM_ENABLE_FAST_JIT != 0
global->data_offset = data_offset;
#endif
data_offset += wasm_value_type_size(global->type);
}
module->global_data_size = data_offset;
}
#if WASM_ENABLE_JIT != 0
static bool
compile_llvm_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
AOTCompOption option = { 0 };
char func_name[32], *aot_last_error;
uint64 size;
uint32 i;
size = sizeof(void *) * (uint64)module->function_count;
if (size > 0
&& !(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->comp_data = aot_create_comp_data(module);
if (!module->comp_data) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
set_error_buf(error_buf, error_buf_size, aot_last_error);
return false;
}
option.is_jit_mode = true;
option.opt_level = 3;
option.size_level = 3;
#if WASM_ENABLE_BULK_MEMORY != 0
option.enable_bulk_memory = true;
#endif
#if WASM_ENABLE_THREAD_MGR != 0
option.enable_thread_mgr = true;
#endif
#if WASM_ENABLE_TAIL_CALL != 0
option.enable_tail_call = true;
#endif
#if WASM_ENABLE_SIMD != 0
option.enable_simd = true;
#endif
#if WASM_ENABLE_REF_TYPES != 0
option.enable_ref_types = true;
#endif
option.enable_aux_stack_check = true;
#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0)
option.enable_aux_stack_frame = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
set_error_buf(error_buf, error_buf_size, aot_last_error);
return false;
}
if (!aot_compile_wasm(module->comp_ctx)) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
set_error_buf(error_buf, error_buf_size, aot_last_error);
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
for (i = 0; i < module->comp_data->func_count; i++) {
LLVMErrorRef error;
LLVMOrcJITTargetAddress func_addr = 0;
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit,
&func_addr, func_name))) {
char *err_msg = LLVMGetErrorMessage(error);
set_error_buf_v(error_buf, error_buf_size,
"failed to compile orc jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
return false;
}
/**
* No need to lock the func_ptr[func_idx] here as it is basic
* data type, the load/store for it can be finished by one cpu
* instruction, and there can be only one cpu instruction
* loading/storing at the same time.
*/
module->func_ptrs[i] = (void *)func_addr;
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
}
#else
/* Resolve function addresses */
bh_assert(module->comp_ctx->exec_engine);
for (i = 0; i < module->comp_data->func_count; i++) {
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress(
module->comp_ctx->exec_engine, func_name))) {
set_error_buf(error_buf, error_buf_size,
"failed to compile llvm mc jit function");
return false;
}
module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i];
}
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
static bool
wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
@ -3352,9 +3464,9 @@ load_from_sections(WASMModule *module, WASMSection *sections,
#endif
}
#if WASM_ENABLE_FAST_JIT != 0
calculate_global_data_offset(module);
#if WASM_ENABLE_FAST_JIT != 0
if (module->function_count
&& !(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count,
@ -3367,6 +3479,12 @@ load_from_sections(WASMModule *module, WASMSection *sections,
}
#endif
#if WASM_ENABLE_JIT != 0
if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) {
return false;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_MEMORY_TRACING != 0
wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module);
#endif
@ -3900,6 +4018,15 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_JIT != 0
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
aot_destroy_comp_context(module->comp_ctx);
if (module->comp_data)
aot_destroy_comp_data(module->comp_data);
#endif
wasm_runtime_free(module);
}
@ -6756,8 +6883,7 @@ re_scan:
}
block_type.is_value_type = false;
block_type.u.type = module->types[type_index];
#if WASM_ENABLE_FAST_INTERP == 0 && WASM_ENABLE_WAMR_COMPILER == 0 \
&& WASM_ENABLE_JIT == 0
#if WASM_ENABLE_FAST_INTERP == 0
/* If block use type index as block type, change the opcode
* to new extended opcode so that interpreter can resolve
* the block quickly.

159
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -15,6 +15,9 @@
#include "../fast-jit/jit_compiler.h"
#include "../fast-jit/jit_codecache.h"
#endif
#if WASM_ENABLE_JIT != 0
#include "../compilation/aot_llvm.h"
#endif
/* Read a value of given type from the address pointed to by the given
pointer and increase the pointer to the position just after the
@ -961,6 +964,11 @@ load_function_section(const uint8 *buf, const uint8 *buf_end,
read_leb_uint32(p, p_end, type_index);
bh_assert(type_index < module->type_count);
#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0)
type_index = wasm_get_smallest_type_idx(
module->types, module->type_count, type_index);
#endif
read_leb_uint32(p_code, buf_code_end, code_size);
bh_assert(code_size > 0 && p_code + code_size <= buf_code_end);
@ -1749,6 +1757,137 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
return true;
}
static void
calculate_global_data_offset(WASMModule *module)
{
uint32 i, data_offset;
data_offset = 0;
for (i = 0; i < module->import_global_count; i++) {
WASMGlobalImport *import_global =
&((module->import_globals + i)->u.global);
#if WASM_ENABLE_FAST_JIT != 0
import_global->data_offset = data_offset;
#endif
data_offset += wasm_value_type_size(import_global->type);
}
for (i = 0; i < module->global_count; i++) {
WASMGlobal *global = module->globals + i;
#if WASM_ENABLE_FAST_JIT != 0
global->data_offset = data_offset;
#endif
data_offset += wasm_value_type_size(global->type);
}
module->global_data_size = data_offset;
}
#if WASM_ENABLE_JIT != 0
static bool
compile_llvm_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
AOTCompOption option = { 0 };
char func_name[32], *aot_last_error;
uint64 size;
uint32 i;
size = sizeof(void *) * (uint64)module->function_count;
if (size > 0
&& !(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->comp_data = aot_create_comp_data(module);
if (!module->comp_data) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
set_error_buf(error_buf, error_buf_size, aot_last_error);
return false;
}
option.is_jit_mode = true;
option.opt_level = 3;
option.size_level = 3;
#if WASM_ENABLE_BULK_MEMORY != 0
option.enable_bulk_memory = true;
#endif
#if WASM_ENABLE_THREAD_MGR != 0
option.enable_thread_mgr = true;
#endif
#if WASM_ENABLE_TAIL_CALL != 0
option.enable_tail_call = true;
#endif
#if WASM_ENABLE_SIMD != 0
option.enable_simd = true;
#endif
#if WASM_ENABLE_REF_TYPES != 0
option.enable_ref_types = true;
#endif
option.enable_aux_stack_check = true;
#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0)
option.enable_aux_stack_frame = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
set_error_buf(error_buf, error_buf_size, aot_last_error);
return false;
}
if (!aot_compile_wasm(module->comp_ctx)) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
set_error_buf(error_buf, error_buf_size, aot_last_error);
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
for (i = 0; i < module->comp_data->func_count; i++) {
LLVMErrorRef error;
LLVMOrcJITTargetAddress func_addr = 0;
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit,
&func_addr, func_name))) {
char *err_msg = LLVMGetErrorMessage(error);
set_error_buf_v(error_buf, error_buf_size,
"failed to compile orc jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
return false;
}
/**
* No need to lock the func_ptr[func_idx] here as it is basic
* data type, the load/store for it can be finished by one cpu
* instruction, and there can be only one cpu instruction
* loading/storing at the same time.
*/
module->func_ptrs[i] = (void *)func_addr;
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
}
#else
/* Resolve function addresses */
bh_assert(module->comp_ctx->exec_engine);
for (i = 0; i < module->comp_data->func_count; i++) {
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress(
module->comp_ctx->exec_engine, func_name))) {
set_error_buf(error_buf, error_buf_size,
"failed to compile llvm mc jit function");
return false;
}
module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i];
}
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
return true;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_REF_TYPES != 0
static bool
get_table_elem_type(const WASMModule *module, uint32 table_idx,
@ -2185,6 +2324,8 @@ load_from_sections(WASMModule *module, WASMSection *sections,
}
}
calculate_global_data_offset(module);
#if WASM_ENABLE_FAST_JIT != 0
if (!(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count, error_buf,
@ -2197,6 +2338,12 @@ load_from_sections(WASMModule *module, WASMSection *sections,
}
#endif
#if WASM_ENABLE_JIT != 0
if (!compile_llvm_jit_functions(module, error_buf, error_buf_size)) {
return false;
}
#endif /* end of WASM_ENABLE_JIT != 0 */
#if WASM_ENABLE_MEMORY_TRACING != 0
wasm_runtime_dump_module_mem_consumption(module);
#endif
@ -2526,6 +2673,15 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_JIT != 0
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
aot_destroy_comp_context(module->comp_ctx);
if (module->comp_data)
aot_destroy_comp_data(module->comp_data);
#endif
wasm_runtime_free(module);
}
@ -5035,8 +5191,7 @@ re_scan:
bh_assert(type_index < module->type_count);
block_type.is_value_type = false;
block_type.u.type = module->types[type_index];
#if WASM_ENABLE_FAST_INTERP == 0 && WASM_ENABLE_WAMR_COMPILER == 0 \
&& WASM_ENABLE_JIT == 0
#if WASM_ENABLE_FAST_INTERP == 0
/* If block use type index as block type, change the opcode
* to new extended opcode so that interpreter can resolve
* the block quickly.

1422
core/iwasm/interpreter/wasm_runtime.c
View File

File diff suppressed because it is too large Load Diff

396
core/iwasm/interpreter/wasm_runtime.h
View File

@ -21,6 +21,53 @@ typedef struct WASMMemoryInstance WASMMemoryInstance;
typedef struct WASMTableInstance WASMTableInstance;
typedef struct WASMGlobalInstance WASMGlobalInstance;
/**
* When LLVM JIT, WAMR compiler or AOT is enabled, we should ensure that
* some offsets of the same field in the interpreter module instance and
* aot module instance are the same, so that the LLVM JITed/AOTed code
* can smoothly access the interpreter module instance.
* Same for the memory instance and table instance.
* We use the macro DefPointer to define some related pointer fields.
*/
#if (WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 \
|| WASM_ENABLE_AOT != 0) \
&& UINTPTR_MAX == UINT32_MAX
/* Add u32 padding if LLVM JIT, WAMR compiler or AOT is enabled on
32-bit platform */
#define DefPointer(type, field) \
type field; \
uint32 field##_padding
#else
#define DefPointer(type, field) type field
#endif
typedef enum WASMExceptionID {
EXCE_UNREACHABLE = 0,
EXCE_OUT_OF_MEMORY,
EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS,
EXCE_INTEGER_OVERFLOW,
EXCE_INTEGER_DIVIDE_BY_ZERO,
EXCE_INVALID_CONVERSION_TO_INTEGER,
EXCE_INVALID_FUNCTION_TYPE_INDEX,
EXCE_INVALID_FUNCTION_INDEX,
EXCE_UNDEFINED_ELEMENT,
EXCE_UNINITIALIZED_ELEMENT,
EXCE_CALL_UNLINKED_IMPORT_FUNC,
EXCE_NATIVE_STACK_OVERFLOW,
EXCE_UNALIGNED_ATOMIC,
EXCE_AUX_STACK_OVERFLOW,
EXCE_AUX_STACK_UNDERFLOW,
EXCE_OUT_OF_BOUNDS_TABLE_ACCESS,
EXCE_OPERAND_STACK_OVERFLOW,
EXCE_ALREADY_THROWN,
EXCE_NUM,
} WASMExceptionID;
typedef union {
uint64 u64;
uint32 u32[2];
} MemBound;
struct WASMMemoryInstance {
/* Module type */
uint32 module_type;
@ -35,59 +82,40 @@ struct WASMMemoryInstance {
uint32 max_page_count;
/* Memory data size */
uint32 memory_data_size;
/**
* Memory data begin address, Note:
* the app-heap might be inserted in to the linear memory,
* when memory is re-allocated, the heap data and memory data
* must be copied to new memory also
*/
uint8 *memory_data;
DefPointer(uint8 *, memory_data);
/* Memory data end address */
uint8 *memory_data_end;
DefPointer(uint8 *, memory_data_end);
/* Heap data base address */
uint8 *heap_data;
DefPointer(uint8 *, heap_data);
/* Heap data end address */
uint8 *heap_data_end;
DefPointer(uint8 *, heap_data_end);
/* The heap created */
void *heap_handle;
DefPointer(void *, heap_handle);
#if WASM_ENABLE_SHARED_MEMORY != 0
/* mutex lock for the memory, used in atomic operation */
korp_mutex mem_lock;
#endif
#if WASM_ENABLE_FAST_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
uint64 mem_bound_check_1byte;
uint64 mem_bound_check_2bytes;
uint64 mem_bound_check_4bytes;
uint64 mem_bound_check_8bytes;
uint64 mem_bound_check_16bytes;
#else
uint32 mem_bound_check_1byte;
uint32 mem_bound_check_2bytes;
uint32 mem_bound_check_4bytes;
uint32 mem_bound_check_8bytes;
uint32 mem_bound_check_16bytes;
#endif
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
|| WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_AOT != 0
MemBound mem_bound_check_1byte;
MemBound mem_bound_check_2bytes;
MemBound mem_bound_check_4bytes;
MemBound mem_bound_check_8bytes;
MemBound mem_bound_check_16bytes;
#endif
};
struct WASMTableInstance {
/* The element type, VALUE_TYPE_FUNCREF/EXTERNREF currently */
uint8 elem_type;
/* Current size */
uint32 cur_size;
/* Maximum size */
uint32 max_size;
#if WASM_ENABLE_MULTI_MODULE != 0
/* just for import, keep the reference here */
WASMTableInstance *table_inst_linked;
#endif
/* Base address */
uint8 base_addr[1];
/* Table elements */
uint32 elems[1];
};
struct WASMGlobalInstance {
@ -149,7 +177,6 @@ typedef struct WASMExportFuncInstance {
WASMFunctionInstance *function;
} WASMExportFuncInstance;
#if WASM_ENABLE_MULTI_MODULE != 0
typedef struct WASMExportGlobInstance {
char *name;
WASMGlobalInstance *global;
@ -164,8 +191,40 @@ typedef struct WASMExportMemInstance {
char *name;
WASMMemoryInstance *memory;
} WASMExportMemInstance;
/* Extra info of WASM module instance for interpreter/jit mode */
typedef struct WASMModuleInstanceExtra {
WASMGlobalInstance *globals;
WASMFunctionInstance *functions;
uint32 global_count;
uint32 function_count;
WASMFunctionInstance *start_function;
WASMFunctionInstance *malloc_function;
WASMFunctionInstance *free_function;
WASMFunctionInstance *retain_function;
#if WASM_ENABLE_SHARED_MEMORY != 0
/* lock for shared memory atomic operations */
korp_mutex mem_lock;
bool mem_lock_inited;
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
bh_list sub_module_inst_list_head;
bh_list *sub_module_inst_list;
/* linked table instances of import table instances */
WASMTableInstance **table_insts_linked;
#endif
#if WASM_ENABLE_MEMORY_PROFILING != 0
uint32 max_aux_stack_used;
#endif
} WASMModuleInstanceExtra;
struct AOTFuncPerfProfInfo;
struct WASMModuleInstance {
/* Module instance type, for module instance loaded from
WASM bytecode binary, this field is Wasm_Module_Bytecode;
@ -175,78 +234,82 @@ struct WASMModuleInstance {
uint32 module_type;
uint32 memory_count;
DefPointer(WASMMemoryInstance **, memories);
/* global and table info */
uint32 global_data_size;
uint32 table_count;
uint32 global_count;
uint32 function_count;
DefPointer(uint8 *, global_data);
/* For AOTModuleInstance, it denotes `AOTTableInstance *` */
DefPointer(WASMTableInstance **, tables);
/* import func ptrs + llvm jit func ptrs */
DefPointer(void **, func_ptrs);
/* function type indexes */
DefPointer(uint32 *, func_type_indexes);
uint32 export_func_count;
#if WASM_ENABLE_MULTI_MODULE != 0
uint32 export_glob_count;
uint32 export_mem_count;
uint32 export_tab_count;
#endif
/* Array of function pointers to import functions */
void **import_func_ptrs;
#if WASM_ENABLE_FAST_JIT != 0
/* point to JITed functions */
void **fast_jit_func_ptrs;
uint32 *func_type_indexes;
#endif
WASMMemoryInstance **memories;
WASMTableInstance **tables;
WASMGlobalInstance *globals;
WASMFunctionInstance *functions;
WASMExportFuncInstance *export_functions;
#if WASM_ENABLE_MULTI_MODULE != 0
WASMExportGlobInstance *export_globals;
WASMExportMemInstance *export_memories;
WASMExportTabInstance *export_tables;
#endif
WASMMemoryInstance *default_memory;
WASMTableInstance *default_table;
/* Global data of global instances */
uint8 *global_data;
WASMFunctionInstance *start_function;
WASMFunctionInstance *malloc_function;
WASMFunctionInstance *free_function;
WASMFunctionInstance *retain_function;
WASMModule *module;
#if WASM_ENABLE_LIBC_WASI != 0
WASIContext *wasi_ctx;
#endif
WASMExecEnv *exec_env_singleton;
/* Default WASM stack size of threads of this Module instance. */
uint32 default_wasm_stack_size;
uint32 export_global_count;
uint32 export_memory_count;
uint32 export_table_count;
/* For AOTModuleInstance, it denotes `AOTFunctionInstance *` */
DefPointer(WASMExportFuncInstance *, export_functions);
DefPointer(WASMExportGlobInstance *, export_globals);
DefPointer(WASMExportMemInstance *, export_memories);
DefPointer(WASMExportTabInstance *, export_tables);
/* The exception buffer of wasm interpreter for current thread. */
char cur_exception[128];
#if WASM_ENABLE_DUMP_CALL_STACK != 0
Vector *frames;
#endif
/* The WASM module or AOT module, for AOTModuleInstance,
it denotes `AOTModule *` */
DefPointer(WASMModule *, module);
/* The custom data that can be set/get by
* wasm_set_custom_data/wasm_get_custom_data */
void *custom_data;
#if WASM_ENABLE_MULTI_MODULE != 0
/* TODO: add mutex for mutli-threads? */
bh_list sub_module_inst_list_head;
bh_list *sub_module_inst_list;
#if WASM_ENABLE_LIBC_WASI
/* WASI context */
DefPointer(WASIContext *, wasi_ctx);
#else
DefPointer(void *, wasi_ctx);
#endif
DefPointer(WASMExecEnv *, exec_env_singleton);
/* Array of function pointers to import functions,
not available in AOTModuleInstance */
DefPointer(void **, import_func_ptrs);
/* Array of function pointers to fast jit functions,
not available in AOTModuleInstance */
DefPointer(void **, fast_jit_func_ptrs);
/* The custom data that can be set/get by wasm_{get|set}_custom_data */
DefPointer(void *, custom_data);
/* Stack frames, used in call stack dump and perf profiling */
DefPointer(Vector *, frames);
/* Function performance profiling info list, only available
in AOTModuleInstance */
DefPointer(struct AOTFuncPerfProfInfo *, func_perf_profilings);
/* WASM/AOT module extra info, for AOTModuleInstance,
it denotes `AOTModuleInstanceExtra *` */
DefPointer(WASMModuleInstanceExtra *, e);
#if WASM_ENABLE_MEMORY_PROFILING != 0
uint32 max_aux_stack_used;
#endif
/* Default WASM operand stack size */
uint32 default_wasm_stack_size;
uint32 reserved[3];
/*
* +------------------------------+ <-- memories
* | WASMMemoryInstance[mem_count], mem_count is always 1 for LLVM JIT/AOT
* +------------------------------+ <-- global_data
* | global data
* +------------------------------+ <-- tables
* | WASMTableInstance[table_count]
* +------------------------------+ <-- e
* | WASMModuleInstanceExtra
* +------------------------------+
*/
union {
uint64 _make_it_8_byte_aligned_;
WASMMemoryInstance memory_instances[1];
uint8 bytes[1];
} global_table_data;
};
struct WASMInterpFrame;
@ -342,12 +405,12 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst,
WASMFunctionInstance *function,
unsigned argc, uint32 argv[]);
bool
wasm_create_exec_env_singleton(WASMModuleInstance *module_inst);
void
wasm_set_exception(WASMModuleInstance *module, const char *exception);
void
wasm_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id);
const char *
wasm_get_exception(WASMModuleInstance *module);
@ -366,45 +429,25 @@ uint32
wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src,
uint32 size);
/**
* Check whether the app address and the buf is inside the linear memory,
* and convert the app address into native address
*/
bool
wasm_validate_app_addr(WASMModuleInstance *module_inst, uint32 app_offset,
uint32 size);
wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
uint32 app_buf_addr, uint32 app_buf_size,
void **p_native_addr);
WASMMemoryInstance *
wasm_get_default_memory(WASMModuleInstance *module_inst);
bool
wasm_validate_app_str_addr(WASMModuleInstance *module_inst, uint32 app_offset);
bool
wasm_validate_native_addr(WASMModuleInstance *module_inst, void *native_ptr,
uint32 size);
void *
wasm_addr_app_to_native(WASMModuleInstance *module_inst, uint32 app_offset);
uint32
wasm_addr_native_to_app(WASMModuleInstance *module_inst, void *native_ptr);
bool
wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset,
uint32 *p_app_start_offset, uint32 *p_app_end_offset);
bool
wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr,
uint8 **p_native_start_addr,
uint8 **p_native_end_addr);
bool
wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count);
wasm_enlarge_memory(WASMModuleInstance *module_inst, uint32 inc_page_count);
bool
wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 argc, uint32 argv[]);
#if WASM_ENABLE_FAST_JIT != 0
bool
jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 type_idx, uint32 argc, uint32 argv[]);
#endif
#if WASM_ENABLE_THREAD_MGR != 0
bool
wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size);
@ -413,16 +456,6 @@ bool
wasm_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size);
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
#ifndef BH_PLATFORM_WINDOWS
void
wasm_signal_handler(WASMSignalInfo *sig_info);
#else
LONG
wasm_exception_handler(WASMSignalInfo *sig_info);
#endif
#endif
void
wasm_get_module_mem_consumption(const WASMModule *module,
WASMModuleMemConsumption *mem_conspn);
@ -456,13 +489,14 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx,
#endif /* WASM_ENABLE_REF_TYPES != 0 */
static inline WASMTableInstance *
wasm_get_table_inst(const WASMModuleInstance *module_inst, const uint32 tbl_idx)
wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx)
{
/* careful, it might be a table in another module */
WASMTableInstance *tbl_inst = module_inst->tables[tbl_idx];
#if WASM_ENABLE_MULTI_MODULE != 0
if (tbl_inst->table_inst_linked) {
tbl_inst = tbl_inst->table_inst_linked;
if (tbl_idx < module_inst->module->import_table_count
&& module_inst->e->table_insts_linked[tbl_idx]) {
tbl_inst = module_inst->e->table_insts_linked[tbl_idx];
}
#endif
bh_assert(tbl_inst);
@ -495,6 +529,82 @@ const uint8 *
wasm_loader_get_custom_section(WASMModule *module, const char *name,
uint32 *len);
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
|| WASM_ENABLE_WAMR_COMPILER != 0
void
jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id);
/**
* Check whether the app address and the buf is inside the linear memory,
* and convert the app address into native address
*/
bool
jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
uint32 app_buf_addr, uint32 app_buf_size,
void **p_native_addr);
#endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
|| WASM_ENABLE_WAMR_COMPILER != 0 */
#if WASM_ENABLE_FAST_JIT != 0
bool
fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 type_idx, uint32 argc, uint32 *argv);
bool
fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
struct WASMInterpFrame *prev_frame);
#endif
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
bool
llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 argc, uint32 *argv);
bool
llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
uint32 *argv);
#if WASM_ENABLE_BULK_MEMORY != 0
bool
llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index,
uint32 offset, uint32 len, uint32 dst);
bool
llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index);
#endif
#if WASM_ENABLE_REF_TYPES != 0
void
llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx);
void
llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx,
uint32 tbl_seg_idx, uint32 length, uint32 src_offset,
uint32 dst_offset);
void
llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx,
uint32 dst_tbl_idx, uint32 length, uint32 src_offset,
uint32 dst_offset);
void
llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx,
uint32 length, uint32 val, uint32 data_offset);
uint32
llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx,
uint32 inc_entries, uint32 init_val);
#endif
#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0
bool
llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index);
void
llvm_jit_free_frame(WASMExecEnv *exec_env);
#endif
#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */
#ifdef __cplusplus
}
#endif