Refactor interpreter/AOT module instance layout (#1559)

Refactor the layout of interpreter and AOT module instance:
- Unify the interp/AOT module instance, use the same WASMModuleInstance/
  WASMMemoryInstance/WASMTableInstance data structures for both interpreter
  and AOT
- Make the offset of most fields the same in module instance for both interpreter
  and AOT, append memory instance structure, global data and table instances to
  the end of module instance for interpreter mode (like AOT mode)
- For extra fields in WASM module instance, use WASMModuleInstanceExtra to
  create a field `e` for interpreter
- Change the LLVM JIT module instance creating process, LLVM JIT uses the WASM
  module and module instance same as interpreter/Fast-JIT mode. So that Fast JIT
  and LLVM JIT can access the same data structures, and make it possible to
  implement the Multi-tier JIT (tier-up from Fast JIT to LLVM JIT) in the future
- Unify some APIs: merge some APIs for module instance and memory instance's
  related operations (only implement one copy)

Note that the AOT ABI is same, the AOT file format, AOT relocation types, how AOT
code accesses the AOT module instance and so on are kept unchanged.

Refer to:
https://github.com/bytecodealliance/wasm-micro-runtime/issues/1384
This commit is contained in:
Wenyong Huang
2022-10-18 10:59:28 +08:00
committed by GitHub
parent dc4dcc3d6f
commit a182926a73
49 changed files with 3790 additions and 3274 deletions

View File

@ -304,14 +304,22 @@ aot_set_last_error_v(const char *format, ...);
#endif
static inline uint32
aot_get_tbl_data_slots(const AOTTable *tbl)
aot_get_imp_tbl_data_slots(const AOTImportTable *tbl, bool is_jit_mode)
{
#if WASM_ENABLE_MULTI_MODULE != 0
if (is_jit_mode)
return tbl->table_max_size;
#endif
return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size;
}
static inline uint32
aot_get_imp_tbl_data_slots(const AOTImportTable *tbl)
aot_get_tbl_data_slots(const AOTTable *tbl, bool is_jit_mode)
{
#if WASM_ENABLE_MULTI_MODULE != 0
if (is_jit_mode)
return tbl->table_max_size;
#endif
return tbl->possible_grow ? tbl->table_max_size : tbl->table_init_size;
}

View File

@ -211,6 +211,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index)
case WASM_OP_BLOCK:
case WASM_OP_LOOP:
case WASM_OP_IF:
{
value_type = *frame_ip++;
if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64
|| value_type == VALUE_TYPE_F32
@ -245,6 +246,25 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index)
param_count, param_types, result_count, result_types))
return false;
break;
}
case EXT_OP_BLOCK:
case EXT_OP_LOOP:
case EXT_OP_IF:
{
read_leb_uint32(frame_ip, frame_ip_end, type_index);
func_type = comp_ctx->comp_data->func_types[type_index];
param_count = func_type->param_count;
param_types = func_type->types;
result_count = func_type->result_count;
result_types = func_type->types + param_count;
if (!aot_compile_op_block(
comp_ctx, func_ctx, &frame_ip, frame_ip_end,
(uint32)(LABEL_TYPE_BLOCK + opcode - EXT_OP_BLOCK),
param_count, param_types, result_count, result_types))
return false;
break;
}
case WASM_OP_ELSE:
if (!aot_compile_op_else(comp_ctx, func_ctx, &frame_ip))

View File

@ -4,7 +4,7 @@
*/
#include "aot_emit_const.h"
#include "aot_intrinsic.h"
#include "../aot/aot_intrinsic.h"
bool
aot_compile_op_i32_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,

View File

@ -4,6 +4,7 @@
*/
#include "aot_emit_exception.h"
#include "../interpreter/wasm_runtime.h"
#include "../aot/aot_runtime.h"
bool
@ -57,7 +58,7 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* Create LLVM function with const function pointer */
if (!(func_const =
I64_CONST((uint64)(uintptr_t)aot_set_exception_with_id))
I64_CONST((uint64)(uintptr_t)jit_set_exception_with_id))
|| !(func = LLVMConstIntToPtr(func_const, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;

View File

@ -162,7 +162,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* JIT mode, call the function directly */
if (!(func = I64_CONST((uint64)(uintptr_t)aot_invoke_native))
if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_invoke_native))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
@ -407,7 +407,10 @@ call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[1] = I32_TYPE;
ret_type = INT8_TYPE;
GET_AOT_FUNCTION(aot_alloc_frame, 2);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_alloc_frame, 2);
else
GET_AOT_FUNCTION(aot_alloc_frame, 2);
param_values[0] = func_ctx->exec_env;
param_values[1] = func_idx;
@ -461,7 +464,10 @@ call_aot_free_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
param_types[0] = comp_ctx->exec_env_type;
ret_type = INT8_TYPE;
GET_AOT_FUNCTION(aot_free_frame, 1);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_free_frame, 1);
else
GET_AOT_FUNCTION(aot_free_frame, 1);
param_values[0] = func_ctx->exec_env;
@ -557,7 +563,7 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* JIT mode, call the function directly */
if (!(func =
I64_CONST((uint64)(uintptr_t)aot_check_app_addr_and_convert))
I64_CONST((uint64)(uintptr_t)jit_check_app_addr_and_convert))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
@ -1047,7 +1053,7 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* JIT mode, call the function directly */
if (!(func = I64_CONST((uint64)(uintptr_t)aot_call_indirect))
if (!(func = I64_CONST((uint64)(uintptr_t)llvm_jit_call_indirect))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
@ -1265,7 +1271,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* load data as i32* */
if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx)
+ offsetof(AOTTableInstance, data)))) {
+ offsetof(AOTTableInstance, elems)))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}

View File

@ -718,7 +718,7 @@ aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_set_last_error("llvm add pointer type failed.");
return false;
}
if (!(value = I64_CONST((uint64)(uintptr_t)aot_enlarge_memory))
if (!(value = I64_CONST((uint64)(uintptr_t)wasm_enlarge_memory))
|| !(func = LLVMConstIntToPtr(value, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
@ -899,7 +899,10 @@ aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[4] = I32_TYPE;
ret_type = INT8_TYPE;
GET_AOT_FUNCTION(aot_memory_init, 5);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_memory_init, 5);
else
GET_AOT_FUNCTION(aot_memory_init, 5);
/* Call function aot_memory_init() */
param_values[0] = func_ctx->aot_inst;
@ -955,7 +958,10 @@ aot_compile_op_data_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[1] = I32_TYPE;
ret_type = INT8_TYPE;
GET_AOT_FUNCTION(aot_data_drop, 2);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_data_drop, 2);
else
GET_AOT_FUNCTION(aot_data_drop, 2);
/* Call function aot_data_drop() */
param_values[0] = func_ctx->aot_inst;
@ -971,6 +977,14 @@ fail:
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
static void *
jit_memmove(void *dest, const void *src, size_t n)
{
return memmove(dest, src, n);
}
#endif
bool
aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
@ -1025,7 +1039,7 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
return false;
}
#else
if (!(func = I64_CONST((uint64)(uintptr_t)aot_memmove))
if (!(func = I64_CONST((uint64)(uintptr_t)jit_memmove))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
@ -1054,6 +1068,12 @@ fail:
return false;
}
static void *
jit_memset(void *s, int c, size_t n)
{
return memset(s, c, n);
}
bool
aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
@ -1084,7 +1104,7 @@ aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
}
if (comp_ctx->is_jit_mode) {
if (!(func = I64_CONST((uint64)(uintptr_t)aot_memset))
if (!(func = I64_CONST((uint64)(uintptr_t)jit_memset))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;

View File

@ -15,16 +15,25 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx,
AOTImportTable *imp_tbls = comp_ctx->comp_data->import_tables;
AOTTable *tbls = comp_ctx->comp_data->tables;
/* from the head of AOTModuleInstance */
offset =
offsetof(AOTModuleInstance, global_table_data.bytes)
+ (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance)
+ comp_ctx->comp_data->global_data_size;
if (comp_ctx->is_jit_mode) {
offset = offsetof(WASMModuleInstance, global_table_data.bytes)
+ (uint64)comp_ctx->comp_data->memory_count
* sizeof(AOTMemoryInstance)
+ comp_ctx->comp_data->global_data_size;
}
else {
offset = offsetof(AOTModuleInstance, global_table_data.bytes)
+ (uint64)comp_ctx->comp_data->memory_count
* sizeof(AOTMemoryInstance)
+ comp_ctx->comp_data->global_data_size;
}
while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) {
offset += offsetof(AOTTableInstance, data);
offset += offsetof(AOTTableInstance, elems);
/* avoid loading from current AOTTableInstance */
offset += sizeof(uint32) * aot_get_imp_tbl_data_slots(imp_tbls + i);
offset +=
sizeof(uint32)
* aot_get_imp_tbl_data_slots(imp_tbls + i, comp_ctx->is_jit_mode);
++i;
}
@ -35,9 +44,10 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx,
tbl_idx -= comp_ctx->comp_data->import_table_count;
i -= comp_ctx->comp_data->import_table_count;
while (i < tbl_idx && i < comp_ctx->comp_data->table_count) {
offset += offsetof(AOTTableInstance, data);
offset += offsetof(AOTTableInstance, elems);
/* avoid loading from current AOTTableInstance */
offset += sizeof(uint32) * aot_get_tbl_data_slots(tbls + i);
offset += sizeof(uint32)
* aot_get_tbl_data_slots(tbls + i, comp_ctx->is_jit_mode);
++i;
}
@ -82,7 +92,10 @@ aot_compile_op_elem_drop(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[1] = I32_TYPE;
ret_type = VOID_TYPE;
GET_AOT_FUNCTION(aot_drop_table_seg, 2);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_drop_table_seg, 2);
else
GET_AOT_FUNCTION(aot_drop_table_seg, 2);
param_values[0] = func_ctx->aot_inst;
if (!(param_values[1] = I32_CONST(tbl_seg_idx))) {
@ -176,7 +189,7 @@ aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* load data as i32* */
if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx)
+ offsetof(AOTTableInstance, data)))) {
+ offsetof(AOTTableInstance, elems)))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
@ -230,7 +243,7 @@ aot_compile_op_table_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* load data as i32* */
if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx)
+ offsetof(AOTTableInstance, data)))) {
+ offsetof(AOTTableInstance, elems)))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
@ -282,7 +295,10 @@ aot_compile_op_table_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[5] = I32_TYPE;
ret_type = VOID_TYPE;
GET_AOT_FUNCTION(aot_table_init, 6);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_table_init, 6);
else
GET_AOT_FUNCTION(aot_table_init, 6);
param_values[0] = func_ctx->aot_inst;
@ -330,7 +346,10 @@ aot_compile_op_table_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[5] = I32_TYPE;
ret_type = VOID_TYPE;
GET_AOT_FUNCTION(aot_table_copy, 6);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_table_copy, 6);
else
GET_AOT_FUNCTION(aot_table_copy, 6);
param_values[0] = func_ctx->aot_inst;
@ -414,7 +433,10 @@ aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[3] = I32_TYPE;
ret_type = I32_TYPE;
GET_AOT_FUNCTION(aot_table_grow, 4);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_table_grow, 4);
else
GET_AOT_FUNCTION(aot_table_grow, 4);
param_values[0] = func_ctx->aot_inst;
@ -455,7 +477,10 @@ aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[4] = I32_TYPE;
ret_type = VOID_TYPE;
GET_AOT_FUNCTION(aot_table_fill, 5);
if (comp_ctx->is_jit_mode)
GET_AOT_FUNCTION(llvm_jit_table_fill, 5);
else
GET_AOT_FUNCTION(aot_table_fill, 5);
param_values[0] = func_ctx->aot_inst;

View File

@ -114,14 +114,23 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
{
AOTCompData *comp_data = comp_ctx->comp_data;
uint32 import_global_count = comp_data->import_global_count;
uint32 global_base_offset =
offsetof(AOTModuleInstance, global_table_data.bytes)
+ sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count;
uint32 global_base_offset;
uint32 global_offset;
uint8 global_type;
LLVMValueRef offset, global_ptr, global, res;
LLVMTypeRef ptr_type = NULL;
if (comp_ctx->is_jit_mode) {
global_base_offset =
offsetof(WASMModuleInstance, global_table_data.bytes)
+ sizeof(WASMMemoryInstance) * comp_ctx->comp_data->memory_count;
}
else {
global_base_offset =
offsetof(AOTModuleInstance, global_table_data.bytes)
+ sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count;
}
bh_assert(global_idx < import_global_count + comp_data->global_count);
if (global_idx < import_global_count) {

View File

@ -264,7 +264,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return false;
}
/* memories[0]->memory_data */
offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data.ptr));
offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1,
"mem_base_addr_offset"))) {
@ -292,15 +292,24 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
else
#endif
{
offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data)
+ offsetof(AOTMemoryInstance, memory_data.ptr));
uint32 offset_of_global_table_data;
if (comp_ctx->is_jit_mode)
offset_of_global_table_data =
offsetof(WASMModuleInstance, global_table_data);
else
offset_of_global_table_data =
offsetof(AOTModuleInstance, global_table_data);
offset = I32_CONST(offset_of_global_table_data
+ offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1,
"mem_base_addr_offset"))) {
aot_set_last_error("llvm build in bounds gep failed");
return false;
}
offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data)
offset = I32_CONST(offset_of_global_table_data
+ offsetof(AOTMemoryInstance, cur_page_count));
if (!(func_ctx->mem_info[0].mem_cur_page_count_addr =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
@ -309,7 +318,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
aot_set_last_error("llvm build in bounds gep failed");
return false;
}
offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data)
offset = I32_CONST(offset_of_global_table_data
+ offsetof(AOTMemoryInstance, memory_data_size));
if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1,
@ -379,7 +388,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* Load memory bound check constants */
offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte)
- offsetof(AOTMemoryInstance, memory_data.ptr));
- offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_bound_check_1byte =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
&offset, 1, "bound_check_1byte_offset"))) {
@ -405,7 +414,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_2bytes)
- offsetof(AOTMemoryInstance, memory_data.ptr));
- offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_bound_check_2bytes =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
&offset, 1, "bound_check_2bytes_offset"))) {
@ -431,7 +440,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_4bytes)
- offsetof(AOTMemoryInstance, memory_data.ptr));
- offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_bound_check_4bytes =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
&offset, 1, "bound_check_4bytes_offset"))) {
@ -457,7 +466,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_8bytes)
- offsetof(AOTMemoryInstance, memory_data.ptr));
- offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_bound_check_8bytes =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
&offset, 1, "bound_check_8bytes_offset"))) {
@ -483,7 +492,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_16bytes)
- offsetof(AOTMemoryInstance, memory_data.ptr));
- offsetof(AOTMemoryInstance, memory_data));
if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildInBoundsGEP2(
comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1,
"bound_check_16bytes_offset"))) {
@ -533,7 +542,7 @@ create_func_type_indexes(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
LLVMValueRef offset, func_type_indexes_ptr;
LLVMTypeRef int32_ptr_type;
offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes.ptr));
offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes));
func_type_indexes_ptr =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst,
&offset, 1, "func_type_indexes_ptr");

View File

@ -338,8 +338,8 @@ aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst,
LLVMErrorRef error;
LLVMOrcJITTargetAddress func_addr = 0;
AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst;
AOTModule *aot_module = (AOTModule *)aot_inst->aot_module.ptr;
void **func_ptrs = (void **)aot_inst->func_ptrs.ptr;
AOTModule *aot_module = (AOTModule *)aot_inst->module;
void **func_ptrs = aot_inst->func_ptrs;
/**
* No need to lock the func_ptr[func_idx] here as it is basic