christoph/wamr
1
Fork 0
Code Releases Activity

Import WAMR Fast JIT (#1343)

Browse Source 
Import WAMR Fast JIT which is a lightweight JIT with quick startup, small footprint,
relatively good performance (~40% to ~50% of LLVM JIT) and good portability.

Platforms supported: Linux, MacOS and Linux SGX.
Arch supported: x86-64.
This commit is contained in:
Wenyong Huang
2022-08-02 16:03:50 +08:00
committed by GitHub
parent 1c6d10095e
commit bf28030993
68 changed files with 22563 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
core/iwasm/interpreter/wasm.h
View File

@ -204,6 +204,10 @@ typedef struct WASMGlobalImport {
WASMModule *import_module;
WASMGlobal *import_global_linked;
#endif
#if WASM_ENABLE_FAST_JIT != 0
/* The data offset of current global in global data */
uint32 data_offset;
#endif
} WASMGlobalImport;
typedef struct WASMImport {
@ -254,12 +258,19 @@ struct WASMFunction {
uint8 *consts;
uint32 const_cell_num;
#endif
#if WASM_ENABLE_FAST_JIT != 0
void *fast_jit_jitted_code;
#endif
};
struct WASMGlobal {
uint8 type;
bool is_mutable;
InitializerExpression init_expr;
#if WASM_ENABLE_FAST_JIT != 0
/* The data offset of current global in global data */
uint32 data_offset;
#endif
};
typedef struct WASMExport {
@ -443,9 +454,12 @@ struct WASMModule {
#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0
bh_list fast_opcode_list;
uint8 *buf_code;
uint64 buf_code_size;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0 \
|| WASM_ENABLE_FAST_JIT != 0
uint8 *load_addr;
uint64 load_size;
uint64 buf_code_size;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
@ -470,6 +484,11 @@ struct WASMModule {
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
WASMCustomSection *custom_section_list;
#endif
#if WASM_ENABLE_FAST_JIT != 0
/* point to JITed functions */
void **fast_jit_func_ptrs;
#endif
};
typedef struct BlockType {
@ -612,6 +631,7 @@ wasm_get_smallest_type_idx(WASMType **types, uint32 type_count,
if (wasm_type_equal(types[cur_type_idx], types[i]))
return i;
}
(void)type_count;
return cur_type_idx;
}

17
core/iwasm/interpreter/wasm_interp.h
View File

@ -26,6 +26,10 @@ typedef struct WASMInterpFrame {
/* Instruction pointer of the bytecode array. */
uint8 *ip;
#if WASM_ENABLE_FAST_JIT != 0
uint8 *jitted_return_addr;
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
uint64 time_started;
#endif
@ -47,12 +51,13 @@ typedef struct WASMInterpFrame {
WASMBranchBlock *csp_boundary;
WASMBranchBlock *csp;
/* Frame data, the layout is:
lp: param_cell_count + local_cell_count
sp_bottom to sp_boundary: stack of data
csp_bottom to csp_boundary: stack of block
ref to frame end: data types of local vairables and stack data
*/
/**
* Frame data, the layout is:
* lp: parameters and local variables
* sp_bottom to sp_boundary: wasm operand stack
* csp_bottom to csp_boundary: wasm label stack
* jit spill cache: only available for fast jit
*/
uint32 lp[1];
#endif
} WASMInterpFrame;

66
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -16,6 +16,9 @@
#include "../libraries/thread-mgr/thread_manager.h"
#include "../libraries/debug-engine/debug_engine.h"
#endif
#if WASM_ENABLE_FAST_JIT != 0
#include "../fast-jit/jit_compiler.h"
#endif
typedef int32 CellType_I32;
typedef int64 CellType_I64;
@ -855,6 +858,20 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst,
wasm_exec_env_set_cur_frame(exec_env, prev_frame);
}
#if WASM_ENABLE_FAST_JIT != 0
bool
jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
WASMInterpFrame *prev_frame)
{
WASMModuleInstance *module_inst =
(WASMModuleInstance *)exec_env->module_inst;
WASMFunctionInstance *cur_func = module_inst->functions + func_idx;
wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame);
return wasm_get_exception(module_inst) ? false : true;
}
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
static void
wasm_interp_call_func_bytecode(WASMModuleInstance *module,
@ -3897,7 +3914,56 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
}
else {
#if WASM_ENABLE_FAST_JIT == 0
wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
#else
JitGlobals *jit_globals = jit_compiler_get_jit_globals();
JitInterpSwitchInfo info;
WASMType *func_type = function->u.func->func_type;
uint8 type = func_type->result_count
? func_type->types[func_type->param_count]
: VALUE_TYPE_VOID;
#if WASM_ENABLE_REF_TYPES != 0
if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
type = VALUE_TYPE_I32;
#endif
info.out.ret.last_return_type = type;
info.frame = frame;
frame->jitted_return_addr =
(uint8 *)jit_globals->return_to_interp_from_jitted;
jit_interp_switch_to_jitted(exec_env, &info,
function->u.func->fast_jit_jitted_code);
if (func_type->result_count) {
switch (type) {
case VALUE_TYPE_I32:
*(frame->sp - function->ret_cell_num) =
info.out.ret.ival[0];
break;
case VALUE_TYPE_I64:
*(frame->sp - function->ret_cell_num) =
info.out.ret.ival[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.ival[1];
break;
case VALUE_TYPE_F32:
*(frame->sp - function->ret_cell_num) =
info.out.ret.fval[0];
break;
case VALUE_TYPE_F64:
*(frame->sp - function->ret_cell_num) =
info.out.ret.fval[0];
*(frame->sp - function->ret_cell_num + 1) =
info.out.ret.fval[1];
break;
default:
bh_assert(0);
break;
}
}
(void)wasm_interp_call_func_bytecode;
#endif
}
/* Output the return value to the caller */

59
core/iwasm/interpreter/wasm_loader.c
View File

@ -14,6 +14,10 @@
#if WASM_ENABLE_DEBUG_INTERP != 0
#include "../libraries/debug-engine/debug_engine.h"
#endif
#if WASM_ENABLE_FAST_JIT != 0
#include "../fast-jit/jit_compiler.h"
#include "../fast-jit/jit_codecache.h"
#endif
/* Read a value of given type from the address pointed to by the given
pointer and increase the pointer to the position just after the
@ -2890,6 +2894,28 @@ fail:
return false;
}
#if WASM_ENABLE_FAST_JIT != 0
static void
calculate_global_data_offset(WASMModule *module)
{
uint32 i, data_offset;
data_offset = 0;
for (i = 0; i < module->import_global_count; i++) {
WASMGlobalImport *import_global =
&((module->import_globals + i)->u.global);
import_global->data_offset = data_offset;
data_offset += wasm_value_type_size(import_global->type);
}
for (i = 0; i < module->global_count; i++) {
WASMGlobal *global = module->globals + i;
global->data_offset = data_offset;
data_offset += wasm_value_type_size(global->type);
}
}
#endif
static bool
wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
uint32 cur_func_idx, char *error_buf,
@ -3277,6 +3303,21 @@ load_from_sections(WASMModule *module, WASMSection *sections,
#endif
}
#if WASM_ENABLE_FAST_JIT != 0
calculate_global_data_offset(module);
if (module->function_count
&& !(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count,
error_buf, error_buf_size))) {
return false;
}
if (!jit_compiler_compile_all(module)) {
set_error_buf(error_buf, error_buf_size, "fast jit compilation failed");
return false;
}
#endif
#if WASM_ENABLE_MEMORY_TRACING != 0
wasm_runtime_dump_module_mem_consumption((WASMModuleCommon *)module);
#endif
@ -3652,7 +3693,7 @@ wasm_loader_load(uint8 *buf, uint32 size,
return NULL;
}
#if WASM_ENABLE_DEBUG_INTERP != 0
#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_FAST_JIT != 0
module->load_addr = (uint8 *)buf;
module->load_size = size;
#endif
@ -3800,6 +3841,16 @@ wasm_loader_unload(WASMModule *module)
wasm_runtime_destroy_custom_sections(module->custom_section_list);
#endif
#if WASM_ENABLE_FAST_JIT != 0
if (module->fast_jit_func_ptrs) {
for (i = 0; i < module->function_count; i++) {
if (module->fast_jit_func_ptrs[i])
jit_code_cache_free(module->fast_jit_func_ptrs[i]);
}
wasm_runtime_free(module->fast_jit_func_ptrs);
}
#endif
wasm_runtime_free(module);
}
@ -7584,7 +7635,7 @@ re_scan:
PUSH_OFFSET_TYPE(local_type);
#else
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \
&& (WASM_ENABLE_DEBUG_INTERP == 0)
&& (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0)
if (local_offset < 0x80) {
*p_org++ = EXT_OP_GET_LOCAL_FAST;
if (is_32bit_type(local_type)) {
@ -7648,7 +7699,7 @@ re_scan:
}
#else
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \
&& (WASM_ENABLE_DEBUG_INTERP == 0)
&& (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0)
if (local_offset < 0x80) {
*p_org++ = EXT_OP_SET_LOCAL_FAST;
if (is_32bit_type(local_type)) {
@ -7708,7 +7759,7 @@ re_scan:
- wasm_value_type_cell_num(local_type)));
#else
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \
&& (WASM_ENABLE_DEBUG_INTERP == 0)
&& (WASM_ENABLE_FAST_JIT == 0) && (WASM_ENABLE_DEBUG_INTERP == 0)
if (local_offset < 0x80) {
*p_org++ = EXT_OP_TEE_LOCAL_FAST;
if (is_32bit_type(local_type)) {

40
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -11,6 +11,10 @@
#include "wasm_runtime.h"
#include "../common/wasm_native.h"
#include "../common/wasm_memory.h"
#if WASM_ENABLE_FAST_JIT != 0
#include "../fast-jit/jit_compiler.h"
#include "../fast-jit/jit_codecache.h"
#endif
/* Read a value of given type from the address pointed to by the given
pointer and increase the pointer to the position just after the
@ -2139,6 +2143,18 @@ load_from_sections(WASMModule *module, WASMSection *sections,
}
}
#if WASM_ENABLE_FAST_JIT != 0
if (!(module->fast_jit_func_ptrs =
loader_malloc(sizeof(void *) * module->function_count, error_buf,
error_buf_size))) {
return false;
}
if (!jit_compiler_compile_all(module)) {
set_error_buf(error_buf, error_buf_size, "fast jit compilation failed");
return false;
}
#endif
#if WASM_ENABLE_MEMORY_TRACING != 0
wasm_runtime_dump_module_mem_consumption(module);
#endif
@ -2356,6 +2372,11 @@ wasm_loader_load(uint8 *buf, uint32 size, char *error_buf,
return NULL;
}
#if WASM_ENABLE_FAST_JIT != 0
module->load_addr = (uint8 *)buf;
module->load_size = size;
#endif
if (!load(buf, size, module, error_buf, error_buf_size)) {
goto fail;
}
@ -2453,6 +2474,16 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_FAST_JIT != 0
if (module->fast_jit_func_ptrs) {
for (i = 0; i < module->function_count; i++) {
if (module->fast_jit_func_ptrs[i])
jit_code_cache_free(module->fast_jit_func_ptrs[i]);
}
wasm_runtime_free(module->fast_jit_func_ptrs);
}
#endif
wasm_runtime_free(module);
}
@ -5778,7 +5809,8 @@ re_scan:
operand_offset = local_offset;
PUSH_OFFSET_TYPE(local_type);
#else
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0)
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \
&& (WASM_ENABLE_FAST_JIT == 0)
if (local_offset < 0x80) {
*p_org++ = EXT_OP_GET_LOCAL_FAST;
if (is_32bit_type(local_type))
@ -5838,7 +5870,8 @@ re_scan:
POP_OFFSET_TYPE(local_type);
}
#else
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0)
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \
&& (WASM_ENABLE_FAST_JIT == 0)
if (local_offset < 0x80) {
*p_org++ = EXT_OP_SET_LOCAL_FAST;
if (is_32bit_type(local_type))
@ -5894,7 +5927,8 @@ re_scan:
*(loader_ctx->frame_offset
- wasm_value_type_cell_num(local_type)));
#else
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0)
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) \
&& (WASM_ENABLE_FAST_JIT == 0)
if (local_offset < 0x80) {
*p_org++ = EXT_OP_TEE_LOCAL_FAST;
if (is_32bit_type(local_type))

94
core/iwasm/interpreter/wasm_runtime.c
View File

@ -56,7 +56,7 @@ wasm_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size)
WASMModule *
wasm_load_from_sections(WASMSection *section_list, char *error_buf,
uint32_t error_buf_size)
uint32 error_buf_size)
{
return wasm_loader_load_from_sections(section_list, error_buf,
error_buf_size);
@ -349,6 +349,24 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
}
}
#if WASM_ENABLE_FAST_JIT != 0
if (memory_data_size > 0) {
#if UINTPTR_MAX == UINT64_MAX
memory->mem_bound_check_1byte = memory_data_size - 1;
memory->mem_bound_check_2bytes = memory_data_size - 2;
memory->mem_bound_check_4bytes = memory_data_size - 4;
memory->mem_bound_check_8bytes = memory_data_size - 8;
memory->mem_bound_check_16bytes = memory_data_size - 16;
#else
memory->mem_bound_check_1byte = (uint32)memory_data_size - 1;
memory->mem_bound_check_2bytes = (uint32)memory_data_size - 2;
memory->mem_bound_check_4bytes = (uint32)memory_data_size - 4;
memory->mem_bound_check_8bytes = (uint32)memory_data_size - 8;
memory->mem_bound_check_16bytes = (uint32)memory_data_size - 16;
#endif
}
#endif
#if WASM_ENABLE_SHARED_MEMORY != 0
if (0 != os_mutex_init(&memory->mem_lock)) {
set_error_buf(error_buf, error_buf_size, "init mutex failed");
@ -693,6 +711,10 @@ functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
function++;
}
#if WASM_ENABLE_FAST_JIT != 0
module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
#endif
bh_assert((uint32)(function - functions) == function_count);
(void)module_inst;
return functions;
@ -2470,6 +2492,22 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
memory->memory_data_end =
memory->memory_data + memory->num_bytes_per_page * total_page_count;
#if WASM_ENABLE_FAST_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
memory->mem_bound_check_1byte = total_size - 1;
memory->mem_bound_check_2bytes = total_size - 2;
memory->mem_bound_check_4bytes = total_size - 4;
memory->mem_bound_check_8bytes = total_size - 8;
memory->mem_bound_check_16bytes = total_size - 16;
#else
memory->mem_bound_check_1byte = (uint32)total_size - 1;
memory->mem_bound_check_2bytes = (uint32)total_size - 2;
memory->mem_bound_check_4bytes = (uint32)total_size - 4;
memory->mem_bound_check_8bytes = (uint32)total_size - 8;
memory->mem_bound_check_16bytes = (uint32)total_size - 16;
#endif
#endif
return ret;
}
#else
@ -2564,14 +2602,14 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx,
}
#endif /* WASM_ENABLE_REF_TYPES != 0 */
bool
wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx,
uint32_t element_indices, uint32_t argc, uint32_t argv[])
static bool
call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 argc, uint32 argv[], bool check_type_idx, uint32 type_idx)
{
WASMModuleInstance *module_inst = NULL;
WASMTableInstance *table_inst = NULL;
uint32_t function_indices = 0;
WASMFunctionInstance *function_inst = NULL;
uint32 func_idx = 0;
WASMFunctionInstance *func_inst = NULL;
module_inst = (WASMModuleInstance *)exec_env->module_inst;
bh_assert(module_inst);
@ -2582,7 +2620,7 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx,
goto got_exception;
}
if (element_indices >= table_inst->cur_size) {
if (elem_idx >= table_inst->cur_size) {
wasm_set_exception(module_inst, "undefined element");
goto got_exception;
}
@ -2591,8 +2629,8 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx,
* please be aware that table_inst->base_addr may point
* to another module's table
**/
function_indices = ((uint32_t *)table_inst->base_addr)[element_indices];
if (function_indices == NULL_REF) {
func_idx = ((uint32 *)table_inst->base_addr)[elem_idx];
if (func_idx == NULL_REF) {
wasm_set_exception(module_inst, "uninitialized element");
goto got_exception;
}
@ -2600,14 +2638,29 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx,
/**
* we insist to call functions owned by the module itself
**/
if (function_indices >= module_inst->function_count) {
if (func_idx >= module_inst->function_count) {
wasm_set_exception(module_inst, "unknown function");
goto got_exception;
}
function_inst = module_inst->functions + function_indices;
func_inst = module_inst->functions + func_idx;
interp_call_wasm(module_inst, exec_env, function_inst, argc, argv);
if (check_type_idx) {
WASMType *cur_type = module_inst->module->types[type_idx];
WASMType *cur_func_type;
if (func_inst->is_import_func)
cur_func_type = func_inst->u.func_import->func_type;
else
cur_func_type = func_inst->u.func->func_type;
if (!wasm_type_equal(cur_type, cur_func_type)) {
wasm_set_exception(module_inst, "indirect call type mismatch");
goto got_exception;
}
}
interp_call_wasm(module_inst, exec_env, func_inst, argc, argv);
(void)clear_wasi_proc_exit_exception(module_inst);
return !wasm_get_exception(module_inst) ? true : false;
@ -2616,6 +2669,23 @@ got_exception:
return false;
}
bool
wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 argc, uint32 argv[])
{
return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
}
#if WASM_ENABLE_FAST_JIT != 0
bool
jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 type_idx, uint32 argc, uint32 argv[])
{
return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true,
type_idx);
}
#endif
#if WASM_ENABLE_THREAD_MGR != 0
bool
wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size)

38
core/iwasm/interpreter/wasm_runtime.h
View File

@ -52,6 +52,22 @@ struct WASMMemoryInstance {
Note: when memory is re-allocated, the heap data and memory data
must be copied to new memory also. */
uint8 *memory_data;
#if WASM_ENABLE_FAST_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
uint64 mem_bound_check_1byte;
uint64 mem_bound_check_2bytes;
uint64 mem_bound_check_4bytes;
uint64 mem_bound_check_8bytes;
uint64 mem_bound_check_16bytes;
#else
uint32 mem_bound_check_1byte;
uint32 mem_bound_check_2bytes;
uint32 mem_bound_check_4bytes;
uint32 mem_bound_check_8bytes;
uint32 mem_bound_check_16bytes;
#endif
#endif
};
struct WASMTableInstance {
@ -167,6 +183,10 @@ struct WASMModuleInstance {
/* Array of function pointers to import functions */
void **import_func_ptrs;
#if WASM_ENABLE_FAST_JIT != 0
/* point to JITed functions */
void **fast_jit_func_ptrs;
#endif
WASMMemoryInstance **memories;
WASMTableInstance **tables;
@ -280,7 +300,7 @@ wasm_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size);
WASMModule *
wasm_load_from_sections(WASMSection *section_list, char *error_buf,
uint32_t error_buf_size);
uint32 error_buf_size);
void
wasm_unload(WASMModule *module);
@ -366,16 +386,22 @@ wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset,
uint32 *p_app_start_offset, uint32 *p_app_end_offset);
bool
wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8_t *native_ptr,
uint8_t **p_native_start_addr,
uint8_t **p_native_end_addr);
wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr,
uint8 **p_native_start_addr,
uint8 **p_native_end_addr);
bool
wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count);
bool
wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx,
uint32_t element_indices, uint32_t argc, uint32_t argv[]);
wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 argc, uint32 argv[]);
#if WASM_ENABLE_FAST_JIT != 0
bool
jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
uint32 type_idx, uint32 argc, uint32 argv[]);
#endif
#if WASM_ENABLE_THREAD_MGR != 0
bool