christoph/wamr
1
Fork 0
Code Releases Activity

Refactor LLVM JIT (#1613)

Browse Source 
Refactor LLVM JIT for some purposes:
- To simplify the source code of JIT compilation
- To simplify the JIT modes
- To align with LLVM latest changes
- To prepare for the Multi-tier JIT compilation, refer to #1302

The changes mainly include:
- Remove the MCJIT mode, replace it with ORC JIT eager mode
- Remove the LLVM legacy pass manager (only keep the LLVM new pass manager)
- Change the lazy mode's LLVM module/function binding:
  change each function in an individual LLVM module into all functions in a single LLVM module
- Upgraded ORC JIT to ORCv2 JIT to enable lazy compilation

Refer to #1468
This commit is contained in:
Wenyong Huang
2022-10-18 20:17:34 +08:00
committed by GitHub
parent 84b1a6c10e
commit e87a554616
22 changed files with 1058 additions and 1104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
core/config.h
View File

@ -81,12 +81,26 @@
#define WASM_ENABLE_LAZY_JIT 0
#endif
#ifndef WASM_LAZY_JIT_COMPILE_THREAD_NUM
#define WASM_LAZY_JIT_COMPILE_THREAD_NUM 4
#ifndef WASM_ORC_JIT_BACKEND_THREAD_NUM
/* The number of backend threads created by runtime */
#define WASM_ORC_JIT_BACKEND_THREAD_NUM 4
#endif
#if WASM_ORC_JIT_BACKEND_THREAD_NUM < 1
#error "WASM_ORC_JIT_BACKEND_THREAD_NUM must be greater than 0"
#endif
#ifndef WASM_ORC_JIT_COMPILE_THREAD_NUM
/* The number of compilation threads created by LLVM JIT */
#define WASM_ORC_JIT_COMPILE_THREAD_NUM 4
#endif
#if WASM_ORC_JIT_COMPILE_THREAD_NUM < 1
#error "WASM_ORC_JIT_COMPILE_THREAD_NUM must be greater than 0"
#endif
#if (WASM_ENABLE_AOT == 0) && (WASM_ENABLE_JIT != 0)
/* LazyJIT or MCJIT can only be enabled when AOT is enabled */
/* LLVM JIT can only be enabled when AOT is enabled */
#undef WASM_ENABLE_JIT
#define WASM_ENABLE_JIT 0
@ -110,14 +124,6 @@
#define WASM_ENABLE_WAMR_COMPILER 0
#endif
#if WASM_ENABLE_WAMR_COMPILER != 0
#ifndef WASM_ENABLE_LLVM_LEGACY_PM
/* Whether to use LLVM legacy pass manager when building wamrc,
by default it is disabled and LLVM new pass manager is used */
#define WASM_ENABLE_LLVM_LEGACY_PM 0
#endif
#endif
#ifndef WASM_ENABLE_LIBC_BUILTIN
#define WASM_ENABLE_LIBC_BUILTIN 0
#endif

8
core/iwasm/aot/aot_runtime.c
View File

@ -1321,6 +1321,9 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function,
}
argc = func_type->param_cell_num;
/* func pointer was looked up previously */
bh_assert(function->u.func.func_ptr != NULL);
/* set thread handle and stack boundary */
wasm_exec_env_set_thread_info(exec_env);
@ -1828,6 +1831,11 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
func_type_idx = func_type_indexes[func_idx];
func_type = aot_module->func_types[func_type_idx];
if (func_idx >= aot_module->import_func_count) {
/* func pointer was looked up previously */
bh_assert(func_ptrs[func_idx] != NULL);
}
if (!(func_ptr = func_ptrs[func_idx])) {
bh_assert(func_idx < aot_module->import_func_count);
import_func = aot_module->import_funcs + func_idx;

4
core/iwasm/aot/aot_runtime.h
View File

@ -148,9 +148,9 @@ typedef struct AOTModule {
/* function info */
uint32 func_count;
/* point to AOTed functions */
/* func pointers of AOTed (un-imported) functions */
void **func_ptrs;
/* function type indexes */
/* func type indexes of AOTed (un-imported) functions */
uint32 *func_type_indexes;
/* export info */

245
core/iwasm/compilation/aot_compiler.c
View File

@ -2602,7 +2602,6 @@ veriy_module(AOTCompContext *comp_ctx)
char *msg = NULL;
bool ret;
#if WASM_ENABLE_LAZY_JIT == 0
ret = LLVMVerifyModule(comp_ctx->module, LLVMPrintMessageAction, &msg);
if (!ret && msg) {
if (msg[0] != '\0') {
@ -2612,135 +2611,10 @@ veriy_module(AOTCompContext *comp_ctx)
}
LLVMDisposeMessage(msg);
}
#else
uint32 i;
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
ret = LLVMVerifyModule(comp_ctx->modules[i], LLVMPrintMessageAction,
&msg);
if (!ret && msg) {
if (msg[0] != '\0') {
aot_set_last_error(msg);
LLVMDisposeMessage(msg);
return false;
}
LLVMDisposeMessage(msg);
}
}
#endif
return true;
}
static bool
apply_func_passes(AOTCompContext *comp_ctx)
{
LLVMPassManagerRef pass_mgr;
uint32 i;
#if WASM_ENABLE_LAZY_JIT == 0
pass_mgr = LLVMCreateFunctionPassManagerForModule(comp_ctx->module);
#else
pass_mgr = LLVMCreatePassManager();
#endif
if (!pass_mgr) {
aot_set_last_error("create LLVM pass manager failed.");
return false;
}
LLVMAddPromoteMemoryToRegisterPass(pass_mgr);
LLVMAddInstructionCombiningPass(pass_mgr);
LLVMAddCFGSimplificationPass(pass_mgr);
LLVMAddJumpThreadingPass(pass_mgr);
#if LLVM_VERSION_MAJOR < 12
LLVMAddConstantPropagationPass(pass_mgr);
#endif
LLVMAddIndVarSimplifyPass(pass_mgr);
if (!comp_ctx->is_jit_mode) {
/* Put Vectorize passes before GVN/LICM passes as the former
might gain more performance improvement and the latter might
break the optimizations for the former */
LLVMAddLoopVectorizePass(pass_mgr);
LLVMAddSLPVectorizePass(pass_mgr);
LLVMAddLoopRotatePass(pass_mgr);
#if LLVM_VERSION_MAJOR < 15
LLVMAddLoopUnswitchPass(pass_mgr);
#else
aot_add_simple_loop_unswitch_pass(pass_mgr);
#endif
LLVMAddInstructionCombiningPass(pass_mgr);
LLVMAddCFGSimplificationPass(pass_mgr);
if (!comp_ctx->enable_thread_mgr) {
/* These two passes may destroy the volatile semantics,
disable them when building as multi-thread mode */
LLVMAddGVNPass(pass_mgr);
LLVMAddLICMPass(pass_mgr);
LLVMAddInstructionCombiningPass(pass_mgr);
LLVMAddCFGSimplificationPass(pass_mgr);
}
}
#if WASM_ENABLE_LAZY_JIT == 0
LLVMInitializeFunctionPassManager(pass_mgr);
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
LLVMRunFunctionPassManager(pass_mgr, comp_ctx->func_ctxes[i]->func);
}
LLVMFinalizeFunctionPassManager(pass_mgr);
#else
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
LLVMRunPassManager(pass_mgr, comp_ctx->modules[i]);
}
#endif
LLVMDisposePassManager(pass_mgr);
return true;
}
#if WASM_ENABLE_LLVM_LEGACY_PM != 0 || LLVM_VERSION_MAJOR < 12
static bool
apply_lto_passes(AOTCompContext *comp_ctx)
{
LLVMPassManagerRef common_pass_mgr;
LLVMPassManagerBuilderRef pass_mgr_builder;
#if WASM_ENABLE_LAZY_JIT != 0
uint32 i;
#endif
if (!(common_pass_mgr = LLVMCreatePassManager())) {
aot_set_last_error("create LLVM pass manager failed");
return false;
}
if (!(pass_mgr_builder = LLVMPassManagerBuilderCreate())) {
aot_set_last_error("create LLVM pass manager builder failed");
LLVMDisposePassManager(common_pass_mgr);
return false;
}
LLVMPassManagerBuilderSetOptLevel(pass_mgr_builder, comp_ctx->opt_level);
LLVMPassManagerBuilderPopulateModulePassManager(pass_mgr_builder,
common_pass_mgr);
#if LLVM_VERSION_MAJOR < 15
LLVMPassManagerBuilderPopulateLTOPassManager(pass_mgr_builder,
common_pass_mgr, true, true);
#endif
#if WASM_ENABLE_LAZY_JIT == 0
LLVMRunPassManager(common_pass_mgr, comp_ctx->module);
#else
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
LLVMRunPassManager(common_pass_mgr, comp_ctx->modules[i]);
}
#endif
LLVMDisposePassManager(common_pass_mgr);
LLVMPassManagerBuilderDispose(pass_mgr_builder);
return true;
}
#endif /* end of WASM_ENABLE_LLVM_LEGACY_PM != 0 || LLVM_VERSION_MAJOR < 12 */
/* Check whether the target supports hardware atomic instructions */
static bool
aot_require_lower_atomic_pass(AOTCompContext *comp_ctx)
@ -2779,9 +2653,6 @@ static bool
apply_passes_for_indirect_mode(AOTCompContext *comp_ctx)
{
LLVMPassManagerRef common_pass_mgr;
#if WASM_ENABLE_LAZY_JIT != 0
uint32 i;
#endif
if (!(common_pass_mgr = LLVMCreatePassManager())) {
aot_set_last_error("create pass manager failed");
@ -2796,13 +2667,7 @@ apply_passes_for_indirect_mode(AOTCompContext *comp_ctx)
if (aot_require_lower_switch_pass(comp_ctx))
LLVMAddLowerSwitchPass(common_pass_mgr);
#if WASM_ENABLE_LAZY_JIT == 0
LLVMRunPassManager(common_pass_mgr, comp_ctx->module);
#else
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
LLVMRunPassManager(common_pass_mgr, comp_ctx->modules[i]);
}
#endif
LLVMDisposePassManager(common_pass_mgr);
return true;
@ -2812,11 +2677,6 @@ bool
aot_compile_wasm(AOTCompContext *comp_ctx)
{
uint32 i;
#if WASM_ENABLE_LAZY_JIT != 0
LLVMErrorRef err;
LLVMOrcJITDylibRef orc_main_dylib;
LLVMOrcThreadSafeModuleRef orc_thread_safe_module;
#endif
if (!aot_validate_wasm(comp_ctx)) {
return false;
@ -2833,87 +2693,72 @@ aot_compile_wasm(AOTCompContext *comp_ctx)
LLVMDIBuilderFinalize(comp_ctx->debug_builder);
#endif
bh_print_time("Begin to verify LLVM module");
if (!veriy_module(comp_ctx)) {
return false;
/* Disable LLVM module verification for jit mode to speedup
the compilation process */
if (!comp_ctx->is_jit_mode) {
bh_print_time("Begin to verify LLVM module");
if (!veriy_module(comp_ctx)) {
return false;
}
}
/* Run IR optimization before feeding in ORCJIT and AOT codegen */
if (comp_ctx->optimize) {
if (comp_ctx->is_jit_mode) {
/* Only run func passes for JIT mode */
bh_print_time("Begin to run func optimization passes");
if (!apply_func_passes(comp_ctx)) {
/* Run specific passes for AOT indirect mode */
if (!comp_ctx->is_jit_mode && comp_ctx->is_indirect_mode) {
bh_print_time("Begin to run optimization passes "
"for indirect mode");
if (!apply_passes_for_indirect_mode(comp_ctx)) {
return false;
}
}
else {
#if WASM_ENABLE_LLVM_LEGACY_PM == 0 && LLVM_VERSION_MAJOR >= 12
/* Run llvm new pass manager for AOT compiler if llvm
legacy pass manager isn't used */
bh_print_time("Begin to run llvm optimization passes");
aot_apply_llvm_new_pass_manager(comp_ctx);
#else
/* Run func passes and lto passes for AOT compiler if llvm
legacy pass manager is used */
bh_print_time("Begin to run func optimization passes");
if (!apply_func_passes(comp_ctx)) {
return false;
}
if (!comp_ctx->disable_llvm_lto) {
bh_print_time("Begin to run lto optimization passes");
if (!apply_lto_passes(comp_ctx)) {
return false;
}
}
/* Run passes for AOT/JIT mode.
TODO: Apply these passes in the do_ir_transform callback of
TransformLayer when compiling each jit function, so as to
speedup the launch process. Now there are two issues in the
JIT: one is memory leak in do_ir_transform, the other is
possible core dump. */
bh_print_time("Begin to run llvm optimization passes");
aot_apply_llvm_new_pass_manager(comp_ctx, comp_ctx->module);
bh_print_time("Finish llvm optimization passes");
}
#ifdef DUMP_MODULE
LLVMDumpModule(comp_ctx->module);
os_printf("\n");
#endif
/* Run passes for AOT indirect mode */
if (comp_ctx->is_indirect_mode) {
bh_print_time("Begin to run optimization passes "
"for indirect mode");
if (!apply_passes_for_indirect_mode(comp_ctx)) {
return false;
}
}
if (comp_ctx->is_jit_mode) {
LLVMErrorRef err;
LLVMOrcJITDylibRef orc_main_dylib;
LLVMOrcThreadSafeModuleRef orc_thread_safe_module;
orc_main_dylib = LLVMOrcLLLazyJITGetMainJITDylib(comp_ctx->orc_jit);
if (!orc_main_dylib) {
aot_set_last_error(
"failed to get orc orc_jit main dynmaic library");
return false;
}
}
#if WASM_ENABLE_LAZY_JIT != 0
orc_main_dylib = LLVMOrcLLJITGetMainJITDylib(comp_ctx->orc_lazyjit);
if (!orc_main_dylib) {
aot_set_last_error("failed to get orc jit main dynmaic library");
return false;
}
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
orc_thread_safe_module = LLVMOrcCreateNewThreadSafeModule(
comp_ctx->modules[i], comp_ctx->orc_thread_safe_context);
comp_ctx->module, comp_ctx->orc_thread_safe_context);
if (!orc_thread_safe_module) {
aot_set_last_error("failed to create thread safe module");
return false;
}
if ((err = LLVMOrcLLJITAddLLVMIRModule(comp_ctx->orc_lazyjit,
orc_main_dylib,
orc_thread_safe_module))) {
if ((err = LLVMOrcLLLazyJITAddLLVMIRModule(
comp_ctx->orc_jit, orc_main_dylib, orc_thread_safe_module))) {
/* If adding the ThreadSafeModule fails then we need to clean it up
by ourselves, otherwise the orc jit will manage the memory. */
by ourselves, otherwise the orc orc_jit will manage the memory.
*/
LLVMOrcDisposeThreadSafeModule(orc_thread_safe_module);
aot_handle_llvm_errmsg("failed to addIRModule", err);
return false;
}
}
#endif
#if 0
#if WASM_ENABLE_LAZY_JIT == 0
LLVMDumpModule(comp_ctx->module);
#else
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
LLVMDumpModule(comp_ctx->modules[i]);
os_printf("\n");
}
#endif
#endif
return true;
}
@ -2947,7 +2792,6 @@ aot_generate_tempfile_name(const char *prefix, const char *extension,
}
#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */
#if WASM_ENABLE_LAZY_JIT == 0
bool
aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name)
{
@ -3071,4 +2915,3 @@ aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name)
return true;
}
#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */

23
core/iwasm/compilation/aot_emit_aot_file.c
View File

@ -6,8 +6,6 @@
#include "aot_compiler.h"
#include "../aot/aot_runtime.h"
#if WASM_ENABLE_LAZY_JIT == 0
#define PUT_U64_TO_ADDR(addr, value) \
do { \
union { \
@ -2794,16 +2792,18 @@ aot_obj_data_create(AOTCompContext *comp_ctx)
}
#endif /* end of defined(_WIN32) || defined(_WIN32_) */
}
else if (LLVMTargetMachineEmitToMemoryBuffer(
comp_ctx->target_machine, comp_ctx->module, LLVMObjectFile,
&err, &obj_data->mem_buf)
!= 0) {
if (err) {
LLVMDisposeMessage(err);
err = NULL;
else {
if (LLVMTargetMachineEmitToMemoryBuffer(
comp_ctx->target_machine, comp_ctx->module, LLVMObjectFile,
&err, &obj_data->mem_buf)
!= 0) {
if (err) {
LLVMDisposeMessage(err);
err = NULL;
}
aot_set_last_error("llvm emit to memory buffer failed.");
goto fail;
}
aot_set_last_error("llvm emit to memory buffer failed.");
goto fail;
}
if (!(obj_data->binary = LLVMCreateBinary(obj_data->mem_buf, NULL, &err))) {
@ -2928,4 +2928,3 @@ fail1:
return ret;
}
#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */

164
core/iwasm/compilation/aot_emit_function.c
View File

@ -267,131 +267,6 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return true;
}
#if WASM_ENABLE_LAZY_JIT != 0
static bool
lookup_orcjit_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef func_idx, LLVMValueRef *p_func)
{
LLVMBasicBlockRef block_curr, block_resolve_func, block_func_resolved;
LLVMValueRef param_values[3], func, value, func_ptr, cmp, phi;
LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type;
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
if (!(block_resolve_func = LLVMAppendBasicBlockInContext(
comp_ctx->context, func_ctx->func, "resolve_func"))) {
aot_set_last_error("llvm add basic block failed.");
return false;
}
if (!(block_func_resolved = LLVMAppendBasicBlockInContext(
comp_ctx->context, func_ctx->func, "func_resolved"))) {
aot_set_last_error("llvm add basic block failed.");
return false;
}
LLVMMoveBasicBlockAfter(block_resolve_func, block_curr);
LLVMMoveBasicBlockAfter(block_func_resolved, block_resolve_func);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_func_resolved);
if (!(phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "phi"))) {
aot_set_last_error("llvm build phi failed.");
return false;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
/* Load function pointer */
if (!(func_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE,
func_ctx->func_ptrs, &func_idx, 1,
"func_ptr_tmp"))) {
aot_set_last_error("llvm build inbounds gep failed.");
return false;
}
if (!(func = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, func_ptr,
"func_ptr"))) {
aot_set_last_error("llvm build load failed.");
return false;
}
/* If func ptr is NULL, call aot_lookup_orcjit_func to resolve it */
if (!(cmp = LLVMBuildIsNull(comp_ctx->builder, func, "cmp"))) {
aot_set_last_error("llvm build is null failed");
return false;
}
/* Create condition br */
if (!LLVMBuildCondBr(comp_ctx->builder, cmp, block_resolve_func,
block_func_resolved)) {
aot_set_last_error("llvm build cond br failed.");
return false;
}
LLVMAddIncoming(phi, &func, &block_curr, 1);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_resolve_func);
param_types[0] = INT8_PTR_TYPE;
param_types[1] = comp_ctx->aot_inst_type;
param_types[2] = I32_TYPE;
ret_type = INT8_PTR_TYPE;
if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))
|| !(func_ptr_type = LLVMPointerType(func_type, 0))) {
aot_set_last_error("llvm add function type failed.");
return false;
}
if (!(value = I64_CONST((uint64)(uintptr_t)aot_lookup_orcjit_func))
|| !(func = LLVMConstIntToPtr(value, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
}
param_values[0] = I64_CONST((uintptr_t)comp_ctx->orc_lazyjit);
if (!param_values[0]) {
aot_set_last_error("llvm build const failed.");
return false;
}
if (!(param_values[0] =
LLVMConstIntToPtr(param_values[0], INT8_PTR_TYPE))) {
aot_set_last_error("llvm build bit cast failed.");
return false;
}
param_values[1] = func_ctx->aot_inst;
param_values[2] = func_idx;
if (!param_values[2]) {
aot_set_last_error("llvm build const failed.");
return false;
}
/* Call the function */
if (!(func = LLVMBuildCall2(comp_ctx->builder, func_type, func,
param_values, 3, "call_orcjit_lookup"))) {
aot_set_last_error("LLVM build call failed.");
return false;
}
/* Check whether exception was thrown when looking up func */
if (!check_exception_thrown(comp_ctx, func_ctx)) {
return false;
}
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMAddIncoming(phi, &func, &block_curr, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_func_resolved)) {
aot_set_last_error("llvm build br failed.");
return false;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_func_resolved);
*p_func = phi;
return true;
}
#endif
#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
static bool
call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
@ -909,43 +784,13 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
}
else {
#if WASM_ENABLE_LAZY_JIT == 0
func = func_ctxes[func_idx - import_func_count]->func;
#else
if (func_ctxes[func_idx - import_func_count] == func_ctx) {
/* recursive call */
func = func_ctx->func;
}
else {
LLVMTypeRef func_ptr_type;
LLVMValueRef func_idx_const = I32_CONST(func_idx);
if (!func_idx_const) {
aot_set_last_error("llvm build const failed.");
goto fail;
}
/* For LAZY JIT, each function belongs to its own module,
we call aot_lookup_orcjit_func to get the func pointer */
if (!lookup_orcjit_func(comp_ctx, func_ctx, func_idx_const,
&func)) {
goto fail;
}
if (!(func_ptr_type = LLVMPointerType(
func_ctxes[func_idx - import_func_count]->func_type,
0))) {
aot_set_last_error("construct func ptr type failed.");
goto fail;
}
if (!(func = LLVMBuildBitCast(comp_ctx->builder, func,
func_ptr_type, "aot_func"))) {
aot_set_last_error("llvm bit cast failed.");
goto fail;
}
func = func_ctxes[func_idx - import_func_count]->func;
}
#endif
}
aot_func = func_ctxes[func_idx - import_func_count]->aot_func;
@ -1574,7 +1419,6 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
+ 16))
goto fail;
#if WASM_ENABLE_LAZY_JIT == 0
/* Load function pointer */
if (!(func_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE,
func_ctx->func_ptrs, &func_idx, 1,
@ -1588,12 +1432,6 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
aot_set_last_error("llvm build load failed.");
goto fail;
}
#else
/* For LAZY JIT, each function belongs to its own module,
we call aot_lookup_orcjit_func to get the func pointer */
if (!lookup_orcjit_func(comp_ctx, func_ctx, func_idx, &func_ptr))
goto fail;
#endif
if (!(llvm_func_type =
LLVMFunctionType(ret_type, param_types, total_param_count, false))

49
core/iwasm/compilation/aot_emit_memory.c
View File

@ -977,14 +977,6 @@ fail:
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
static void *
jit_memmove(void *dest, const void *src, size_t n)
{
return memmove(dest, src, n);
}
#endif
bool
aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
@ -1001,18 +993,10 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len)))
return false;
#if WASM_ENABLE_LAZY_JIT != 0
call_aot_memmove = true;
#endif
if (comp_ctx->is_indirect_mode)
call_aot_memmove = true;
call_aot_memmove = comp_ctx->is_indirect_mode || comp_ctx->is_jit_mode;
if (call_aot_memmove) {
LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type;
LLVMValueRef func, params[3];
#if WASM_ENABLE_LAZY_JIT == 0
int32 func_idx;
#endif
param_types[0] = INT8_PTR_TYPE;
param_types[1] = INT8_PTR_TYPE;
@ -1029,22 +1013,25 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
return false;
}
#if WASM_ENABLE_LAZY_JIT == 0
func_idx = aot_get_native_symbol_index(comp_ctx, "memmove");
if (func_idx < 0) {
return false;
if (comp_ctx->is_jit_mode) {
if (!(func = I64_CONST((uint64)(uintptr_t)aot_memmove))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
}
}
if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol,
func_ptr_type, func_idx))) {
return false;
else {
int32 func_index;
func_index = aot_get_native_symbol_index(comp_ctx, "memmove");
if (func_index < 0) {
return false;
}
if (!(func =
aot_get_func_from_table(comp_ctx, func_ctx->native_symbol,
func_ptr_type, func_index))) {
return false;
}
}
#else
if (!(func = I64_CONST((uint64)(uintptr_t)jit_memmove))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
}
#endif
params[0] = dst_addr;
params[1] = src_addr;

478
core/iwasm/compilation/aot_llvm.c
View File

@ -48,7 +48,10 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module,
LLVMValueRef func = NULL;
LLVMTypeRef *param_types, ret_type, func_type;
LLVMValueRef local_value;
char func_name[32];
LLVMTypeRef func_type_wrapper;
LLVMValueRef func_wrapper;
LLVMBasicBlockRef func_begin;
char func_name[48];
uint64 size;
uint32 i, j = 0, param_count = (uint64)aot_func_type->param_count;
@ -116,6 +119,34 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module,
if (p_func_type)
*p_func_type = func_type;
if (comp_ctx->is_jit_mode) {
func_type_wrapper = LLVMFunctionType(VOID_TYPE, NULL, 0, false);
if (!func_type_wrapper) {
aot_set_last_error("create LLVM function type failed.");
goto fail;
}
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX,
func_index, "_wrapper");
if (!(func_wrapper =
LLVMAddFunction(module, func_name, func_type_wrapper))) {
aot_set_last_error("add LLVM function failed.");
goto fail;
}
if (!(func_begin = LLVMAppendBasicBlockInContext(
comp_ctx->context, func_wrapper, "func_begin"))) {
aot_set_last_error("add LLVM basic block failed.");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, func_begin);
if (!LLVMBuildRetVoid(comp_ctx->builder)) {
aot_set_last_error("llvm build ret failed.");
goto fail;
}
}
fail:
wasm_runtime_free(param_types);
return func;
@ -646,11 +677,7 @@ aot_create_func_context(AOTCompData *comp_data, AOTCompContext *comp_ctx,
memset(func_ctx, 0, (uint32)size);
func_ctx->aot_func = func;
#if WASM_ENABLE_LAZY_JIT == 0
func_ctx->module = comp_ctx->module;
#else
func_ctx->module = comp_ctx->modules[func_index];
#endif
/* Add LLVM function */
if (!(func_ctx->func =
@ -1244,16 +1271,6 @@ get_target_arch_from_triple(const char *triple, char *arch_buf, uint32 buf_size)
bh_assert(*triple == '-' || *triple == '\0');
}
LLVMBool
WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
LLVMModuleRef M,
struct LLVMMCJITCompilerOptions *Options,
size_t SizeOfOptions, char **OutError);
void
LLVMAddPromoteMemoryToRegisterPass(LLVMPassManagerRef PM);
#if WASM_ENABLE_LAZY_JIT != 0
void
aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err)
{
@ -1262,41 +1279,67 @@ aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err)
LLVMDisposeErrorMessage(err_msg);
}
static bool
orc_lazyjit_create(AOTCompContext *comp_ctx, uint32 func_count)
#ifndef NDEBUG
static LLVMErrorRef
run_pass(void *ctx, LLVMModuleRef module)
{
uint32 i;
/*AOTCompContext *comp_ctx = (AOTCompContext *)ctx;*/
size_t len;
LOG_VERBOSE("--- In IRTransformLayer @ T#%ld---",
LLVMGetModuleIdentifier(module, &len), pthread_self());
/* TODO: enable this for JIT mode after fixing LLVM issues */
/*aot_apply_llvm_new_pass_manager(comp_ctx, module);*/
bh_print_time("Begin to generate machine code");
return LLVMErrorSuccess;
}
static LLVMErrorRef
do_ir_transform(void *ctx, LLVMOrcThreadSafeModuleRef *module,
LLVMOrcMaterializationResponsibilityRef mr)
{
(void)mr;
return LLVMOrcThreadSafeModuleWithModuleDo(*module, run_pass, ctx);
}
static LLVMErrorRef
do_obj_transform(void *Ctx, LLVMMemoryBufferRef *ObjInOut)
{
bh_print_time("Finish generating machine code");
LOG_VERBOSE("--- In ObjectTransformLayer @ T#%ld ---", pthread_self());
(void)Ctx;
(void)ObjInOut;
return LLVMErrorSuccess;
}
#endif
static bool
create_target_machine_detect_host(AOTCompContext *comp_ctx)
{
char *triple = NULL;
LLVMTargetRef target = NULL;
char *err_msg = NULL;
char *cpu = NULL;
char *features = NULL;
char *llvm_triple = NULL;
char func_name[32] = { 0 };
LLVMErrorRef err;
LLVMTargetRef llvm_targetref = NULL;
LLVMTargetMachineRef target_machine_for_orcjit = NULL;
LLVMOrcLLJITRef orc_lazyjit = NULL;
LLVMOrcJITTargetMachineBuilderRef target_machine_builder = NULL;
LLVMOrcLLJITBuilderRef orc_lazyjit_builder = NULL;
LLVMOrcMaterializationUnitRef orc_material_unit = NULL;
LLVMOrcExecutionSessionRef orc_execution_session = NULL;
LLVMOrcLazyCallThroughManagerRef orc_call_through_mgr = NULL;
LLVMOrcIndirectStubsManagerRef orc_indirect_stub_mgr = NULL;
LLVMOrcCSymbolAliasMapPair *orc_symbol_map_pairs = NULL;
LLVMTargetMachineRef target_machine = NULL;
bool ret = false;
llvm_triple = LLVMGetDefaultTargetTriple();
if (llvm_triple == NULL) {
triple = LLVMGetDefaultTargetTriple();
if (triple == NULL) {
aot_set_last_error("failed to get default target triple.");
goto fail;
}
if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &err_msg) != 0) {
if (LLVMGetTargetFromTriple(triple, &target, &err_msg) != 0) {
aot_set_last_error_v("failed to get llvm target from triple %s.",
err_msg);
LLVMDisposeMessage(err_msg);
goto fail;
}
if (!LLVMTargetHasJIT(llvm_targetref)) {
if (!LLVMTargetHasJIT(target)) {
aot_set_last_error("unspported JIT on this platform.");
goto fail;
}
@ -1316,161 +1359,98 @@ orc_lazyjit_create(AOTCompContext *comp_ctx, uint32 func_count)
LOG_VERBOSE("LLVM ORCJIT detected CPU \"%s\", with features \"%s\"\n", cpu,
features);
comp_ctx->target_machine = LLVMCreateTargetMachine(
llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelDefault,
/* create TargetMachine */
target_machine = LLVMCreateTargetMachine(
target, triple, cpu, features, LLVMCodeGenLevelDefault,
LLVMRelocDefault, LLVMCodeModelJITDefault);
if (!comp_ctx->target_machine) {
if (!target_machine) {
aot_set_last_error("failed to create target machine.");
goto fail;
}
comp_ctx->target_machine = target_machine;
target_machine_for_orcjit = LLVMCreateTargetMachine(
llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelDefault,
LLVMRelocDefault, LLVMCodeModelJITDefault);
if (!target_machine_for_orcjit) {
aot_set_last_error("failed to create target machine.");
goto fail;
}
target_machine_builder =
LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(
target_machine_for_orcjit);
if (!target_machine_builder) {
aot_set_last_error("failed to create target machine builder.");
goto fail;
}
/* The target_machine_for_orcjit has been disposed before
LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine() returns */
target_machine_for_orcjit = NULL;
orc_lazyjit_builder = LLVMOrcCreateLLJITBuilder();
if (!orc_lazyjit_builder) {
aot_set_last_error("failed to create lazy jit builder.");
goto fail;
}
LLVMOrcLLJITBuilderSetNumCompileThreads(orc_lazyjit_builder,
WASM_LAZY_JIT_COMPILE_THREAD_NUM);
LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(orc_lazyjit_builder,
target_machine_builder);
/* Should not dispose of the JITTargetMachineBuilder after calling
LLVMOrcLLJITBuilderSetJITTargetMachineBuilder() */
target_machine_builder = NULL;
err = LLVMOrcCreateLLJIT(&orc_lazyjit, orc_lazyjit_builder);
if (err) {
aot_handle_llvm_errmsg("failed to create llvm lazy orcjit instance",
err);
goto fail;
}
/* The orc_lazyjit_builder is managed by orc_lazyjit after calling
LLVMOrcCreateLLJIT(), here we should not dispose it again */
orc_lazyjit_builder = NULL;
if (func_count > 0) {
orc_execution_session = LLVMOrcLLJITGetExecutionSession(orc_lazyjit);
if (!orc_execution_session) {
aot_set_last_error("failed to get orc execution session");
goto fail;
}
err = LLVMOrcCreateLocalLazyCallThroughManager(
llvm_triple, orc_execution_session, 0, &orc_call_through_mgr);
if (err) {
aot_handle_llvm_errmsg("failed to create orc call through manager",
err);
goto fail;
}
orc_indirect_stub_mgr =
LLVMOrcCreateLocalIndirectStubsManager(llvm_triple);
if (!orc_indirect_stub_mgr) {
aot_set_last_error("failed to create orc indirect stub manager");
goto fail;
}
if (!(orc_symbol_map_pairs = wasm_runtime_malloc(
sizeof(LLVMOrcCSymbolAliasMapPair) * func_count))) {
aot_set_last_error("failed to allocate memory");
goto fail;
}
memset(orc_symbol_map_pairs, 0,
sizeof(LLVMOrcCSymbolAliasMapPair) * func_count);
for (i = 0; i < func_count; i++) {
snprintf(func_name, sizeof(func_name), "orcjit_%s%d",
AOT_FUNC_PREFIX, i);
orc_symbol_map_pairs[i].Name =
LLVMOrcExecutionSessionIntern(orc_execution_session, func_name);
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
orc_symbol_map_pairs[i].Entry.Name =
LLVMOrcExecutionSessionIntern(orc_execution_session, func_name);
orc_symbol_map_pairs[i].Entry.Flags.GenericFlags =
LLVMJITSymbolGenericFlagsExported
| LLVMJITSymbolGenericFlagsCallable;
orc_symbol_map_pairs[i].Entry.Flags.TargetFlags =
LLVMJITSymbolGenericFlagsExported
| LLVMJITSymbolGenericFlagsCallable;
if (!orc_symbol_map_pairs[i].Name
|| !orc_symbol_map_pairs[i].Entry.Name) {
aot_set_last_error("failed to allocate memory");
goto fail;
}
}
orc_material_unit =
LLVMOrcLazyReexports(orc_call_through_mgr, orc_indirect_stub_mgr,
LLVMOrcLLJITGetMainJITDylib(orc_lazyjit),
orc_symbol_map_pairs, func_count);
if (!orc_material_unit) {
aot_set_last_error("failed to orc re-exports");
goto fail;
}
}
comp_ctx->orc_lazyjit = orc_lazyjit;
comp_ctx->orc_material_unit = orc_material_unit;
comp_ctx->orc_symbol_map_pairs = orc_symbol_map_pairs;
comp_ctx->orc_call_through_mgr = orc_call_through_mgr;
comp_ctx->orc_indirect_stub_mgr = orc_indirect_stub_mgr;
LLVMDisposeMessage(llvm_triple);
LLVMDisposeMessage(cpu);
LLVMDisposeMessage(features);
return true;
/* Save target arch */
get_target_arch_from_triple(triple, comp_ctx->target_arch,
sizeof(comp_ctx->target_arch));
ret = true;
fail:
if (orc_symbol_map_pairs)
wasm_runtime_free(orc_symbol_map_pairs);
if (orc_call_through_mgr)
LLVMOrcDisposeLazyCallThroughManager(orc_call_through_mgr);
if (orc_indirect_stub_mgr)
LLVMOrcDisposeIndirectStubsManager(orc_indirect_stub_mgr);
if (orc_lazyjit)
LLVMOrcDisposeLLJIT(orc_lazyjit);
if (target_machine_builder)
LLVMOrcDisposeJITTargetMachineBuilder(target_machine_builder);
if (orc_lazyjit_builder)
LLVMOrcDisposeLLJITBuilder(orc_lazyjit_builder);
if (target_machine_for_orcjit)
LLVMDisposeTargetMachine(target_machine_for_orcjit);
if (triple)
LLVMDisposeMessage(triple);
if (features)
LLVMDisposeMessage(features);
if (cpu)
LLVMDisposeMessage(cpu);
if (llvm_triple)
LLVMDisposeMessage(llvm_triple);
return false;
return ret;
}
static bool
orc_jit_create(AOTCompContext *comp_ctx)
{
LLVMErrorRef err;
LLVMOrcLLLazyJITRef orc_jit = NULL;
LLVMOrcLLLazyJITBuilderRef builder = NULL;
LLVMOrcJITTargetMachineBuilderRef jtmb = NULL;
bool ret = false;
builder = LLVMOrcCreateLLLazyJITBuilder();
if (builder == NULL) {
aot_set_last_error("failed to create jit builder.");
goto fail;
}
err = LLVMOrcJITTargetMachineBuilderDetectHost(&jtmb);
if (err != LLVMErrorSuccess) {
aot_handle_llvm_errmsg(
"quited to create LLVMOrcJITTargetMachineBuilderRef", err);
goto fail;
}
LLVMOrcLLLazyJITBuilderSetNumCompileThreads(
builder, WASM_ORC_JIT_COMPILE_THREAD_NUM);
/* Ownership transfer:
LLVMOrcJITTargetMachineBuilderRef -> LLVMOrcLLJITBuilderRef */
LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder(builder, jtmb);
err = LLVMOrcCreateLLLazyJIT(&orc_jit, builder);
if (err != LLVMErrorSuccess) {
aot_handle_llvm_errmsg("quited to create llvm lazy orcjit instance",
err);
goto fail;
}
/* Ownership transfer: LLVMOrcLLJITBuilderRef -> LLVMOrcLLJITRef */
builder = NULL;
#ifndef NDEBUG
/* Setup TransformLayer */
LLVMOrcIRTransformLayerSetTransform(
LLVMOrcLLLazyJITGetIRTransformLayer(orc_jit), *do_ir_transform,
comp_ctx);
LLVMOrcObjectTransformLayerSetTransform(
LLVMOrcLLLazyJITGetObjTransformLayer(orc_jit), *do_obj_transform,
comp_ctx);
#endif
/* Ownership transfer: local -> AOTCompContext */
comp_ctx->orc_jit = orc_jit;
orc_jit = NULL;
ret = true;
fail:
if (builder)
LLVMOrcDisposeLLLazyJITBuilder(builder);
if (orc_jit)
LLVMOrcDisposeLLLazyJIT(orc_jit);
return ret;
}
#endif /* WASM_ENABLE_LAZY_JIT != 0 */
AOTCompContext *
aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
{
AOTCompContext *comp_ctx, *ret = NULL;
#if WASM_ENABLE_LAZY_JIT == 0
struct LLVMMCJITCompilerOptions jit_options;
#endif
LLVMTargetRef target;
char *triple = NULL, *triple_norm, *arch, *abi;
char *cpu = NULL, *features, buf[128];
@ -1483,18 +1463,13 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
LLVMTargetDataRef target_data_ref;
/* Initialize LLVM environment */
#if WASM_ENABLE_LAZY_JIT != 0
LLVMInitializeCore(LLVMGetGlobalPassRegistry());
LLVMInitializeNativeTarget();
LLVMInitializeNativeAsmPrinter();
LLVMInitializeNativeAsmParser();
#else
/* To all available target */
LLVMInitializeAllTargetInfos();
LLVMInitializeAllTargets();
LLVMInitializeAllTargetMCs();
LLVMInitializeAllAsmPrinters();
LLVMLinkInMCJIT();
#endif
LLVMInitializeAllAsmParsers();
/* Allocate memory */
if (!(comp_ctx = wasm_runtime_malloc(sizeof(AOTCompContext)))) {
@ -1506,7 +1481,6 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
comp_ctx->comp_data = comp_data;
/* Create LLVM context, module and builder */
#if WASM_ENABLE_LAZY_JIT != 0
comp_ctx->orc_thread_safe_context = LLVMOrcCreateNewThreadSafeContext();
if (!comp_ctx->orc_thread_safe_context) {
aot_set_last_error("create LLVM ThreadSafeContext failed.");
@ -1521,53 +1495,26 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
aot_set_last_error("get context from LLVM ThreadSafeContext failed.");
goto fail;
}
#else
if (!(comp_ctx->context = LLVMContextCreate())) {
aot_set_last_error("create LLVM context failed.");
goto fail;
}
#endif
if (!(comp_ctx->builder = LLVMCreateBuilderInContext(comp_ctx->context))) {
aot_set_last_error("create LLVM builder failed.");
goto fail;
}
#if WASM_ENABLE_LAZY_JIT == 0
/* Create LLVM module for each jit function, note:
different from non ORC JIT mode, no need to dispose it,
it will be disposed when the thread safe context is disposed */
if (!(comp_ctx->module = LLVMModuleCreateWithNameInContext(
"WASM Module", comp_ctx->context))) {
aot_set_last_error("create LLVM module failed.");
goto fail;
}
#else
if (comp_data->func_count > 0) {
if (!(comp_ctx->modules = wasm_runtime_malloc(
sizeof(LLVMModuleRef) * comp_data->func_count))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
memset(comp_ctx->modules, 0,
sizeof(LLVMModuleRef) * comp_data->func_count);
for (i = 0; i < comp_data->func_count; i++) {
char module_name[32];
snprintf(module_name, sizeof(module_name), "WASM Module %d", i);
/* Create individual modules for each aot function, note:
different from non LAZY JIT mode, no need to dispose them,
they will be disposed when the thread safe context is disposed */
if (!(comp_ctx->modules[i] = LLVMModuleCreateWithNameInContext(
module_name, comp_ctx->context))) {
aot_set_last_error("create LLVM module failed.");
goto fail;
}
}
}
#endif
if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) {
goto fail;
}
#if WASM_ENABLE_DEBUG_AOT != 0 && WASM_ENABLE_LAZY_JIT == 0
#if WASM_ENABLE_DEBUG_AOT != 0
if (!(comp_ctx->debug_builder = LLVMCreateDIBuilder(comp_ctx->module))) {
aot_set_last_error("create LLVM Debug Infor builder failed.");
goto fail;
@ -1624,65 +1571,21 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
comp_ctx->custom_sections_count = option->custom_sections_count;
if (option->is_jit_mode) {
char *triple_jit = NULL;
comp_ctx->is_jit_mode = true;
#if WASM_ENABLE_LAZY_JIT != 0
/* Create LLJIT Instance */
if (!orc_lazyjit_create(comp_ctx, comp_data->func_count)) {
/* Create TargetMachine */
if (!create_target_machine_detect_host(comp_ctx))
goto fail;
}
#else
/* Create LLVM execution engine */
LLVMInitializeMCJITCompilerOptions(&jit_options, sizeof(jit_options));
jit_options.OptLevel = LLVMCodeGenLevelAggressive;
jit_options.EnableFastISel = true;
/*jit_options.CodeModel = LLVMCodeModelSmall;*/
if (WAMRCreateMCJITCompilerForModule(&comp_ctx->exec_engine,
comp_ctx->module, &jit_options,
sizeof(jit_options), &err)
!= 0) {
if (err) {
LLVMDisposeMessage(err);
err = NULL;
}
aot_set_last_error("create LLVM JIT compiler failed.");
/* Create LLJIT Instance */
if (!orc_jit_create(comp_ctx))
goto fail;
}
comp_ctx->target_machine =
LLVMGetExecutionEngineTargetMachine(comp_ctx->exec_engine);
#endif
#ifndef OS_ENABLE_HW_BOUND_CHECK
comp_ctx->enable_bound_check = true;
#else
comp_ctx->enable_bound_check = false;
#endif
#if WASM_ENABLE_LAZY_JIT != 0
if (!(triple_jit =
(char *)LLVMOrcLLJITGetTripleString(comp_ctx->orc_lazyjit))) {
aot_set_last_error("can not get triple from the target machine");
goto fail;
}
/* Save target arch */
get_target_arch_from_triple(triple_jit, comp_ctx->target_arch,
sizeof(comp_ctx->target_arch));
#else
if (!(triple_jit =
LLVMGetTargetMachineTriple(comp_ctx->target_machine))) {
aot_set_last_error("can not get triple from the target machine");
goto fail;
}
/* Save target arch */
get_target_arch_from_triple(triple_jit, comp_ctx->target_arch,
sizeof(comp_ctx->target_arch));
LLVMDisposeMessage(triple_jit);
#endif
}
else {
/* Create LLVM target machine */
@ -1942,17 +1845,9 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
aot_set_last_error("create metadata string failed.");
goto fail;
}
#if WASM_ENABLE_LAZY_JIT == 0
LLVMAddModuleFlag(comp_ctx->module, LLVMModuleFlagBehaviorError,
"target-abi", strlen("target-abi"),
meta_target_abi);
#else
for (i = 0; i < comp_data->func_count; i++) {
LLVMAddModuleFlag(comp_ctx->modules[i],
LLVMModuleFlagBehaviorError, "target-abi",
strlen("target-abi"), meta_target_abi);
}
#endif
if (!strcmp(abi, "lp64d") || !strcmp(abi, "ilp32d")) {
if (features) {
@ -2059,10 +1954,6 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
aot_set_last_error("create LLVM target machine failed.");
goto fail;
}
#if WASM_ENABLE_LAZY_JIT == 0
LLVMSetTarget(comp_ctx->module, triple_norm);
#endif
}
if (option->enable_simd && strcmp(comp_ctx->target_arch, "x86_64") != 0
@ -2171,56 +2062,23 @@ aot_destroy_comp_context(AOTCompContext *comp_ctx)
if (!comp_ctx)
return;
#if WASM_ENABLE_LAZY_JIT != 0
if (comp_ctx->orc_symbol_map_pairs)
wasm_runtime_free(comp_ctx->orc_symbol_map_pairs);
if (comp_ctx->orc_call_through_mgr)
LLVMOrcDisposeLazyCallThroughManager(comp_ctx->orc_call_through_mgr);
if (comp_ctx->orc_indirect_stub_mgr)
LLVMOrcDisposeIndirectStubsManager(comp_ctx->orc_indirect_stub_mgr);
if (comp_ctx->orc_material_unit)
LLVMOrcDisposeMaterializationUnit(comp_ctx->orc_material_unit);
if (comp_ctx->target_machine)
LLVMDisposeTargetMachine(comp_ctx->target_machine);
if (comp_ctx->builder)
LLVMDisposeBuilder(comp_ctx->builder);
if (comp_ctx->orc_lazyjit)
LLVMOrcDisposeLLJIT(comp_ctx->orc_lazyjit);
if (comp_ctx->orc_thread_safe_context)
LLVMOrcDisposeThreadSafeContext(comp_ctx->orc_thread_safe_context);
if (comp_ctx->modules)
wasm_runtime_free(comp_ctx->modules);
/* Note: don't dispose comp_ctx->context and comp_ctx->modules[i] as
/* Note: don't dispose comp_ctx->context and comp_ctx->module as
they are disposed when disposing the thread safe context */
/* Has to be the last one */
if (comp_ctx->orc_jit)
LLVMOrcDisposeLLLazyJIT(comp_ctx->orc_jit);
LLVMShutdown();
#else
if (comp_ctx->target_machine && !comp_ctx->is_jit_mode)
LLVMDisposeTargetMachine(comp_ctx->target_machine);
if (comp_ctx->builder)
LLVMDisposeBuilder(comp_ctx->builder);
if (comp_ctx->exec_engine) {
LLVMDisposeExecutionEngine(comp_ctx->exec_engine);
/* The LLVM module is freed when disposing execution engine,
no need to dispose it again. */
}
else if (comp_ctx->module)
LLVMDisposeModule(comp_ctx->module);
if (comp_ctx->context)
LLVMContextDispose(comp_ctx->context);
#endif
if (comp_ctx->func_ctxes)
aot_destroy_func_contexts(comp_ctx->func_ctxes,

54
core/iwasm/compilation/aot_llvm.h
View File

@ -20,20 +20,18 @@
#include "llvm-c/Transforms/Vectorize.h"
#include "llvm-c/Transforms/PassManagerBuilder.h"
#if WASM_ENABLE_LAZY_JIT != 0
#include "llvm-c/Orc.h"
#include "llvm-c/Error.h"
#include "llvm-c/Support.h"
#include "llvm-c/Initialization.h"
#include "llvm-c/TargetMachine.h"
#if LLVM_VERSION_MAJOR >= 12
#include "llvm-c/LLJIT.h"
#endif
#endif
#if WASM_ENABLE_DEBUG_AOT != 0
#include "llvm-c/DebugInfo.h"
#endif
#include "aot_orc_extra.h"
#ifdef __cplusplus
extern "C" {
#endif
@ -52,6 +50,16 @@ extern "C" {
#define OPQ_PTR_TYPE INT8_PTR_TYPE
#endif
#ifndef NDEBUG
#undef DEBUG_PASS
#undef DUMP_MODULE
// #define DEBUG_PASS
// #define DUMP_MODULE
#else
#undef DEBUG_PASS
#undef DUMP_MODULE
#endif
/**
* Value in the WASM operation stack, each stack element
* is an LLVM value
@ -270,12 +278,6 @@ typedef struct AOTCompContext {
/* LLVM variables required to emit LLVM IR */
LLVMContextRef context;
#if WASM_ENABLE_LAZY_JIT == 0
/* Create one module only for non LAZY JIT mode,
for LAZY JIT mode, modules are created, each
aot function has its own module */
LLVMModuleRef module;
#endif
LLVMBuilderRef builder;
#if WASM_ENABLE_DEBUG_AOT
LLVMDIBuilderRef debug_builder;
@ -290,19 +292,11 @@ typedef struct AOTCompContext {
/* Hardware intrinsic compability flags */
uint64 flags[8];
/* LLVM execution engine required by JIT */
#if WASM_ENABLE_LAZY_JIT != 0
LLVMOrcLLJITRef orc_lazyjit;
LLVMOrcMaterializationUnitRef orc_material_unit;
LLVMOrcLazyCallThroughManagerRef orc_call_through_mgr;
LLVMOrcIndirectStubsManagerRef orc_indirect_stub_mgr;
LLVMOrcCSymbolAliasMapPairs orc_symbol_map_pairs;
/* required by JIT */
LLVMOrcLLLazyJITRef orc_jit;
LLVMOrcThreadSafeContextRef orc_thread_safe_context;
/* Each aot function has its own module */
LLVMModuleRef *modules;
#else
LLVMExecutionEngineRef exec_engine;
#endif
LLVMModuleRef module;
bool is_jit_mode;
@ -361,6 +355,7 @@ typedef struct AOTCompContext {
AOTLLVMConsts llvm_consts;
/* Function contexts */
/* TODO: */
AOTFuncContext **func_ctxes;
uint32 func_ctx_count;
char **custom_sections_wp;
@ -503,24 +498,11 @@ void
aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass);
void
aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx);
#if WASM_ENABLE_LAZY_JIT != 0
LLVMOrcJITTargetMachineBuilderRef
LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM);
void
LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef orcjit_builder,
unsigned num_compile_threads);
aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module);
void
aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err);
void *
aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst,
uint32 func_idx);
#endif
#ifdef __cplusplus
} /* end of extern "C" */
#endif

256
core/iwasm/compilation/aot_llvm_extra.cpp
View File

@ -3,6 +3,8 @@
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include <llvm/Passes/StandardInstrumentations.h>
#include <llvm/Support/Error.h>
#include <llvm/ADT/SmallVector.h>
#include <llvm/ADT/Twine.h>
#include <llvm/ADT/Triple.h>
@ -42,23 +44,15 @@
#if LLVM_VERSION_MAJOR >= 12
#include <llvm/Analysis/AliasAnalysis.h>
#endif
#include <cstring>
#if WASM_ENABLE_LAZY_JIT != 0
#include "../aot/aot_runtime.h"
#endif
#include <cstring>
#include "../aot/aot_runtime.h"
#include "aot_llvm.h"
using namespace llvm;
using namespace llvm::orc;
extern "C" {
LLVMBool
WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
LLVMModuleRef M,
LLVMMCJITCompilerOptions *PassedOptions,
size_t SizeOfPassedOptions, char **OutError);
LLVM_C_EXTERN_C_BEGIN
bool
aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str);
@ -70,93 +64,11 @@ void
aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass);
void
aot_func_disable_tce(LLVMValueRef func);
aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module);
void
aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx);
}
LLVM_C_EXTERN_C_END
static TargetMachine *
unwrap(LLVMTargetMachineRef P)
{
return reinterpret_cast<TargetMachine *>(P);
}
LLVMBool
WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
LLVMModuleRef M,
LLVMMCJITCompilerOptions *PassedOptions,
size_t SizeOfPassedOptions, char **OutError)
{
LLVMMCJITCompilerOptions options;
// If the user passed a larger sized options struct, then they were compiled
// against a newer LLVM. Tell them that something is wrong.
if (SizeOfPassedOptions > sizeof(options)) {
*OutError = strdup("Refusing to use options struct that is larger than "
"my own; assuming LLVM library mismatch.");
return 1;
}
// Defend against the user having an old version of the API by ensuring that
// any fields they didn't see are cleared. We must defend against fields
// being set to the bitwise equivalent of zero, and assume that this means
// "do the default" as if that option hadn't been available.
LLVMInitializeMCJITCompilerOptions(&options, sizeof(options));
memcpy(&options, PassedOptions, SizeOfPassedOptions);
TargetOptions targetOptions;
targetOptions.EnableFastISel = options.EnableFastISel;
std::unique_ptr<Module> Mod(unwrap(M));
if (Mod) {
// Set function attribute "frame-pointer" based on
// NoFramePointerElim.
for (auto &F : *Mod) {
auto Attrs = F.getAttributes();
StringRef Value = options.NoFramePointerElim ? "all" : "none";
#if LLVM_VERSION_MAJOR <= 13
Attrs =
Attrs.addAttribute(F.getContext(), AttributeList::FunctionIndex,
"frame-pointer", Value);
#else
Attrs = Attrs.addAttributeAtIndex(F.getContext(),
AttributeList::FunctionIndex,
"frame-pointer", Value);
#endif
F.setAttributes(Attrs);
}
}
std::string Error;
bool JIT;
char *host_cpu = LLVMGetHostCPUName();
if (!host_cpu) {
*OutError = NULL;
return false;
}
std::string mcpu(host_cpu);
LLVMDisposeMessage(host_cpu);
EngineBuilder builder(std::move(Mod));
builder.setEngineKind(EngineKind::JIT)
.setErrorStr(&Error)
.setMCPU(mcpu)
.setOptLevel((CodeGenOpt::Level)options.OptLevel)
.setTargetOptions(targetOptions);
if (Optional<CodeModel::Model> CM = unwrap(options.CodeModel, JIT))
builder.setCodeModel(*CM);
if (options.MCJMM)
builder.setMCJITMemoryManager(
std::unique_ptr<RTDyldMemoryManager>(unwrap(options.MCJMM)));
if (ExecutionEngine *JIT = builder.create()) {
*OutJIT = wrap(JIT);
return 0;
}
*OutError = strdup(Error.c_str());
return 1;
}
ExitOnError ExitOnErr;
class ExpandMemoryOpPass : public llvm::ModulePass
{
@ -258,13 +170,15 @@ ExpandMemoryOpPass::runOnModule(Module &M)
void
aot_add_expand_memory_op_pass(LLVMPassManagerRef pass)
{
unwrap(pass)->add(new ExpandMemoryOpPass());
reinterpret_cast<legacy::PassManager *>(pass)->add(
new ExpandMemoryOpPass());
}
void
aot_add_simple_loop_unswitch_pass(LLVMPassManagerRef pass)
{
unwrap(pass)->add(createSimpleLoopUnswitchLegacyPass());
reinterpret_cast<legacy::PassManager *>(pass)->add(
createSimpleLoopUnswitchLegacyPass());
}
bool
@ -308,123 +222,54 @@ aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str)
#endif /* WASM_ENABLE_SIMD */
}
#if WASM_ENABLE_LAZY_JIT != 0
#if LLVM_VERSION_MAJOR < 12
LLVMOrcJITTargetMachineBuilderRef
LLVMOrcJITTargetMachineBuilderFromTargetMachine(LLVMTargetMachineRef TM);
LLVMOrcJITTargetMachineBuilderRef
LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM)
{
return LLVMOrcJITTargetMachineBuilderFromTargetMachine(TM);
}
#endif
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef)
void
LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef orcjit_builder,
unsigned num_compile_threads)
aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module)
{
unwrap(orcjit_builder)->setNumCompileThreads(num_compile_threads);
}
void *
aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst,
uint32 func_idx)
{
char func_name[32], buf[128], *err_msg = NULL;
LLVMErrorRef error;
LLVMOrcJITTargetAddress func_addr = 0;
AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst;
AOTModule *aot_module = (AOTModule *)aot_inst->module;
void **func_ptrs = aot_inst->func_ptrs;
/**
* No need to lock the func_ptr[func_idx] here as it is basic
* data type, the load/store for it can be finished by one cpu
* instruction, and there can be only one cpu instruction
* loading/storing at the same time.
*/
if (func_ptrs[func_idx])
return func_ptrs[func_idx];
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX,
func_idx - aot_module->import_func_count);
if ((error = LLVMOrcLLJITLookup(orc_lazyjit, &func_addr, func_name))) {
err_msg = LLVMGetErrorMessage(error);
snprintf(buf, sizeof(buf), "failed to lookup orcjit function: %s",
err_msg);
aot_set_exception(aot_inst, buf);
LLVMDisposeErrorMessage(err_msg);
return NULL;
}
func_ptrs[func_idx] = (void *)func_addr;
return (void *)func_addr;
}
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
void
aot_func_disable_tce(LLVMValueRef func)
{
Function *F = unwrap<Function>(func);
auto Attrs = F->getAttributes();
#if LLVM_VERSION_MAJOR <= 13
Attrs = Attrs.addAttribute(F->getContext(), AttributeList::FunctionIndex,
"disable-tail-calls", "true");
#else
Attrs =
Attrs.addAttributeAtIndex(F->getContext(), AttributeList::FunctionIndex,
"disable-tail-calls", "true");
#endif
F->setAttributes(Attrs);
}
#if LLVM_VERSION_MAJOR >= 12
void
aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx)
{
Module *M;
TargetMachine *TM = unwrap(comp_ctx->target_machine);
bool disable_llvm_lto = false;
LoopAnalysisManager LAM;
FunctionAnalysisManager FAM;
CGSCCAnalysisManager CGAM;
ModuleAnalysisManager MAM;
TargetMachine *TM =
reinterpret_cast<TargetMachine *>(comp_ctx->target_machine);
PipelineTuningOptions PTO;
PTO.LoopVectorization = true;
PTO.SLPVectorization = true;
PTO.LoopUnrolling = true;
#ifdef DEBUG_PASS
PassInstrumentationCallbacks PIC;
PassBuilder PB(TM, PTO, None, &PIC);
#else
#if LLVM_VERSION_MAJOR == 12
PassBuilder PB(false, TM, PTO);
#else
PassBuilder PB(TM, PTO);
#endif
#endif
// Register the target library analysis directly and give it a
// customized preset TLI.
/* Register all the basic analyses with the managers */
LoopAnalysisManager LAM;
FunctionAnalysisManager FAM;
CGSCCAnalysisManager CGAM;
ModuleAnalysisManager MAM;
/* Register the target library analysis directly and give it a
customized preset TLI */
std::unique_ptr<TargetLibraryInfoImpl> TLII(
new TargetLibraryInfoImpl(Triple(TM->getTargetTriple())));
FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
// Register the AA manager first so that our version is the one used.
/* Register the AA manager first so that our version is the one used */
AAManager AA = PB.buildDefaultAAPipeline();
FAM.registerPass([&] { return std::move(AA); });
// Register all the basic analyses with the managers.
PB.registerModuleAnalyses(MAM);
PB.registerCGSCCAnalyses(CGAM);
#ifdef DEBUG_PASS
StandardInstrumentations SI(true, false);
SI.registerCallbacks(PIC, &FAM);
#endif
PB.registerFunctionAnalyses(FAM);
PB.registerLoopAnalyses(LAM);
PB.registerModuleAnalyses(MAM);
PB.registerCGSCCAnalyses(CGAM);
PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
ModulePassManager MPM;
#if LLVM_VERSION_MAJOR <= 13
PassBuilder::OptimizationLevel OL;
@ -463,25 +308,23 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx)
}
#endif /* end of LLVM_VERSION_MAJOR */
if (comp_ctx->disable_llvm_lto) {
disable_llvm_lto = true;
}
bool disable_llvm_lto = comp_ctx->disable_llvm_lto;
#if WASM_ENABLE_SPEC_TEST != 0
disable_llvm_lto = true;
#endif
Module *M = reinterpret_cast<Module *>(module);
if (disable_llvm_lto) {
uint32 i;
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
aot_func_disable_tce(comp_ctx->func_ctxes[i]->func);
for (Function &F : *M) {
F.addFnAttr("disable-tail-calls", "true");
}
}
ModulePassManager MPM;
if (comp_ctx->is_jit_mode) {
/* Apply normal pipeline for JIT mode, without
Vectorize related passes, without LTO */
MPM.addPass(PB.buildPerModuleDefaultPipeline(OL));
const char *Passes =
"mem2reg,instcombine,simplifycfg,jump-threading,indvars";
ExitOnErr(PB.parsePassPipeline(MPM, Passes));
}
else {
FunctionPassManager FPM;
@ -512,16 +355,5 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx)
}
}
#if WASM_ENABLE_LAZY_JIT == 0
M = unwrap(comp_ctx->module);
MPM.run(*M, MAM);
#else
uint32 i;
for (i = 0; i < comp_ctx->func_ctx_count; i++) {
M = unwrap(comp_ctx->modules[i]);
MPM.run(*M, MAM);
}
#endif
}
#endif /* end of LLVM_VERSION_MAJOR >= 12 */

289
core/iwasm/compilation/aot_orc_extra.cpp Normal file
View File

@ -0,0 +1,289 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "llvm-c/LLJIT.h"
#include "llvm-c/Orc.h"
#include "llvm-c/OrcEE.h"
#include "llvm-c/TargetMachine.h"
#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
#include "llvm/ExecutionEngine/Orc/LLJIT.h"
#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/Support/CBindingWrapping.h"
#include "aot_orc_extra.h"
#include "aot.h"
using namespace llvm;
using namespace llvm::orc;
using GlobalValueSet = std::set<const GlobalValue *>;
namespace llvm {
namespace orc {
class InProgressLookupState;
class OrcV2CAPIHelper
{
public:
using PoolEntry = SymbolStringPtr::PoolEntry;
using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr;
// Move from SymbolStringPtr to PoolEntryPtr (no change in ref count).
static PoolEntryPtr moveFromSymbolStringPtr(SymbolStringPtr S)
{
PoolEntryPtr Result = nullptr;
std::swap(Result, S.S);
return Result;
}
// Move from a PoolEntryPtr to a SymbolStringPtr (no change in ref count).
static SymbolStringPtr moveToSymbolStringPtr(PoolEntryPtr P)
{
SymbolStringPtr S;
S.S = P;
return S;
}
// Copy a pool entry to a SymbolStringPtr (increments ref count).
static SymbolStringPtr copyToSymbolStringPtr(PoolEntryPtr P)
{
return SymbolStringPtr(P);
}
static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S)
{
return S.S;
}
static void retainPoolEntry(PoolEntryPtr P)
{
SymbolStringPtr S(P);
S.S = nullptr;
}
static void releasePoolEntry(PoolEntryPtr P)
{
SymbolStringPtr S;
S.S = P;
}
static InProgressLookupState *extractLookupState(LookupState &LS)
{
return LS.IPLS.release();
}
static void resetLookupState(LookupState &LS, InProgressLookupState *IPLS)
{
return LS.reset(IPLS);
}
};
} // namespace orc
} // namespace llvm
// ORC.h
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRTransformLayer, LLVMOrcIRTransformLayerRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib, LLVMOrcJITDylibRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITTargetMachineBuilder,
LLVMOrcJITTargetMachineBuilderRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectTransformLayer,
LLVMOrcObjectTransformLayerRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry,
LLVMOrcSymbolStringPoolEntryRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef)
// LLJIT.h
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLLazyJITBuilder, LLVMOrcLLLazyJITBuilderRef)
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLLazyJIT, LLVMOrcLLLazyJITRef)
void
LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef Builder,
unsigned NumCompileThreads)
{
unwrap(Builder)->setNumCompileThreads(NumCompileThreads);
}
LLVMOrcLLLazyJITBuilderRef
LLVMOrcCreateLLLazyJITBuilder(void)
{
return wrap(new LLLazyJITBuilder());
}
void
LLVMOrcDisposeLLLazyJITBuilder(LLVMOrcLLLazyJITBuilderRef Builder)
{
delete unwrap(Builder);
}
void
LLVMOrcLLLazyJITBuilderSetNumCompileThreads(LLVMOrcLLLazyJITBuilderRef Builder,
unsigned NumCompileThreads)
{
unwrap(Builder)->setNumCompileThreads(NumCompileThreads);
}
void
LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder(
LLVMOrcLLLazyJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMP)
{
unwrap(Builder)->setJITTargetMachineBuilder(*unwrap(JTMP));
/* Destroy the JTMP, similar to
LLVMOrcLLJITBuilderSetJITTargetMachineBuilder */
LLVMOrcDisposeJITTargetMachineBuilder(JTMP);
}
static Optional<CompileOnDemandLayer::GlobalValueSet>
PartitionFunction(GlobalValueSet Requested)
{
std::vector<const GlobalValue *> GVsToAdd;
for (auto *GV : Requested) {
if (isa<Function>(GV) && GV->hasName()) {
auto &F = cast<Function>(*GV); /* get LLVM function */
const Module *M = F.getParent(); /* get LLVM module */
auto GVName = GV->getName(); /* get the function name */
const char *gvname = GVName.begin(); /* C function name */
const char *wrapper;
uint32 prefix_len = strlen(AOT_FUNC_PREFIX);
/* Convert "aot_func#n_wrapper" to "aot_func#n" */
if (strstr(gvname, AOT_FUNC_PREFIX)
&& (wrapper = strstr(gvname + prefix_len, "_wrapper"))) {
char buf[16] = { 0 };
char func_name[64];
int group_stride, i, j;
bh_assert(wrapper - (gvname + prefix_len) > 0);
/* Get AOT function index */
bh_memcpy_s(buf, (uint32)sizeof(buf), gvname + prefix_len,
(uint32)(wrapper - (gvname + prefix_len)));
i = atoi(buf);
group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
/* Compile some functions each time */
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
snprintf(func_name, sizeof(func_name), "%s%d",
AOT_FUNC_PREFIX, i + j * group_stride);
Function *F1 = M->getFunction(func_name);
if (F1) {
LOG_DEBUG("compile func %s", func_name);
GVsToAdd.push_back(cast<GlobalValue>(F1));
}
}
}
}
}
for (auto *GV : GVsToAdd) {
Requested.insert(GV);
}
return Requested;
}
LLVMErrorRef
LLVMOrcCreateLLLazyJIT(LLVMOrcLLLazyJITRef *Result,
LLVMOrcLLLazyJITBuilderRef Builder)
{
assert(Result && "Result can not be null");
if (!Builder)
Builder = LLVMOrcCreateLLLazyJITBuilder();
auto J = unwrap(Builder)->create();
LLVMOrcDisposeLLLazyJITBuilder(Builder);
if (!J) {
Result = nullptr;
return 0;
}
LLLazyJIT *lazy_jit = J->release();
lazy_jit->setPartitionFunction(PartitionFunction);
*Result = wrap(lazy_jit);
return LLVMErrorSuccess;
}
LLVMErrorRef
LLVMOrcDisposeLLLazyJIT(LLVMOrcLLLazyJITRef J)
{
delete unwrap(J);
return LLVMErrorSuccess;
}
LLVMErrorRef
LLVMOrcLLLazyJITAddLLVMIRModule(LLVMOrcLLLazyJITRef J, LLVMOrcJITDylibRef JD,
LLVMOrcThreadSafeModuleRef TSM)
{
std::unique_ptr<ThreadSafeModule> TmpTSM(unwrap(TSM));
return wrap(unwrap(J)->addLazyIRModule(*unwrap(JD), std::move(*TmpTSM)));
}
LLVMErrorRef
LLVMOrcLLLazyJITLookup(LLVMOrcLLLazyJITRef J, LLVMOrcExecutorAddress *Result,
const char *Name)
{
assert(Result && "Result can not be null");
auto Sym = unwrap(J)->lookup(Name);
if (!Sym) {
*Result = 0;
return wrap(Sym.takeError());
}
#if LLVM_VERSION_MAJOR < 15
*Result = Sym->getAddress();
#else
*Result = Sym->getValue();
#endif
return LLVMErrorSuccess;
}
LLVMOrcSymbolStringPoolEntryRef
LLVMOrcLLLazyJITMangleAndIntern(LLVMOrcLLLazyJITRef J,
const char *UnmangledName)
{
return wrap(OrcV2CAPIHelper::moveFromSymbolStringPtr(
unwrap(J)->mangleAndIntern(UnmangledName)));
}
LLVMOrcJITDylibRef
LLVMOrcLLLazyJITGetMainJITDylib(LLVMOrcLLLazyJITRef J)
{
return wrap(&unwrap(J)->getMainJITDylib());
}
const char *
LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J)
{
return unwrap(J)->getTargetTriple().str().c_str();
}
LLVMOrcExecutionSessionRef
LLVMOrcLLLazyJITGetExecutionSession(LLVMOrcLLLazyJITRef J)
{
return wrap(&unwrap(J)->getExecutionSession());
}
LLVMOrcIRTransformLayerRef
LLVMOrcLLLazyJITGetIRTransformLayer(LLVMOrcLLLazyJITRef J)
{
return wrap(&unwrap(J)->getIRTransformLayer());
}
LLVMOrcObjectTransformLayerRef
LLVMOrcLLLazyJITGetObjTransformLayer(LLVMOrcLLLazyJITRef J)
{
return wrap(&unwrap(J)->getObjTransformLayer());
}

75
core/iwasm/compilation/aot_orc_extra.h Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _AOT_ORC_LAZINESS_H_
#define _AOT_ORC_LAZINESS_H_
#include "llvm-c/Error.h"
#include "llvm-c/ExternC.h"
#include "llvm-c/LLJIT.h"
#include "llvm-c/Orc.h"
#include "llvm-c/Types.h"
LLVM_C_EXTERN_C_BEGIN
typedef struct LLVMOrcOpaqueLLLazyJITBuilder *LLVMOrcLLLazyJITBuilderRef;
typedef struct LLVMOrcOpaqueLLLazyJIT *LLVMOrcLLLazyJITRef;
// Extra bindings for LLJIT
void
LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef Builder,
unsigned NumCompileThreads);
// Extra bindings for LLLazyJIT
LLVMOrcLLLazyJITBuilderRef
LLVMOrcCreateLLLazyJITBuilder(void);
void
LLVMOrcDisposeLLLazyJITBuilder(LLVMOrcLLLazyJITBuilderRef Builder);
void
LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder(
LLVMOrcLLLazyJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMP);
void
LLVMOrcLLLazyJITBuilderSetNumCompileThreads(LLVMOrcLLLazyJITBuilderRef Builder,
unsigned NumCompileThreads);
LLVMErrorRef
LLVMOrcCreateLLLazyJIT(LLVMOrcLLLazyJITRef *Result,
LLVMOrcLLLazyJITBuilderRef Builder);
LLVMErrorRef
LLVMOrcDisposeLLLazyJIT(LLVMOrcLLLazyJITRef J);
LLVMErrorRef
LLVMOrcLLLazyJITAddLLVMIRModule(LLVMOrcLLLazyJITRef J, LLVMOrcJITDylibRef JD,
LLVMOrcThreadSafeModuleRef TSM);
LLVMErrorRef
LLVMOrcLLLazyJITLookup(LLVMOrcLLLazyJITRef J, LLVMOrcExecutorAddress *Result,
const char *Name);
LLVMOrcSymbolStringPoolEntryRef
LLVMOrcLLLazyJITMangleAndIntern(LLVMOrcLLLazyJITRef J,
const char *UnmangledName);
LLVMOrcJITDylibRef
LLVMOrcLLLazyJITGetMainJITDylib(LLVMOrcLLLazyJITRef J);
const char *
LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J);
LLVMOrcExecutionSessionRef
LLVMOrcLLLazyJITGetExecutionSession(LLVMOrcLLLazyJITRef J);
LLVMOrcIRTransformLayerRef
LLVMOrcLLLazyJITGetIRTransformLayer(LLVMOrcLLLazyJITRef J);
LLVMOrcObjectTransformLayerRef
LLVMOrcLLLazyJITGetObjTransformLayer(LLVMOrcLLLazyJITRef J);
LLVM_C_EXTERN_C_END
#endif

15
core/iwasm/interpreter/wasm.h
View File

@ -369,6 +369,13 @@ typedef struct WASMCustomSection {
#if WASM_ENABLE_JIT != 0
struct AOTCompData;
struct AOTCompContext;
/* Orc JIT thread arguments */
typedef struct OrcJitThreadArg {
struct AOTCompContext *comp_ctx;
struct WASMModule *module;
uint32 group_idx;
} OrcJitThreadArg;
#endif
struct WASMModule {
@ -501,14 +508,20 @@ struct WASMModule {
#endif
#if WASM_ENABLE_FAST_JIT != 0
/* point to JITed functions */
/* func pointers of Fast JITed (un-imported) functions */
void **fast_jit_func_ptrs;
#endif
#if WASM_ENABLE_JIT != 0
struct AOTCompData *comp_data;
struct AOTCompContext *comp_ctx;
/* func pointers of LLVM JITed (un-imported) functions */
void **func_ptrs;
/* whether the func pointers are compiled */
bool *func_ptrs_compiled;
bool orcjit_stop_compiling;
korp_tid orcjit_threads[WASM_ORC_JIT_BACKEND_THREAD_NUM];
OrcJitThreadArg orcjit_thread_args[WASM_ORC_JIT_BACKEND_THREAD_NUM];
#endif
};

163
core/iwasm/interpreter/wasm_loader.c
View File

@ -2953,20 +2953,91 @@ calculate_global_data_offset(WASMModule *module)
}
#if WASM_ENABLE_JIT != 0
static void *
orcjit_thread_callback(void *arg)
{
LLVMOrcJITTargetAddress func_addr = 0;
OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg;
AOTCompContext *comp_ctx = thread_arg->comp_ctx;
WASMModule *module = thread_arg->module;
uint32 group_idx = thread_arg->group_idx;
uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
uint32 func_count = module->function_count;
uint32 i, j;
typedef void (*F)(void);
LLVMErrorRef error;
char func_name[48];
union {
F f;
void *v;
} u;
/* Compile jit functions of this group */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i,
"_wrapper");
LOG_DEBUG("compile func %s", func_name);
error =
LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile orc jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
continue;
}
/* Call the jit wrapper function to trigger its compilation, so as
to compile the actual jit functions, since we add the latter to
function list in the PartitionFunction callback */
u.v = (void *)func_addr;
u.f();
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count)
module->func_ptrs_compiled[i + j * group_stride] = true;
}
if (module->orcjit_stop_compiling) {
break;
}
}
return NULL;
}
static void
orcjit_stop_compile_threads(WASMModule *module)
{
uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args)
/ sizeof(OrcJitThreadArg));
module->orcjit_stop_compiling = true;
for (i = 0; i < thread_num; i++) {
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
}
static bool
compile_llvm_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
AOTCompOption option = { 0 };
char func_name[32], *aot_last_error;
char *aot_last_error;
uint64 size;
uint32 i;
uint32 thread_num, i;
size = sizeof(void *) * (uint64)module->function_count;
if (size > 0
&& !(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
if (module->function_count > 0) {
size = sizeof(void *) * (uint64)module->function_count
+ sizeof(bool) * (uint64)module->function_count;
if (!(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->func_ptrs_compiled =
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
}
module->comp_data = aot_create_comp_data(module);
@ -3015,20 +3086,24 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
for (i = 0; i < module->comp_data->func_count; i++) {
LLVMErrorRef error;
bh_print_time("Begin to lookup jit functions");
for (i = 0; i < module->function_count; i++) {
LLVMOrcJITTargetAddress func_addr = 0;
LLVMErrorRef error;
char func_name[48];
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit,
&func_addr, func_name))) {
error = LLVMOrcLLLazyJITLookup(module->comp_ctx->orc_jit, &func_addr,
func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
set_error_buf_v(error_buf, error_buf_size,
"failed to compile orc jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
return false;
}
/**
* No need to lock the func_ptr[func_idx] here as it is basic
* data type, the load/store for it can be finished by one cpu
@ -3038,20 +3113,43 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
module->func_ptrs[i] = (void *)func_addr;
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
}
#else
/* Resolve function addresses */
bh_assert(module->comp_ctx->exec_engine);
for (i = 0; i < module->comp_data->func_count; i++) {
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress(
module->comp_ctx->exec_engine, func_name))) {
bh_print_time("Begin to compile jit functions");
thread_num =
(uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg));
/* Create threads to compile the jit functions */
for (i = 0; i < thread_num; i++) {
module->orcjit_thread_args[i].comp_ctx = module->comp_ctx;
module->orcjit_thread_args[i].module = module;
module->orcjit_thread_args[i].group_idx = i;
if (os_thread_create(&module->orcjit_threads[i], orcjit_thread_callback,
(void *)&module->orcjit_thread_args[i],
APP_THREAD_STACK_SIZE_DEFAULT)
!= 0) {
uint32 j;
set_error_buf(error_buf, error_buf_size,
"failed to compile llvm mc jit function");
"create orcjit compile thread failed");
/* Terminate the threads created */
module->orcjit_stop_compiling = true;
for (j = 0; j < i; j++) {
os_thread_join(module->orcjit_threads[j], NULL);
}
return false;
}
module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i];
}
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
#if WASM_ENABLE_LAZY_JIT == 0
/* Wait until all jit functions are compiled for eager mode */
for (i = 0; i < thread_num; i++) {
os_thread_join(module->orcjit_threads[i], NULL);
}
#endif
bh_print_time("End compile jit functions");
return true;
}
@ -3893,6 +3991,18 @@ wasm_loader_unload(WASMModule *module)
if (!module)
return;
#if WASM_ENABLE_JIT != 0
/* Stop LLVM JIT compilation firstly to avoid accessing
module internal data after they were freed */
orcjit_stop_compile_threads(module);
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
aot_destroy_comp_context(module->comp_ctx);
if (module->comp_data)
aot_destroy_comp_data(module->comp_data);
#endif
if (module->types) {
for (i = 0; i < module->type_count; i++) {
if (module->types[i])
@ -4018,15 +4128,6 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_JIT != 0
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
aot_destroy_comp_context(module->comp_ctx);
if (module->comp_data)
aot_destroy_comp_data(module->comp_data);
#endif
wasm_runtime_free(module);
}

169
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -1784,20 +1784,91 @@ calculate_global_data_offset(WASMModule *module)
}
#if WASM_ENABLE_JIT != 0
static void *
orcjit_thread_callback(void *arg)
{
LLVMOrcJITTargetAddress func_addr = 0;
OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg;
AOTCompContext *comp_ctx = thread_arg->comp_ctx;
WASMModule *module = thread_arg->module;
uint32 group_idx = thread_arg->group_idx;
uint32 group_stride = WASM_ORC_JIT_BACKEND_THREAD_NUM;
uint32 func_count = module->function_count;
uint32 i, j;
typedef void (*F)(void);
LLVMErrorRef error;
char func_name[48];
union {
F f;
void *v;
} u;
/* Compile jit functions of this group */
for (i = group_idx; i < func_count;
i += group_stride * WASM_ORC_JIT_COMPILE_THREAD_NUM) {
snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX, i,
"_wrapper");
LOG_DEBUG("compile func %s", func_name);
error =
LLVMOrcLLLazyJITLookup(comp_ctx->orc_jit, &func_addr, func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
os_printf("failed to compile orc jit function: %s", err_msg);
LLVMDisposeErrorMessage(err_msg);
continue;
}
/* Call the jit wrapper function to trigger its compilation, so as
to compile the actual jit functions, since we add the latter to
function list in the PartitionFunction callback */
u.v = (void *)func_addr;
u.f();
for (j = 0; j < WASM_ORC_JIT_COMPILE_THREAD_NUM; j++) {
if (i + j * group_stride < func_count)
module->func_ptrs_compiled[i + j * group_stride] = true;
}
if (module->orcjit_stop_compiling) {
break;
}
}
return NULL;
}
static void
orcjit_stop_compile_threads(WASMModule *module)
{
uint32 i, thread_num = (uint32)(sizeof(module->orcjit_thread_args)
/ sizeof(OrcJitThreadArg));
module->orcjit_stop_compiling = true;
for (i = 0; i < thread_num; i++) {
if (module->orcjit_threads[i])
os_thread_join(module->orcjit_threads[i], NULL);
}
}
static bool
compile_llvm_jit_functions(WASMModule *module, char *error_buf,
uint32 error_buf_size)
{
AOTCompOption option = { 0 };
char func_name[32], *aot_last_error;
char *aot_last_error;
uint64 size;
uint32 i;
uint32 thread_num, i;
size = sizeof(void *) * (uint64)module->function_count;
if (size > 0
&& !(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
if (module->function_count > 0) {
size = sizeof(void *) * (uint64)module->function_count
+ sizeof(bool) * (uint64)module->function_count;
if (!(module->func_ptrs =
loader_malloc(size, error_buf, error_buf_size))) {
return false;
}
module->func_ptrs_compiled =
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
}
module->comp_data = aot_create_comp_data(module);
@ -1846,20 +1917,26 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
return false;
}
#if WASM_ENABLE_LAZY_JIT != 0
for (i = 0; i < module->comp_data->func_count; i++) {
LLVMErrorRef error;
bh_print_time("Begin to lookup jit functions");
for (i = 0; i < module->function_count; i++) {
LLVMOrcJITTargetAddress func_addr = 0;
LLVMErrorRef error;
char func_name[48];
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if ((error = LLVMOrcLLJITLookup(module->comp_ctx->orc_lazyjit,
&func_addr, func_name))) {
error = LLVMOrcLLLazyJITLookup(module->comp_ctx->orc_jit, &func_addr,
func_name);
if (error != LLVMErrorSuccess) {
char *err_msg = LLVMGetErrorMessage(error);
set_error_buf_v(error_buf, error_buf_size,
"failed to compile orc jit function: %s", err_msg);
char buf[128];
snprintf(buf, sizeof(buf), "failed to compile orc jit function: %s",
err_msg);
set_error_buf(error_buf, error_buf_size, buf);
LLVMDisposeErrorMessage(err_msg);
return false;
}
/**
* No need to lock the func_ptr[func_idx] here as it is basic
* data type, the load/store for it can be finished by one cpu
@ -1869,20 +1946,43 @@ compile_llvm_jit_functions(WASMModule *module, char *error_buf,
module->func_ptrs[i] = (void *)func_addr;
module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
}
#else
/* Resolve function addresses */
bh_assert(module->comp_ctx->exec_engine);
for (i = 0; i < module->comp_data->func_count; i++) {
snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i);
if (!(module->func_ptrs[i] = (void *)LLVMGetFunctionAddress(
module->comp_ctx->exec_engine, func_name))) {
bh_print_time("Begin to compile jit functions");
thread_num =
(uint32)(sizeof(module->orcjit_thread_args) / sizeof(OrcJitThreadArg));
/* Create threads to compile the jit functions */
for (i = 0; i < thread_num; i++) {
module->orcjit_thread_args[i].comp_ctx = module->comp_ctx;
module->orcjit_thread_args[i].module = module;
module->orcjit_thread_args[i].group_idx = i;
if (os_thread_create(&module->orcjit_threads[i], orcjit_thread_callback,
(void *)&module->orcjit_thread_args[i],
APP_THREAD_STACK_SIZE_DEFAULT)
!= 0) {
uint32 j;
set_error_buf(error_buf, error_buf_size,
"failed to compile llvm mc jit function");
"create orcjit compile thread failed");
/* Terminate the threads created */
module->orcjit_stop_compiling = true;
for (j = 0; j < i; j++) {
os_thread_join(module->orcjit_threads[j], NULL);
}
return false;
}
module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i];
}
#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
#if WASM_ENABLE_LAZY_JIT == 0
/* Wait until all jit functions are compiled for eager mode */
for (i = 0; i < thread_num; i++) {
os_thread_join(module->orcjit_threads[i], NULL);
}
#endif
bh_print_time("End compile jit functions");
return true;
}
@ -2586,6 +2686,18 @@ wasm_loader_unload(WASMModule *module)
if (!module)
return;
#if WASM_ENABLE_JIT != 0
/* Stop LLVM JIT compilation firstly to avoid accessing
module internal data after they were freed */
orcjit_stop_compile_threads(module);
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
aot_destroy_comp_context(module->comp_ctx);
if (module->comp_data)
aot_destroy_comp_data(module->comp_data);
#endif
if (module->types) {
for (i = 0; i < module->type_count; i++) {
if (module->types[i])
@ -2673,15 +2785,6 @@ wasm_loader_unload(WASMModule *module)
}
#endif
#if WASM_ENABLE_JIT != 0
if (module->func_ptrs)
wasm_runtime_free(module->func_ptrs);
if (module->comp_ctx)
aot_destroy_comp_context(module->comp_ctx);
if (module->comp_data)
aot_destroy_comp_data(module->comp_data);
#endif
wasm_runtime_free(module);
}