christoph/wamr
1
Fork 0
Code Releases Activity

Enable aarch64 support, add component test framework and test suite (#211)

Browse Source 
and refine aot call indirect op
This commit is contained in:
wenyongh
2020-03-24 19:04:29 +08:00
committed by GitHub
parent 8ae161b779
commit 01e85144f4
77 changed files with 3293 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
core/config.h
View File

@ -8,6 +8,7 @@
#if !defined(BUILD_TARGET_X86_64) \
&& !defined(BUILD_TARGET_AMD_64) \
&& !defined(BUILD_TARGET_AARCH64) \
&& !defined(BUILD_TARGET_X86_32) \
&& !defined(BUILD_TARGET_ARM) \
&& !defined(BUILD_TARGET_ARM_VFP) \
@ -19,6 +20,8 @@
#define BUILD_TARGET_X86_64
#elif defined(__amd64__) || defined(__amd64)
#define BUILD_TARGET_AMD_64
#elif defined(__aarch64__)
#define BUILD_TARGET_AARCH64
#elif defined(__i386__) || defined(__i386) || defined(i386)
#define BUILD_TARGET_X86_32
#elif defined(__thumb__)

2
core/iwasm/aot/aot_loader.c
View File

@ -140,6 +140,7 @@ GET_U64_FROM_ADDR(uint32 *addr)
#define E_MACHINE_MIPS 8 /* MIPS R3000 big-endian */
#define E_MACHINE_MIPS_RS3_LE 10 /* MIPS R3000 little-endian */
#define E_MACHINE_ARM 40 /* ARM/Thumb */
#define E_MACHINE_AARCH64 183 /* AArch64 */
#define E_MACHINE_ARC 45 /* Argonaut RISC Core */
#define E_MACHINE_IA_64 50 /* Intel Merced */
#define E_MACHINE_MIPS_X 51 /* Stanford MIPS-X */
@ -196,6 +197,7 @@ get_aot_file_target(AOTTargetInfo *target_info,
machine_type = "i386";
break;
case E_MACHINE_ARM:
case E_MACHINE_AARCH64:
machine_type = target_info->arch;
break;
case E_MACHINE_MIPS:

3
core/iwasm/aot/aot_reloc.h
View File

@ -14,9 +14,8 @@ typedef struct {
#define REG_COMMON_SYMBOLS \
REG_SYM(aot_set_exception_with_id), \
REG_SYM(aot_get_exception), \
REG_SYM(aot_is_wasm_type_equal), \
REG_SYM(aot_invoke_native), \
REG_SYM(aot_call_indirect), \
REG_SYM(wasm_runtime_enlarge_memory), \
REG_SYM(wasm_runtime_set_exception), \
REG_SYM(fmin), \

84
core/iwasm/aot/aot_runtime.c
View File

@ -811,7 +811,7 @@ aot_is_wasm_type_equal(AOTModuleInstance *module_inst,
return wasm_type_equal(type1, type2);
}
void
bool
aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
uint32 *frame_lp, uint32 argc, uint32 *argv_ret)
{
@ -827,18 +827,76 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
const char *signature = NULL;
char buf[128];
if (func_idx < aot_module->import_func_count) {
import_func = aot_module->import_funcs + func_idx;
if (!func_ptr) {
snprintf(buf, sizeof(buf),
"fail to call unlinked import function (%s, %s)",
import_func->module_name, import_func->func_name);
aot_set_exception(module_inst, buf);
return;
}
signature = import_func->signature;
bh_assert(func_idx < aot_module->import_func_count);
import_func = aot_module->import_funcs + func_idx;
if (!func_ptr) {
snprintf(buf, sizeof(buf),
"fail to call unlinked import function (%s, %s)",
import_func->module_name, import_func->func_name);
aot_set_exception(module_inst, buf);
return false;
}
wasm_runtime_invoke_native(exec_env, func_ptr,
func_type, signature, frame_lp, argc, argv_ret);
signature = import_func->signature;
return wasm_runtime_invoke_native(exec_env, func_ptr,
func_type, signature,
frame_lp, argc, argv_ret);
}
bool
aot_call_indirect(WASMExecEnv *exec_env,
uint32 func_type_idx, uint32 table_elem_idx,
uint32 *frame_lp, uint32 argc, uint32 *argv_ret)
{
AOTModuleInstance *module_inst = (AOTModuleInstance*)
wasm_runtime_get_module_inst(exec_env);
AOTModule *aot_module = (AOTModule*)module_inst->aot_module.ptr;
uint32 *func_type_indexes = (uint32*)module_inst->func_type_indexes.ptr;
uint32 *table_data = (uint32*)module_inst->table_data.ptr;
AOTFuncType *func_type = aot_module->func_types[func_type_idx];;
void **func_ptrs = (void**)module_inst->func_ptrs.ptr, *func_ptr;
uint32 table_size = module_inst->table_size;
uint32 func_idx, func_type_idx1;
AOTImportFunc *import_func;
const char *signature = NULL;
char buf[128];
if (table_elem_idx >= table_size) {
aot_set_exception_with_id(module_inst, EXCE_UNDEFINED_ELEMENT);
return false;
}
func_idx = table_data[table_elem_idx];
if (func_idx == (uint32)-1) {
aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT);
return false;
}
func_type_idx1 = func_type_indexes[func_idx];
if (!aot_is_wasm_type_equal(module_inst, func_type_idx, func_type_idx1)) {
aot_set_exception_with_id(module_inst, EXCE_INVALID_FUNCTION_TYPE_INDEX);
return false;
}
if (func_idx < aot_module->import_func_count) {
/* Call native function */
import_func = aot_module->import_funcs + func_idx;
signature = import_func->signature;
}
if (!(func_ptr = func_ptrs[func_idx])) {
bh_assert(func_idx < aot_module->import_func_count);
import_func = aot_module->import_funcs + func_idx;
snprintf(buf, sizeof(buf),
"fail to call unlinked import function (%s, %s)",
import_func->module_name, import_func->func_name);
aot_set_exception(module_inst, buf);
return false;
}
return wasm_runtime_invoke_native(exec_env, func_ptr,
func_type, signature,
frame_lp, argc, argv_ret);
}

7
core/iwasm/aot/aot_runtime.h
View File

@ -435,10 +435,15 @@ aot_is_wasm_type_equal(AOTModuleInstance *module_inst,
/**
* Invoke native function from aot code
*/
void
bool
aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
uint32 *frame_lp, uint32 argc, uint32 *argv_ret);
bool
aot_call_indirect(WASMExecEnv *exec_env,
uint32 func_type_idx, uint32 table_elem_idx,
uint32 *frame_lp, uint32 argc, uint32 *argv_ret);
uint32
aot_get_plt_table_size();

238
core/iwasm/aot/arch/aot_reloc_aarch64.c Normal file
View File

@ -0,0 +1,238 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "aot_reloc.h"
#define R_AARCH64_ADR_PREL_PG_HI21 275
#define R_AARCH64_ADD_ABS_LO12_NC 277
#define R_AARCH64_CALL26 283
static SymbolMap target_sym_map[] = {
REG_COMMON_SYMBOLS
};
static void
set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
{
if (error_buf != NULL)
snprintf(error_buf, error_buf_size, "%s", string);
}
SymbolMap *
get_target_symbol_map(uint32 *sym_num)
{
*sym_num = sizeof(target_sym_map) / sizeof(SymbolMap);
return target_sym_map;
}
void
get_current_target(char *target_buf, uint32 target_buf_size)
{
char *build_target = BUILD_TARGET;
char *p = target_buf, *p_end;
snprintf(target_buf, target_buf_size, "%s", build_target);
p_end = p + strlen(target_buf);
while (p < p_end) {
if (*p >= 'A' && *p <= 'Z')
*p++ += 'a' - 'A';
else
p++;
}
if (!strcmp(target_buf, "aarch64"))
snprintf(target_buf, target_buf_size, "aarch64v8");
}
static uint32
get_plt_item_size()
{
/* 8*4 bytes instructions and 8 bytes symbol address */
return 40;
}
void
init_plt_table(uint8 *plt)
{
uint32 i, num = sizeof(target_sym_map) / sizeof(SymbolMap);
for (i = 0; i < num; i++) {
uint32 *p = (uint32*)plt;
*p++ = 0xd10023ff; /* sub sp, sp, #0x8 */
*p++ = 0xf90003fe; /* str x30, [sp] */
*p++ = 0x100000de; /* adr x30, #24 */
*p++ = 0xf94003de; /* ldr x30, [x30] */
*p++ = 0xd63f03c0; /* blr x30 */
*p++ = 0xf94003fe; /* ldr x30, [sp] */
*p++ = 0x910023ff; /* add sp, sp, #0x8 */
*p++ = 0xd61f03c0; /* br x30 */
/* symbol addr */
*(uint64*)p = (uint64)(uintptr_t)target_sym_map[i].symbol_addr;
p += 2;
plt += get_plt_item_size();
}
}
uint32
get_plt_table_size()
{
return get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap));
}
#define SIGN_EXTEND_TO_INT64(val, bits, val_ext) do { \
int64 m = ((int64)1 << (bits - 1)); \
val_ext = ((int64)val ^ m) - m; \
} while (0)
#define Page(expr) ((expr) & ~0xFFF)
static bool
check_reloc_offset(uint32 target_section_size,
uint64 reloc_offset, uint32 reloc_data_size,
char *error_buf, uint32 error_buf_size)
{
if (!(reloc_offset < (uint64)target_section_size
&& reloc_offset + reloc_data_size <= (uint64)target_section_size)) {
set_error_buf(error_buf, error_buf_size,
"AOT module load failed: invalid relocation offset.");
return false;
}
return true;
}
bool
apply_relocation(AOTModule *module,
uint8 *target_section_addr, uint32 target_section_size,
uint64 reloc_offset, uint64 reloc_addend,
uint32 reloc_type, void *symbol_addr, int32 symbol_index,
char *error_buf, uint32 error_buf_size)
{
switch (reloc_type) {
case R_AARCH64_CALL26:
{
void *S, *P = (void*)(target_section_addr + reloc_offset);
int64 X, A, initial_addend;
int32 insn, imm26;
CHECK_RELOC_OFFSET(sizeof(int32));
insn = *(int32*)P;
imm26 = insn & 0x3FFFFFF;
SIGN_EXTEND_TO_INT64(imm26 << 2, 28, initial_addend);
A = initial_addend;
A += (int64)reloc_addend;
if (symbol_index < 0) {
/* Symbol address itself is an AOT function.
* Apply relocation with the symbol directly.
* Suppose the symbol address is in +-128MB relative
* to the relocation address.
*/
S = symbol_addr;
}
else {
uint8 *plt;
if (reloc_addend > 0) {
set_error_buf(error_buf, error_buf_size,
"AOT module load failed: relocate to plt table "
"with reloc addend larger than 0 is unsupported.");
return false;
}
/* Symbol address is not an AOT function,
* but a function of runtime or native. Its address is
* beyond of the +-128MB space. Apply relocation with
* the PLT which branch to the target symbol address.
*/
S = plt = (uint8*)module->code + module->code_size
- get_plt_table_size()
+ get_plt_item_size() * symbol_index;
}
/* S + A - P */
X = (int64)S + A - (int64)P;
/* Check overflow: +-128MB */
if (X > (128 * BH_MB) || X < (-128 * BH_MB)) {
set_error_buf(error_buf, error_buf_size,
"AOT module load failed: "
"target address out of range.");
return false;
}
/* write the imm26 back to instruction */
*(int32*)P = (insn & 0xFC000000) | ((int32)((X >> 2) & 0x3FFFFFF));
break;
}
case R_AARCH64_ADR_PREL_PG_HI21:
{
void *S = symbol_addr, *P = (void*)(target_section_addr + reloc_offset);
int64 X, A, initial_addend;
int32 insn, immhi19, immlo2, imm21;
CHECK_RELOC_OFFSET(sizeof(int32));
insn = *(int32*)P;
immhi19 = (insn >> 5) & 0x7FFFF;
immlo2 = (insn >> 29) & 0x3;
imm21 = (immhi19 << 2) | immlo2;
SIGN_EXTEND_TO_INT64(imm21 << 12, 33, initial_addend);
A = initial_addend;
A += (int64)reloc_addend;
/* Page(S+A) - Page(P) */
X = Page((int64)S + A) - Page((int64)P);
/* Check overflow: +-4GB */
if (X > ((int64)4 * BH_GB) || X < ((int64)-4 * BH_GB)) {
set_error_buf(error_buf, error_buf_size,
"AOT module load failed: "
"target address out of range.");
return false;
}
/* write the imm21 back to instruction */
immhi19 = (int32)(((X >> 12) >> 2) & 0x7FFFF);
immlo2 = (int32)((X >> 12) & 0x3);
*(int32*)P = (insn & 0x9F00001F) | (immlo2 << 29) | (immhi19 << 5);
break;
}
case R_AARCH64_ADD_ABS_LO12_NC:
{
void *S = symbol_addr, *P = (void*)(target_section_addr + reloc_offset);
int64 X, A, initial_addend;
int32 insn, imm12;
CHECK_RELOC_OFFSET(sizeof(int32));
insn = *(int32*)P;
imm12 = (insn >> 10) & 0xFFF;
SIGN_EXTEND_TO_INT64(imm12, 12, initial_addend);
A = initial_addend;
A += (int64)reloc_addend;
/* S + A */
X = (int64)S + A;
/* No need to check overflow for this reloction type */
/* write the imm12 back to instruction */
*(int32*)P = (insn & 0xFFC003FF) | ((int32)((X & 0xFFF) << 10));
break;
}
default:
if (error_buf != NULL)
snprintf(error_buf, error_buf_size,
"Load relocation section failed: "
"invalid relocation type %d.",
reloc_type);
return false;
}
return true;
}

6
core/iwasm/aot/arch/aot_reloc_arm.c
View File

@ -9,10 +9,6 @@
#define R_ARM_JMP24 29 /* PC relative 24 bit (B/BL<cond>). */
#define R_ARM_ABS32 2 /* Direct 32 bit */
#ifndef BH_MB
#define BH_MB 1024 * 1024
#endif
void __divdi3();
void __udivdi3();
void __moddi3();
@ -163,7 +159,7 @@ init_plt_table(uint8 *plt)
/* nop */
*p++ = 0xe1a00000;
/* symbol addr */
*p++ = (uint32)(uintptr_t)target_sym_map[i].symbol_addr;;
*p++ = (uint32)(uintptr_t)target_sym_map[i].symbol_addr;
plt += get_plt_item_size();
}
}

4
core/iwasm/aot/arch/aot_reloc_thumb.c
View File

@ -8,10 +8,6 @@
#define R_ARM_THM_CALL 10 /* PC relative (Thumb BL and ARMv5 Thumb BLX). */
#define R_ARM_THM_JMP24 30 /* B.W */
#ifndef BH_MB
#define BH_MB 1024 * 1024
#endif
void __divdi3();
void __udivdi3();
void __moddi3();

2
core/iwasm/aot/iwasm_aot.cmake
View File

@ -13,6 +13,8 @@ if (${WAMR_BUILD_TARGET} STREQUAL "X86_64" OR ${WAMR_BUILD_TARGET} STREQUAL "AMD
set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_x86_64.c)
elseif (${WAMR_BUILD_TARGET} STREQUAL "X86_32")
set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_x86_32.c)
elseif (${WAMR_BUILD_TARGET} MATCHES "AARCH64.*")
set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_aarch64.c)
elseif (${WAMR_BUILD_TARGET} MATCHES "ARM.*")
set (arch_source ${IWASM_AOT_DIR}/arch/aot_reloc_arm.c)
elseif (${WAMR_BUILD_TARGET} MATCHES "THUMB.*")

76
core/iwasm/common/arch/invokeNative_aarch64.s Normal file
View File

@ -0,0 +1,76 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
.text
.align 2
.global invokeNative
.type invokeNative,function
/*
* Arguments passed in:
*
* x0 function ptr
* x1 argv
* x2 nstacks
*/
invokeNative:
sub sp, sp, #0x30
stp x19, x20, [sp, #0x20] /* save the registers */
stp x21, x22, [sp, #0x10]
stp x23, x24, [sp, #0x0]
mov x19, x0 /* x19 = function ptr */
mov x20, x1 /* x20 = argv */
mov x21, x2 /* x21 = nstacks */
mov x22, sp /* save the sp before call function */
/* Fill in float-point registers */
ldp d0, d1, [x20], #16 /* d0 = argv[0], d1 = argv[1] */
ldp d2, d3, [x20], #16 /* d2 = argv[2], d3 = argv[3] */
ldp d4, d5, [x20], #16 /* d4 = argv[4], d5 = argv[5] */
ldp d6, d7, [x20], #16 /* d6 = argv[6], d7 = argv[7] */
/* Fill inteter registers */
ldp x0, x1, [x20], #16 /* x0 = argv[8] = exec_env, x1 = argv[9] */
ldp x2, x3, [x20], #16 /* x2 = argv[10], x3 = argv[11] */
ldp x4, x5, [x20], #16 /* x4 = argv[12], x5 = argv[13] */
ldp x6, x7, [x20], #16 /* x6 = argv[14], x7 = argv[15] */
/* Now x20 points to stack args */
/* Directly call the fucntion if no args in stack */
cmp x21, #0
beq call_func
/* Fill all stack args: reserve stack space and fill ony by one */
mov x23, sp
bic sp, x23, #15 /* Ensure stack is 16 bytes aligned */
lsl x23, x21, #3 /* x23 = nstacks * 8 */
add x23, x23, #15 /* x23 = (x23 + 15) & ~15 */
bic x23, x23, #15
sub sp, sp, x23 /* reserved stack space for stack arguments */
mov x23, sp
loop_stack_args: /* copy stack arguments to stack */
cmp x21, #0
beq call_func
ldr x24, [x20], #8
str x24, [x23], #8
sub x21, x21, #1
b loop_stack_args
call_func:
mov x20, x30 /* save x30(lr) */
blr x19
mov sp, x22 /* restore sp which is saved before calling fuction*/
return:
mov x30, x20 /* restore x30(lr) */
ldp x19, x20, [sp, #0x20] /* restore the registers in stack */
ldp x21, x22, [sp, #0x10]
ldp x23, x24, [sp, #0x0]
add sp, sp, #0x30 /* restore sp */
ret

2
core/iwasm/common/iwasm_common.cmake
View File

@ -26,6 +26,8 @@ elseif (${WAMR_BUILD_TARGET} MATCHES "THUMB.*")
else ()
set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_thumb.s)
endif ()
elseif (${WAMR_BUILD_TARGET} MATCHES "AARCH64.*")
set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_aarch64.s)
elseif (${WAMR_BUILD_TARGET} STREQUAL "MIPS")
set (source_all ${c_source_all} ${IWASM_COMMON_DIR}/arch/invokeNative_mips.s)
elseif (${WAMR_BUILD_TARGET} STREQUAL "XTENSA")

14
core/iwasm/common/wasm_runtime_common.c
View File

@ -1811,7 +1811,9 @@ fail:
|| defined(BUILD_TARGET_MIPS) \
|| defined(BUILD_TARGET_XTENSA) */
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
#if defined(BUILD_TARGET_X86_64) \
|| defined(BUILD_TARGET_AMD_64) \
|| defined(BUILD_TARGET_AARCH64)
typedef void (*GenericFunctionPointer)();
int64 invokeNative(GenericFunctionPointer f, uint64 *args, uint64 n_stacks);
@ -1832,8 +1834,12 @@ static VoidFuncPtr invokeNative_Void = (VoidFuncPtr)(uintptr_t)invokeNative;
#define MAX_REG_INTS 4
#else
#define MAX_REG_FLOATS 8
#if defined(BUILD_TARGET_AARCH64)
#define MAX_REG_INTS 8
#else
#define MAX_REG_INTS 6
#endif
#endif /* end of defined(BUILD_TARGET_AARCH64 */
#endif /* end of defined(_WIN32) || defined(_WIN32_) */
bool
wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
@ -1963,4 +1969,6 @@ fail:
return ret;
}
#endif /* end of defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) */
#endif /* end of defined(BUILD_TARGET_X86_64) \
|| defined(BUILD_TARGET_AMD_64) \
|| defined(BUILD_TARGET_AARCH64) */

496
core/iwasm/compilation/aot_emit_function.c
View File

@ -8,41 +8,17 @@
#include "aot_emit_control.h"
#include "../aot/aot_runtime.h"
/* Check whether there was exception thrown, if yes, return directly */
static bool
check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
create_func_return_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder);
AOTFuncType *aot_func_type = func_ctx->aot_func->func_type;
LLVMBasicBlockRef block_curr, check_exce_succ;
LLVMValueRef value, cmp;
/* Load the first byte of aot_module_inst->cur_exception, and check
whether it is '\0'. If yes, no exception was thrown. */
if (!(value = LLVMBuildLoad(comp_ctx->builder, func_ctx->cur_exception,
"exce_value"))
|| !(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
value, I8_ZERO, "cmp"))) {
aot_set_last_error("llvm build icmp failed.");
return false;
}
/* Add check exection success block */
if (!(check_exce_succ = LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_exce_succ"))) {
aot_set_last_error("llvm add basic block failed.");
return false;
}
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMMoveBasicBlockAfter(check_exce_succ, block_curr);
/* Create function return block if it isn't created */
if (!func_ctx->func_return_block) {
if (!(func_ctx->func_return_block =
LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"func_ret"))) {
LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func, "func_ret"))) {
aot_set_last_error("llvm add basic block failed.");
return false;
}
@ -70,6 +46,42 @@ check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
}
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
return true;
}
/* Check whether there was exception thrown, if yes, return directly */
static bool
check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMBasicBlockRef block_curr, check_exce_succ;
LLVMValueRef value, cmp;
/* Create function return block if it isn't created */
if (!create_func_return_block(comp_ctx, func_ctx))
return false;
/* Load the first byte of aot_module_inst->cur_exception, and check
whether it is '\0'. If yes, no exception was thrown. */
if (!(value = LLVMBuildLoad(comp_ctx->builder, func_ctx->cur_exception,
"exce_value"))
|| !(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
value, I8_ZERO, "cmp"))) {
aot_set_last_error("llvm build icmp failed.");
return false;
}
/* Add check exection success block */
if (!(check_exce_succ = LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_exce_succ"))) {
aot_set_last_error("llvm add basic block failed.");
return false;
}
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMMoveBasicBlockAfter(check_exce_succ, block_curr);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
/* Create condition br */
if (!LLVMBuildCondBr(comp_ctx->builder, cmp,
@ -82,18 +94,59 @@ check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
return true;
}
/* Check whether there was exception thrown, if yes, return directly */
static bool
check_call_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef res)
{
LLVMBasicBlockRef block_curr, check_call_succ;
LLVMValueRef cmp;
/* Create function return block if it isn't created */
if (!create_func_return_block(comp_ctx, func_ctx))
return false;
if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE,
res, I8_ZERO, "cmp"))) {
aot_set_last_error("llvm build icmp failed.");
return false;
}
/* Add check exection success block */
if (!(check_call_succ = LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_exce_succ"))) {
aot_set_last_error("llvm add basic block failed.");
return false;
}
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMMoveBasicBlockAfter(check_call_succ, block_curr);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
/* Create condition br */
if (!LLVMBuildCondBr(comp_ctx->builder, cmp,
check_call_succ, func_ctx->func_return_block)) {
aot_set_last_error("llvm build cond br failed.");
return false;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, check_call_succ);
return true;
}
static bool
call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef func_idx, AOTFuncType *aot_func_type,
LLVMTypeRef *param_types, LLVMValueRef *param_values,
uint32 param_count, uint32 param_cell_num,
LLVMTypeRef ret_type, uint8 wasm_ret_type,
LLVMValueRef *p_value_ret)
LLVMValueRef *p_value_ret, LLVMValueRef *p_res)
{
LLVMTypeRef func_type, func_ptr_type, func_param_types[5];
LLVMTypeRef ret_ptr_type, elem_ptr_type;
LLVMValueRef func, elem_idx, elem_ptr;
LLVMValueRef func_param_values[5], value_ret, value_ret_ptr, res;
LLVMValueRef func_param_values[5], value_ret = NULL, value_ret_ptr, res;
char buf[32], *func_name = "aot_invoke_native";
uint32 i, cell_num = 0;
@ -103,7 +156,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
func_param_types[2] = INT32_PTR_TYPE; /* frame_lp */
func_param_types[3] = I32_TYPE; /* argc */
func_param_types[4] = INT32_PTR_TYPE; /* argv_ret */
if (!(func_type = LLVMFunctionType(VOID_TYPE, func_param_types, 5, false))) {
if (!(func_type = LLVMFunctionType(INT8_TYPE, func_param_types, 5, false))) {
aot_set_last_error("llvm add function type failed.");
return false;
}
@ -198,7 +251,8 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* call aot_invoke_native() function */
if (!(LLVMBuildCall(comp_ctx->builder, func, func_param_values, 5, ""))) {
if (!(res = LLVMBuildCall(comp_ctx->builder, func,
func_param_values, 5, "res"))) {
aot_set_last_error("llvm build call failed.");
return false;
}
@ -207,6 +261,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* get function return value */
*p_value_ret = LLVMBuildLoad(comp_ctx->builder, value_ret, "value_ret");
*p_res = res;
return true;
}
@ -221,7 +276,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
AOTFuncType *func_type;
LLVMTypeRef *param_types = NULL, ret_type;
LLVMValueRef *param_values = NULL, value_ret = NULL, func;
LLVMValueRef import_func_idx;
LLVMValueRef import_func_idx, res;
int32 i, j = 0, param_count;
uint64 total_size;
uint8 wasm_ret_type;
@ -292,13 +347,17 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
if (!call_aot_invoke_native_func(comp_ctx, func_ctx, import_func_idx, func_type,
param_types + 1, param_values + 1,
param_count, param_cell_num,
ret_type, wasm_ret_type, &value_ret))
ret_type, wasm_ret_type, &value_ret, &res))
goto fail;
/* Check whether there was exception thrown when executing the function */
if (!check_call_return(comp_ctx, func_ctx, res))
goto fail;
}
else {
func = func_ctxes[func_idx - import_func_count]->func;
/* Call the function */
/* Call the function */
if (!(value_ret = LLVMBuildCall(comp_ctx->builder, func,
param_values, (uint32)param_count + 1,
(func_type->result_count > 0
@ -309,15 +368,15 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* Set calling convention for the call with the func's calling convention */
LLVMSetInstructionCallConv(value_ret, LLVMGetFunctionCallConv(func));
/* Check whether there was exception thrown when executing the function */
if (!check_exception_thrown(comp_ctx, func_ctx))
goto fail;
}
if (func_type->result_count > 0)
PUSH(value_ret, func_type->types[func_type->param_count]);
/* Check whether there was exception thrown when executing the function */
if (!check_exception_thrown(comp_ctx, func_ctx))
goto fail;
ret = true;
fail:
if (param_types)
@ -327,21 +386,148 @@ fail:
return ret;
}
static bool
call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
AOTFuncType *aot_func_type,
LLVMValueRef func_type_idx, LLVMValueRef table_elem_idx,
LLVMTypeRef *param_types, LLVMValueRef *param_values,
uint32 param_count, uint32 param_cell_num,
LLVMTypeRef ret_type, uint8 wasm_ret_type,
LLVMValueRef *p_value_ret, LLVMValueRef *p_res)
{
LLVMTypeRef func_type, func_ptr_type, func_param_types[6];
LLVMTypeRef ret_ptr_type, elem_ptr_type;
LLVMValueRef func, elem_idx, elem_ptr;
LLVMValueRef func_param_values[6], value_ret = NULL, value_ret_ptr, res = NULL;
char buf[32], *func_name = "aot_call_indirect";
uint32 i, cell_num = 0;
/* prepare function type of aot_call_indirect */
func_param_types[0] = comp_ctx->exec_env_type; /* exec_env */
func_param_types[1] = I32_TYPE; /* func_type_idx */
func_param_types[2] = I32_TYPE; /* table_elem_idx */
func_param_types[3] = INT32_PTR_TYPE; /* frame_lp */
func_param_types[4] = I32_TYPE; /* argc */
func_param_types[5] = INT32_PTR_TYPE; /* argv_ret */
if (!(func_type = LLVMFunctionType(INT8_TYPE, func_param_types, 6, false))) {
aot_set_last_error("llvm add function type failed.");
return false;
}
/* prepare function pointer */
if (comp_ctx->is_jit_mode) {
if (!(func_ptr_type = LLVMPointerType(func_type, 0))) {
aot_set_last_error("create LLVM function type failed.");
return false;
}
/* JIT mode, call the function directly */
if (!(func = I64_CONST((uint64)(uintptr_t)aot_call_indirect))
|| !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
return false;
}
}
else {
if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name))
&& !(func = LLVMAddFunction(comp_ctx->module,
func_name, func_type))) {
aot_set_last_error("add LLVM function failed.");
return false;
}
}
if (param_count > 64) {
aot_set_last_error("prepare native arguments failed: "
"maximum 64 parameter cell number supported.");
return false;
}
/* prepare frame_lp */
for (i = 0; i < param_count; i++) {
if (!(elem_idx = I32_CONST(cell_num))
|| !(elem_ptr_type = LLVMPointerType(param_types[i], 0))) {
aot_set_last_error("llvm add const or pointer type failed.");
return false;
}
snprintf(buf, sizeof(buf), "%s%d", "elem", i);
if (!(elem_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder,
func_ctx->argv_buf, &elem_idx, 1, buf))
|| !(elem_ptr = LLVMBuildBitCast(comp_ctx->builder, elem_ptr,
elem_ptr_type, buf))) {
aot_set_last_error("llvm build bit cast failed.");
return false;
}
if (!(res = LLVMBuildStore(comp_ctx->builder, param_values[i], elem_ptr))) {
aot_set_last_error("llvm build store failed.");
return false;
}
LLVMSetAlignment(res, 1);
cell_num += wasm_value_type_cell_num(aot_func_type->types[i]);
}
if (wasm_ret_type != VALUE_TYPE_VOID) {
if (!(ret_ptr_type = LLVMPointerType(ret_type, 0))) {
aot_set_last_error("llvm add pointer type failed.");
return false;
}
if (!(value_ret = LLVMBuildBitCast(comp_ctx->builder, func_ctx->argv_buf,
ret_ptr_type, "argv_ret"))) {
aot_set_last_error("llvm build bit cast failed.");
return false;
}
/* convert to int32 pointer */
if (!(value_ret_ptr = LLVMBuildBitCast(comp_ctx->builder, value_ret,
INT32_PTR_TYPE, "argv_ret_ptr"))) {
aot_set_last_error("llvm build store failed.");
return false;
}
}
else {
value_ret_ptr = LLVMConstNull(INT32_PTR_TYPE);
}
func_param_values[0] = func_ctx->exec_env;
func_param_values[1] = func_type_idx;
func_param_values[2] = table_elem_idx;
func_param_values[3] = func_ctx->argv_buf;
func_param_values[4] = I32_CONST(param_cell_num);
func_param_values[5] = value_ret_ptr;
if (!func_param_values[4]) {
aot_set_last_error("llvm create const failed.");
return false;
}
/* call aot_call_indirect() function */
if (!(res = LLVMBuildCall(comp_ctx->builder, func,
func_param_values, 6, "res"))) {
aot_set_last_error("llvm build call failed.");
return false;
}
if (wasm_ret_type != VALUE_TYPE_VOID)
/* get function return value */
*p_value_ret = LLVMBuildLoad(comp_ctx->builder, value_ret, "value_ret");
*p_res = res;
return true;
}
bool
aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_idx)
{
AOTFuncType *func_type;
LLVMValueRef elem_idx, table_elem, func_idx, ftype_idx_ptr, ftype_idx;
LLVMValueRef cmp_elem_idx, cmp_func_idx, is_ftype_match, is_ftype_mismatch;
LLVMValueRef func, func_ptr, func_const, table_size_const, cmp_func_ptr;
LLVMValueRef *param_values = NULL, param_values_tmp[3], value_ret;
LLVMTypeRef *param_types = NULL, param_types_tmp[3], ret_type,
f_type, f_ptr_type;
LLVMBasicBlockRef check_elem_idx_succ, check_ftype_idx_succ;
LLVMBasicBlockRef check_func_idx_succ, check_func_ptr_succ;
char *func_name = "aot_is_wasm_type_equal";
int32 i, j = 0, param_count;
LLVMValueRef elem_idx, ftype_idx;
LLVMValueRef *param_values = NULL, value_ret = NULL, res = NULL;
LLVMTypeRef *param_types = NULL, ret_type;
int32 i, param_count;
uint32 param_cell_num;
uint64 total_size;
uint8 wasm_ret_type;
@ -353,210 +539,26 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return false;
}
ftype_idx = I32_CONST(type_idx);
CHECK_LLVM_CONST(ftype_idx);
func_type = comp_ctx->comp_data->func_types[type_idx];
param_cell_num = wasm_type_param_cell_num(func_type);
POP_I32(elem_idx);
table_size_const = I32_CONST(comp_ctx->comp_data->table_size);
CHECK_LLVM_CONST(table_size_const);
/* Check if (uint32)elem index >= table size */
if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE,
elem_idx, table_size_const,
"cmp_elem_idx"))) {
aot_set_last_error("llvm build icmp failed.");
goto fail;
}
/* Throw exception if elem index >= table size */
if (!(check_elem_idx_succ =
LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_elem_idx_succ"))) {
aot_set_last_error("llvm add basic block failed.");
goto fail;
}
LLVMMoveBasicBlockAfter(check_elem_idx_succ,
LLVMGetInsertBlock(comp_ctx->builder));
if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNDEFINED_ELEMENT,
true, cmp_elem_idx, check_elem_idx_succ)))
goto fail;
/* Load function index */
if (!(table_elem = LLVMBuildInBoundsGEP(comp_ctx->builder,
func_ctx->table_base,
&elem_idx, 1, "table_elem"))) {
aot_set_last_error("llvm build add failed.");
goto fail;
}
if (!(func_idx = LLVMBuildLoad(comp_ctx->builder, table_elem, "func_idx"))) {
aot_set_last_error("llvm build load failed.");
goto fail;
}
/* Check if func_idx == -1 */
if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
func_idx, I32_NEG_ONE,
"cmp_func_idx"))) {
aot_set_last_error("llvm build icmp failed.");
goto fail;
}
/* Throw exception if func_idx == -1 */
if (!(check_func_idx_succ =
LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_func_idx_succ"))) {
aot_set_last_error("llvm add basic block failed.");
goto fail;
}
LLVMMoveBasicBlockAfter(check_func_idx_succ,
LLVMGetInsertBlock(comp_ctx->builder));
if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNINITIALIZED_ELEMENT,
true, cmp_func_idx, check_func_idx_succ)))
goto fail;
/* Load function type index */
if (!(ftype_idx_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder,
func_ctx->func_type_indexes,
&func_idx, 1,
"ftype_idx_ptr"))) {
aot_set_last_error("llvm build inbounds gep failed.");
goto fail;
}
if (!(ftype_idx = LLVMBuildLoad(comp_ctx->builder, ftype_idx_ptr,
"ftype_idx"))) {
aot_set_last_error("llvm build load failed.");
goto fail;
}
/* Call aot_is_type_equal() to check whether function type match */
param_types_tmp[0] = INT8_PTR_TYPE;
param_types_tmp[1] = I32_TYPE;
param_types_tmp[2] = I32_TYPE;
ret_type = INT8_TYPE;
/* Create function type */
if (!(f_type = LLVMFunctionType(ret_type, param_types_tmp,
3, false))) {
aot_set_last_error("create LLVM function type failed.");
goto fail;
}
if (comp_ctx->is_jit_mode) {
/* Create function type */
if (!(f_ptr_type = LLVMPointerType(f_type, 0))) {
aot_set_last_error("create LLVM function type failed.");
goto fail;
}
/* Create LLVM function with const function pointer */
if (!(func_const = I64_CONST((uint64)(uintptr_t)aot_is_wasm_type_equal))
|| !(func = LLVMConstIntToPtr(func_const, f_ptr_type))) {
aot_set_last_error("create LLVM value failed.");
goto fail;
}
}
else {
/* Create LLVM function with external function pointer */
if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name))
&& !(func = LLVMAddFunction(comp_ctx->module, func_name, f_type))) {
aot_set_last_error("add LLVM function failed.");
goto fail;
}
}
/* Call the aot_is_type_equal() function */
param_values_tmp[0] = func_ctx->aot_inst;
param_values_tmp[1] = I32_CONST(type_idx);
param_values_tmp[2] = ftype_idx;
CHECK_LLVM_CONST(param_values_tmp[1]);
if (!(is_ftype_match = LLVMBuildCall(comp_ctx->builder, func,
param_values_tmp, 3,
"is_ftype_match"))) {
aot_set_last_error("llvm build icmp failed.");
goto fail;
}
if (!(is_ftype_mismatch = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
is_ftype_match, I8_ZERO,
"is_ftype_mismatch"))) {
aot_set_last_error("llvm build icmp failed.");
goto fail;
}
if (!(check_ftype_idx_succ =
LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_ftype_idx_success"))) {
aot_set_last_error("llvm add basic block failed.");
goto fail;
}
LLVMMoveBasicBlockAfter(check_ftype_idx_succ,
LLVMGetInsertBlock(comp_ctx->builder));
if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INVALID_FUNCTION_TYPE_INDEX,
true, is_ftype_mismatch, check_ftype_idx_succ)))
goto fail;
/* Load function pointer */
if (!(func_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->func_ptrs,
&func_idx, 1, "func_ptr"))) {
aot_set_last_error("llvm build inbounds gep failed.");
goto fail;
}
if (!(func = LLVMBuildLoad(comp_ctx->builder, func_ptr, "func_tmp"))) {
aot_set_last_error("llvm build load failed.");
goto fail;
}
/* Check whether import function is NULL */
if (!(cmp_func_ptr = LLVMBuildIsNull(comp_ctx->builder, func, "is_func_null"))) {
aot_set_last_error("llvm build is null failed.");
goto fail;
}
/* Throw exception if import function is NULL */
if (!(check_func_ptr_succ =
LLVMAppendBasicBlockInContext(comp_ctx->context,
func_ctx->func,
"check_func_ptr_succ"))) {
aot_set_last_error("llvm add basic block failed.");
goto fail;
}
LLVMMoveBasicBlockAfter(check_func_ptr_succ,
LLVMGetInsertBlock(comp_ctx->builder));
if (!(aot_emit_exception(comp_ctx, func_ctx,
EXCE_CALL_UNLINKED_IMPORT_FUNC,
true, cmp_func_ptr, check_func_ptr_succ)))
goto fail;
/* Initialize parameter types of the LLVM function */
param_count = (int32)func_type->param_count;
total_size = sizeof(LLVMTypeRef) * (uint64)(param_count + 1);
total_size = sizeof(LLVMTypeRef) * (uint64)param_count;
if (total_size >= UINT32_MAX
|| !(param_types = wasm_runtime_malloc((uint32)total_size))) {
aot_set_last_error("Allocate memory failed.");
goto fail;
}
j = 0;
param_types[j++] = comp_ctx->exec_env_type;
for (i = 0; i < param_count; i++)
param_types[j++] = TO_LLVM_TYPE(func_type->types[i]);
param_types[i] = TO_LLVM_TYPE(func_type->types[i]);
/* Resolve return type of the LLVM function */
if (func_type->result_count) {
@ -569,32 +571,30 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* Allocate memory for parameters */
total_size = sizeof(LLVMValueRef) * (uint64)(param_count + 1);
total_size = sizeof(LLVMValueRef) * (uint64)param_count;
if (total_size >= UINT32_MAX
|| !(param_values = wasm_runtime_malloc((uint32)total_size))) {
aot_set_last_error("Allocate memory failed.");
goto fail;
}
/* First parameter is exec env */
j = 0;
param_values[j++] = func_ctx->exec_env;
/* Pop parameters from stack */
for (i = param_count - 1; i >= 0; i--)
POP(param_values[i + j], func_type->types[i]);
POP(param_values[i], func_type->types[i]);
if (!call_aot_invoke_native_func(comp_ctx, func_ctx, func_idx, func_type,
param_types + 1, param_values + 1,
if (!call_aot_call_indirect_func(comp_ctx, func_ctx,
func_type, ftype_idx, elem_idx,
param_types, param_values,
param_count, param_cell_num,
ret_type, wasm_ret_type, &value_ret))
ret_type, wasm_ret_type,
&value_ret, &res))
goto fail;
if (func_type->result_count > 0)
PUSH(value_ret, func_type->types[func_type->param_count]);
/* Check whether there was exception thrown when executing the function */
if (!check_exception_thrown(comp_ctx, func_ctx))
if (!check_call_return(comp_ctx, func_ctx, res))
goto fail;
ret = true;

16
core/iwasm/compilation/aot_llvm.c
View File

@ -741,6 +741,18 @@ static ArchItem valid_archs[] = {
{ "x86_64", false },
{ "i386", false },
{ "mips", true },
{ "aarch64v8", false },
{ "aarch64v8.1", false },
{ "aarch64v8.2", false },
{ "aarch64v8.3", false },
{ "aarch64v8.4", false },
{ "aarch64v8.5", false },
{ "aarch64_bev8", false }, /* big endian */
{ "aarch64_bev8.1", false },
{ "aarch64_bev8.2", false },
{ "aarch64_bev8.3", false },
{ "aarch64_bev8.4", false },
{ "aarch64_bev8.5", false },
{ "armv4", true },
{ "armv4t", true },
{ "armv5t", true },
@ -939,6 +951,10 @@ aot_create_comp_context(AOTCompData *comp_data,
arch = "thumbv4t";
else if (!strcmp(arch, "thumbeb"))
arch = "thumbv4teb";
else if (!strcmp(arch, "aarch64"))
arch = "aarch64v8";
else if (!strcmp(arch, "aarch64_be"))
arch = "aarch64_bev8";
}
/* Check target arch */