christoph/wamr
1
Fork 0
Code Releases Activity

Implement shared heap for AOT (#3815)

Browse Source
This commit is contained in:
Wenyong Huang
2024-09-29 12:50:59 +08:00
committed by GitHub
parent c4aa1deda5
commit 9ba36e284c
29 changed files with 684 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/config.h
View File

@ -396,7 +396,9 @@
#define APP_HEAP_SIZE_DEFAULT (8 * 1024)
#endif
#define APP_HEAP_SIZE_MIN (256)
#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024)
/* The ems memory allocator supports maximal heap size 1GB,
see ems_gc_internal.h */
#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024)
/* Default min/max gc heap size of each app */
#ifndef GC_HEAP_SIZE_DEFAULT

21
core/iwasm/aot/aot_runtime.c
View File

@ -57,6 +57,9 @@ bh_static_assert(sizeof(AOTMemoryInstance) == 120);
bh_static_assert(offsetof(AOTTableInstance, elems) == 24);
bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0);
bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj)
== 8);
bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16);
bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3);
@ -1885,6 +1888,24 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent,
extra->stack_sizes =
aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL);
/*
* The AOT code checks whether the n bytes to access are in shared heap
* by checking whether the beginning address meets:
* addr >= start_off && addr <= end_off - n-bytes + 1
* where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g.,
* UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit
* target. To simplify the check, when shared heap is disabled, we set
* the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit
* target, so in the checking, the above formula will be false, we don't
* need to check whether the shared heap is enabled or not in the AOT
* code.
*/
#if UINTPTR_MAX == UINT64_MAX
extra->shared_heap_start_off.u64 = UINT64_MAX;
#else
extra->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
total_size = sizeof(AOTFuncPerfProfInfo)
* ((uint64)module->import_func_count + module->func_count);

8
core/iwasm/aot/aot_runtime.h
View File

@ -111,6 +111,14 @@ typedef struct AOTFunctionInstance {
typedef struct AOTModuleInstanceExtra {
DefPointer(const uint32 *, stack_sizes);
/*
* Adjusted shared heap based addr to simple the calculation
* in the aot code. The value is:
* shared_heap->base_addr - shared_heap->start_off
*/
DefPointer(uint8 *, shared_heap_base_addr_adj);
MemBound shared_heap_start_off;
WASMModuleInstanceExtraCommon common;
AOTFunctionInstance **functions;
uint32 function_count;

117
core/iwasm/common/wasm_memory.c
View File

@ -165,7 +165,7 @@ runtime_malloc(uint64 size)
WASMSharedHeap *
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)
{
uint64 heap_struct_size = sizeof(WASMSharedHeap);
uint64 heap_struct_size = sizeof(WASMSharedHeap), map_size;
uint32 size = init_args->size;
WASMSharedHeap *heap;
@ -192,7 +192,18 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)
goto fail3;
}
if (!(heap->base_addr = wasm_mmap_linear_memory(size, size))) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = size;
#else
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
#endif
if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) {
goto fail3;
}
if (!mem_allocator_create_with_struct_and_pool(
@ -213,7 +224,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)
return heap;
fail4:
wasm_munmap_linear_memory(heap->base_addr, size, size);
wasm_munmap_linear_memory(heap->base_addr, size, map_size);
fail3:
wasm_runtime_free(heap->heap_handle);
fail2:
@ -245,18 +256,52 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
if (((WASMModuleInstance *)module_inst)->e->shared_heap) {
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
if (e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap;
}
e->shared_heap = shared_heap;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
if (memory->is_memory64)
e->shared_heap_start_off.u64 = shared_heap->start_off_mem64;
else
e->shared_heap_start_off.u64 = shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u64;
#else
e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u32[0];
#endif
#endif /* end of WASM_ENABLE_JIT != 0 */
}
#endif /* end of WASM_ENABLE_INTERP != 0 */
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
if (e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
e->shared_heap = shared_heap;
#if UINTPTR_MAX == UINT64_MAX
if (memory->is_memory64)
e->shared_heap_start_off.u64 = shared_heap->start_off_mem64;
else
e->shared_heap_start_off.u64 = shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u64;
#else
e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u32[0];
#endif
}
#endif /* end of WASM_ENABLE_AOT != 0 */
return true;
}
@ -277,14 +322,32 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst)
{
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
((WASMModuleInstance *)module_inst)->e->shared_heap = NULL;
}
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
e->shared_heap = NULL;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
#else
e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
e->shared_heap_base_addr_adj = NULL;
#endif
}
#endif /* end of WASM_ENABLE_INTERP != 0 */
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
e->shared_heap = NULL;
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
#else
e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
e->shared_heap_base_addr_adj = NULL;
}
#endif /* end of WASM_ENABLE_AOT != 0 */
}
void
@ -307,13 +370,21 @@ get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst_comm->module_type == Wasm_Module_AoT) {
// TODO
return NULL;
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_comm)
->e;
return e->shared_heap;
}
#endif
return NULL;
}
WASMSharedHeap *
wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
{
return get_shared_heap(module_inst_comm);
}
static bool
is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
bool is_memory64, uint64 app_offset, uint32 bytes)
@ -324,6 +395,10 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
return false;
}
if (bytes == 0) {
bytes = 1;
}
if (!is_memory64) {
if (app_offset >= heap->start_off_mem32
&& app_offset <= UINT32_MAX - bytes + 1) {
@ -457,17 +532,23 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
#if WASM_ENABLE_SHARED_HEAP != 0
static void
wasm_runtime_destroy_shared_heaps()
destroy_shared_heaps()
{
WASMSharedHeap *heap = shared_heap_list;
WASMSharedHeap *cur;
uint64 map_size;
while (heap) {
cur = heap;
heap = heap->next;
mem_allocator_destroy(cur->heap_handle);
wasm_runtime_free(cur->heap_handle);
wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size);
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = cur->size;
#else
map_size = 8 * (uint64)BH_GB;
#endif
wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size);
wasm_runtime_free(cur);
}
}
@ -477,7 +558,7 @@ void
wasm_runtime_memory_destroy(void)
{
#if WASM_ENABLE_SHARED_HEAP != 0
wasm_runtime_destroy_shared_heaps();
destroy_shared_heaps();
#endif
if (memory_mode == MEMORY_MODE_POOL) {
@ -1178,7 +1259,7 @@ wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size,
}
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size)
wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size)
{
return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size);
}

4
core/iwasm/common/wasm_memory.h
View File

@ -54,9 +54,13 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
void
wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst);
void
wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst);
WASMSharedHeap *
wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm);
uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64 size, void **p_native_addr);

18
core/iwasm/common/wasm_runtime_common.c
View File

@ -185,6 +185,9 @@ static bool
is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst)
{
WASMMemoryInstance *memory_inst;
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#endif
uint8 *mapped_mem_start_addr = NULL;
uint8 *mapped_mem_end_addr = NULL;
uint32 i;
@ -202,6 +205,21 @@ is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst)
}
}
#if WASM_ENABLE_SHARED_HEAP != 0
shared_heap =
wasm_runtime_get_shared_heap((WASMModuleInstanceCommon *)module_inst);
if (shared_heap) {
mapped_mem_start_addr = shared_heap->base_addr;
mapped_mem_end_addr = shared_heap->base_addr + 8 * (uint64)BH_GB;
if (mapped_mem_start_addr <= (uint8 *)sig_addr
&& (uint8 *)sig_addr < mapped_mem_end_addr) {
/* The address which causes segmentation fault is inside
the shared heap's guard regions */
return true;
}
}
#endif
return false;
}

7
core/iwasm/common/wasm_shared_memory.c
View File

@ -8,6 +8,9 @@
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif
#if WASM_ENABLE_AOT != 0
#include "../aot/aot_runtime.h"
#endif
/*
* Note: this lock can be per memory.
@ -257,7 +260,9 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
shared_heap = e->shared_heap;
}
#endif

278
core/iwasm/compilation/aot_emit_memory.c
View File

@ -118,10 +118,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
{
LLVMValueRef offset_const =
MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset));
LLVMValueRef addr, maddr, offset1, cmp1, cmp2, cmp;
LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp;
LLVMValueRef mem_base_addr, mem_check_bound;
LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMBasicBlockRef check_succ;
LLVMBasicBlockRef check_succ, block_maddr_phi = NULL;
AOTValue *aot_value_top;
uint32 local_idx_of_aot_value = 0;
uint64 const_value;
@ -131,6 +131,11 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
bool is_shared_memory =
comp_ctx->comp_data->memories[0].flags & SHARED_MEMORY_FLAG;
#endif
#if WASM_ENABLE_MEMORY64 == 0
bool is_memory64 = false;
#else
bool is_memory64 = IS_MEMORY64;
#endif
is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false;
@ -268,8 +273,137 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* offset1 = offset + addr; */
/* TODO: check whether integer overflow occurs when memory is 64-bit
and boundary check is enabled */
BUILD_OP(Add, offset_const, addr, offset1, "offset1");
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem;
LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL;
/* Add basic blocks */
ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap");
ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem");
ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi");
LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr);
LLVMMoveBasicBlockAfter(app_addr_in_linear_mem,
app_addr_in_shared_heap);
LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
if (!(maddr_phi =
LLVMBuildPhi(comp_ctx->builder,
enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE,
"maddr_phi"))) {
aot_set_last_error("llvm build phi failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
if (!is_target_64bit) {
/* Check whether interger overflow occurs in addr + offset */
LLVMBasicBlockRef check_integer_overflow_end;
ADD_BASIC_BLOCK(check_integer_overflow_end,
"check_integer_overflow_end");
LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr);
BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1");
if (!aot_emit_exception(comp_ctx, func_ctx,
EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true,
cmp1, check_integer_overflow_end)) {
goto fail;
}
SET_BUILD_POS(check_integer_overflow_end);
}
shared_heap_check_bound =
is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1)
: (comp_ctx->pointer_size == sizeof(uint64)
? I64_CONST(UINT32_MAX - bytes + 1)
: I32_CONST(UINT32_MAX - bytes + 1));
CHECK_LLVM_CONST(shared_heap_check_bound);
/* Check whether the bytes to access are in shared heap */
if (!comp_ctx->enable_bound_check) {
/* Use IntUGT but not IntUGE to compare, since (1) in the ems
memory allocator, the hmu node includes hmu header and hmu
memory, only the latter is returned to the caller as the
allocated memory, the hmu header isn't returned so the
first byte of the shared heap won't be accesed, (2) using
IntUGT gets better performance than IntUGE in some cases */
BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off,
is_in_shared_heap, "is_in_shared_heap");
/* We don't check the shared heap's upper boundary if boundary
check isn't enabled, the runtime may also use the guard pages
of shared heap to check the boundary if hardware boundary
check feature is enabled. */
}
else {
/* Use IntUGT but not IntUGE to compare, same as above */
BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off,
cmp1, "cmp1");
/* Check the shared heap's upper boundary if boundary check is
enabled */
BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2,
"cmp2");
BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap");
}
if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap,
app_addr_in_shared_heap, app_addr_in_linear_mem)) {
aot_set_last_error("llvm build cond br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap);
/* Get native address inside shared heap */
if (!(maddr =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->shared_heap_base_addr_adj,
&offset1, 1, "maddr_shared_heap"))) {
aot_set_last_error("llvm build inbounds gep failed");
goto fail;
}
if (enable_segue) {
LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base;
if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr,
I64_TYPE, "maddr_u64"))
|| !(mem_base_addr_u64 =
LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr,
I64_TYPE, "mem_base_addr_u64"))) {
aot_set_last_error("llvm build ptr to int failed");
goto fail;
}
if (!(offset_to_mem_base =
LLVMBuildSub(comp_ctx->builder, maddr_u64,
mem_base_addr_u64, "offset_to_mem_base"))) {
aot_set_last_error("llvm build sub failed");
goto fail;
}
if (!(maddr = LLVMBuildIntToPtr(
comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS,
"maddr_shared_heap_segue"))) {
aot_set_last_error("llvm build int to ptr failed.");
goto fail;
}
}
LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem);
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
}
if (comp_ctx->enable_bound_check
&& !(is_local_of_aot_value
&& aot_checked_addr_list_find(func_ctx, local_idx_of_aot_value,
@ -305,10 +439,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp");
}
else {
/* Check integer overflow */
BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1");
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2");
BUILD_OP(Or, cmp1, cmp2, cmp, "cmp");
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
/* Check integer overflow has been checked above */
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp");
}
else {
/* Check integer overflow */
BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1");
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2");
BUILD_OP(Or, cmp1, cmp2, cmp, "cmp");
}
}
/* Add basic blocks */
@ -354,7 +494,19 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
goto fail;
}
}
return maddr;
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
return maddr_phi;
}
else
return maddr;
fail:
return NULL;
}
@ -985,10 +1137,15 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef offset, LLVMValueRef bytes)
{
LLVMValueRef maddr, max_addr, cmp;
LLVMValueRef mem_base_addr;
LLVMValueRef mem_base_addr, maddr_phi = NULL;
LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMBasicBlockRef check_succ;
LLVMBasicBlockRef check_succ, block_maddr_phi = NULL;
LLVMValueRef mem_size;
#if WASM_ENABLE_MEMORY64 == 0
bool is_memory64 = false;
#else
bool is_memory64 = IS_MEMORY64;
#endif
/* Get memory base address and memory data size */
#if WASM_ENABLE_SHARED_MEMORY != 0
@ -1053,9 +1210,96 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
offset =
LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset");
bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len");
if (!offset || !bytes) {
aot_set_last_error("llvm build zext failed.");
goto fail;
}
/* TODO: check whether integer overflow occurs when memory is 64-bit
and boundary check is enabled */
BUILD_OP(Add, offset, bytes, max_addr, "max_addr");
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem;
LLVMValueRef shared_heap_start_off, shared_heap_check_bound;
LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap;
/* Add basic blocks */
ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap");
ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem");
ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi");
LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr);
LLVMMoveBasicBlockAfter(app_addr_in_linear_mem,
app_addr_in_shared_heap);
LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE,
"maddr_phi"))) {
aot_set_last_error("llvm build phi failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
shared_heap_start_off = func_ctx->shared_heap_start_off;
if (comp_ctx->pointer_size == sizeof(uint32)) {
if (!(shared_heap_start_off =
LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off,
I64_TYPE, "shared_heap_start_off_u64"))) {
aot_set_last_error("llvm build zext failed");
goto fail;
}
}
shared_heap_check_bound =
is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX);
CHECK_LLVM_CONST(shared_heap_check_bound);
/* Check whether the bytes to access are in shared heap */
if (!comp_ctx->enable_bound_check) {
/* Use IntUGT but not IntUGE to compare, same as the check
in aot_check_memory_overflow */
BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off,
is_in_shared_heap, "is_in_shared_heap");
}
else {
BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off,
cmp1, "cmp1");
BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset");
BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2,
"cmp2");
BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap");
}
if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap,
app_addr_in_shared_heap, app_addr_in_linear_mem)) {
aot_set_last_error("llvm build cond br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap);
/* Get native address inside shared heap */
if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->shared_heap_base_addr_adj,
&offset, 1, "maddr_shared_heap"))) {
aot_set_last_error("llvm build inbounds gep failed");
goto fail;
}
LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem);
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
}
BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr");
if (!aot_emit_exception(comp_ctx, func_ctx,
EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp,
check_succ)) {
@ -1068,11 +1312,23 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
aot_set_last_error("llvm build add failed.");
goto fail;
}
return maddr;
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
return maddr_phi;
}
else
return maddr;
fail:
return NULL;
}
#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 or WASM_ENABLE_STRINGREF != 0 */
#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 */
#if WASM_ENABLE_BULK_MEMORY != 0
bool

78
core/iwasm/compilation/aot_llvm.c
View File

@ -1518,6 +1518,75 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return true;
}
static bool
create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef offset, base_addr_p, start_off_p, cmp;
uint32 offset_u32;
/* Load aot_inst->e->shared_heap_base_addr_adj */
offset_u32 = get_module_inst_extra_offset(comp_ctx);
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0
if (comp_ctx->is_jit_mode)
offset_u32 +=
offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj);
else
#endif
offset_u32 +=
offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj);
offset = I32_CONST(offset_u32);
CHECK_LLVM_CONST(offset);
if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
"shared_heap_base_addr_adj_p"))) {
aot_set_last_error("llvm build inbounds gep failed");
return false;
}
if (!(func_ctx->shared_heap_base_addr_adj =
LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p,
"shared_heap_base_addr_adj"))) {
aot_set_last_error("llvm build load failed");
return false;
}
/* Load aot_inst->e->shared_heap_start_off */
offset_u32 = get_module_inst_extra_offset(comp_ctx);
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0
if (comp_ctx->is_jit_mode)
offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off);
else
#endif
offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off);
offset = I32_CONST(offset_u32);
CHECK_LLVM_CONST(offset);
if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
"shared_heap_start_off_p"))) {
aot_set_last_error("llvm build inbounds gep failed");
return false;
}
if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2(
comp_ctx->builder,
comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE,
start_off_p, "shared_heap_start_off"))) {
aot_set_last_error("llvm build load failed");
return false;
}
if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder,
func_ctx->shared_heap_base_addr_adj,
"has_shared_heap"))) {
aot_set_last_error("llvm build is not null failed");
return false;
}
return true;
fail:
return false;
}
static bool
create_cur_exception(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
@ -1808,6 +1877,12 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx,
goto fail;
}
/* Load shared heap, shared heap start off mem32 or mem64 */
if (comp_ctx->enable_shared_heap
&& !create_shared_heap_info(comp_ctx, func_ctx)) {
goto fail;
}
return func_ctx;
fail:
@ -2619,6 +2694,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
if (option->enable_gc)
comp_ctx->enable_gc = true;
if (option->enable_shared_heap)
comp_ctx->enable_shared_heap = true;
comp_ctx->opt_level = option->opt_level;
comp_ctx->size_level = option->size_level;

5
core/iwasm/compilation/aot_llvm.h
View File

@ -242,6 +242,9 @@ typedef struct AOTFuncContext {
bool mem_space_unchanged;
AOTCheckedAddrList checked_addr_list;
LLVMValueRef shared_heap_base_addr_adj;
LLVMValueRef shared_heap_start_off;
LLVMBasicBlockRef got_exception_block;
LLVMBasicBlockRef func_return_block;
LLVMValueRef exception_id_phi;
@ -467,6 +470,8 @@ typedef struct AOTCompContext {
/* Enable GC */
bool enable_gc;
bool enable_shared_heap;
uint32 opt_level;
uint32 size_level;

1
core/iwasm/include/aot_comp_option.h
View File

@ -71,6 +71,7 @@ typedef struct AOTCompOption {
bool enable_llvm_pgo;
bool enable_stack_estimation;
bool quick_invoke_c_api_import;
bool enable_shared_heap;
char *use_prof_file;
uint32_t opt_level;
uint32_t size_level;

10
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -47,8 +47,14 @@ typedef float64 CellType_F64;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
#define app_addr_in_shared_heap(app_addr, bytes) \
(shared_heap && (app_addr) >= shared_heap_start_off \
#if WASM_ENABLE_MULTI_MEMORY != 0
/* Only enable shared heap for the default memory */
#define is_default_memory (memidx == 0)
#else
#define is_default_memory true
#endif
#define app_addr_in_shared_heap(app_addr, bytes) \
(shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \
&& (app_addr) <= shared_heap_end_off - bytes + 1)
#define shared_heap_addr_app_to_native(app_addr, native_addr) \

3
core/iwasm/interpreter/wasm_loader.c
View File

@ -5330,6 +5330,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
option.enable_memory_profiling = true;
option.enable_stack_estimation = true;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
option.enable_shared_heap = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {

3
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -2158,6 +2158,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
option.enable_memory_profiling = true;
option.enable_stack_estimation = true;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
option.enable_shared_heap = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {

8
core/iwasm/interpreter/wasm_runtime.c
View File

@ -2791,6 +2791,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
}
}
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0
#if UINTPTR_MAX == UINT64_MAX
module_inst->e->shared_heap_start_off.u64 = UINT64_MAX;
#else
module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
#endif
#if WASM_ENABLE_GC != 0
/* Initialize the table data with init expr */
for (i = 0; i < module->table_count; i++) {

11
core/iwasm/interpreter/wasm_runtime.h
View File

@ -92,7 +92,6 @@ typedef union {
uint32 u32[2];
} MemBound;
#if WASM_ENABLE_SHARED_HEAP != 0
typedef struct WASMSharedHeap {
struct WASMSharedHeap *next;
void *heap_handle;
@ -101,7 +100,6 @@ typedef struct WASMSharedHeap {
uint64 start_off_mem64;
uint64 start_off_mem32;
} WASMSharedHeap;
#endif
struct WASMMemoryInstance {
/* Module type */
@ -366,6 +364,15 @@ typedef struct WASMModuleInstanceExtra {
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#if WASM_ENABLE_JIT != 0
/*
* Adjusted shared heap based addr to simple the calculation
* in the aot code. The value is:
* shared_heap->base_addr - shared_heap->start_off
*/
uint8 *shared_heap_base_addr_adj;
MemBound shared_heap_start_off;
#endif
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \

8
core/iwasm/libraries/shared-heap/shared_heap_wrapper.c
View File

@ -20,14 +20,14 @@
/* clang-format on */
static uint32
shared_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size)
shared_heap_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size)
{
wasm_module_inst_t module_inst = get_module_inst(exec_env);
return (uint32)module_shared_malloc((uint64)size, NULL);
}
static void
shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr)
shared_heap_free_wrapper(wasm_exec_env_t exec_env, void *ptr)
{
wasm_module_inst_t module_inst = get_module_inst(exec_env);
@ -45,8 +45,8 @@ shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr)
/* clang-format on */
static NativeSymbol native_symbols_shared_heap[] = {
REG_NATIVE_FUNC(shared_malloc, "(i)i"),
REG_NATIVE_FUNC(shared_free, "(*)"),
REG_NATIVE_FUNC(shared_heap_malloc, "(i)i"),
REG_NATIVE_FUNC(shared_heap_free, "(*)"),
};
uint32

10
core/iwasm/libraries/thread-mgr/thread_manager.c
View File

@ -1428,16 +1428,6 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
{
WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst);
if (module_inst->module_type == Wasm_Module_Bytecode) {
if (((WASMModuleInstance *)module_inst)->e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
}
else if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
if (exec_env == NULL) {
/* Maybe threads have not been started yet. */
return wasm_runtime_attach_shared_heap_internal(module_inst, heap);