christoph/wamr
1
Fork 0
Code Releases Activity

enable pthread for AoT && update AOT current version to 2 (#311)

Browse Source
This commit is contained in:
Xu Jun
2020-07-16 20:35:04 +08:00
committed by GitHub
parent ca938f3634
commit 32b2943369
32 changed files with 1549 additions and 584 deletions
Download Patch File Download Diff File Expand all files Collapse all files

643
core/iwasm/aot/aot_runtime.c
View File

@ -6,6 +6,9 @@
#include "aot_runtime.h"
#include "bh_log.h"
#include "mem_alloc.h"
#if WASM_ENABLE_SHARED_MEMORY != 0
#include "../common/wasm_shared_memory.h"
#endif
static void
set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
@ -38,7 +41,7 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
uint32 i;
InitializerExpression *init_expr;
uint8 *p = (uint8*)module_inst->global_data.ptr;
AOTImportGlobal *import_global = module->import_globals;;
AOTImportGlobal *import_global = module->import_globals;
AOTGlobal *global = module->globals;
/* Initialize import global data */
@ -146,22 +149,93 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
return true;
}
static bool
static void
memories_deinstantiate(AOTModuleInstance *module_inst)
{
uint32 i;
AOTMemoryInstance *memory_inst;
for (i = 0; i < module_inst->memory_count; i++) {
memory_inst = ((AOTMemoryInstance **)module_inst->memories.ptr)[i];
if (memory_inst) {
#if WASM_ENABLE_SHARED_MEMORY != 0
if (memory_inst->is_shared) {
int32 ref_count =
shared_memory_dec_reference(
(WASMModuleCommon *)module_inst->aot_module.ptr);
bh_assert(ref_count >= 0);
/* if the reference count is not zero,
don't free the memory */
if (ref_count > 0)
continue;
}
#endif
if (memory_inst->heap_handle.ptr)
mem_allocator_destroy(memory_inst->heap_handle.ptr);
if (memory_inst->heap_data.ptr) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
wasm_runtime_free(memory_inst->heap_data.ptr);
#else
os_munmap((uint8*)memory_inst->memory_data.ptr - 2 * (uint64)BH_GB,
8 * (uint64)BH_GB);
#endif
}
}
}
wasm_runtime_free(module_inst->memories.ptr);
}
static AOTMemoryInstance*
memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
AOTMemoryInstance *memory_inst, AOTMemory *memory,
uint32 heap_size, char *error_buf, uint32 error_buf_size)
{
uint32 i, global_index, global_data_offset, base_offset, length;
AOTMemInitData *data_seg;
void *heap_handle;
uint64 memory_data_size = (uint64)module->num_bytes_per_page
* module->mem_init_page_count;
uint64 memory_data_size = (uint64)memory->num_bytes_per_page
* memory->mem_init_page_count;
uint64 total_size = heap_size + memory_data_size;
uint8 *p;
#if WASM_ENABLE_SHARED_MEMORY != 0
AOTMemoryInstance *shared_memory_instance;
bool is_shared_memory = memory->memory_flags & 0x02 ? true : false;
uint64 max_memory_data_size = (uint64)memory->num_bytes_per_page
* memory->mem_max_page_count;
/* Shared memory */
if (is_shared_memory) {
WASMSharedMemNode *node =
wasm_module_get_shared_memory((WASMModuleCommon *)module);
/* If the memory of this module has been instantiated,
return the memory instance directly */
if (node) {
uint32 ref_count;
ref_count = shared_memory_inc_reference(
(WASMModuleCommon *)module);
bh_assert(ref_count > 0);
shared_memory_instance =
(AOTMemoryInstance *)shared_memory_get_memory_inst(node);
bh_assert(shared_memory_instance);
/* Set the shared memory flag, so the runtime will get the
actual memory inst through module_inst->memories array */
memory_inst->is_shared = true;
(void)ref_count;
return shared_memory_instance;
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
/* Allocate max page for shared memory */
total_size = heap_size + max_memory_data_size;
#endif
}
#endif
#ifndef OS_ENABLE_HW_BOUND_CHECK
/* Allocate memory */
if (!(p = runtime_malloc(total_size, error_buf, error_buf_size))) {
return false;
return NULL;
}
#else
uint8 *mapped_mem;
@ -178,7 +252,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
MMAP_PROT_NONE, MMAP_MAP_NONE))) {
set_error_buf(error_buf, error_buf_size,
"AOT module instantiate failed: mmap memory failed.");
return false;
return NULL;
}
p = mapped_mem + 2 * (uint64)BH_GB - heap_size;
@ -186,40 +260,128 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
set_error_buf(error_buf, error_buf_size,
"AOT module instantiate failed: mprotec memory failed.");
os_munmap(mapped_mem, map_size);
return false;
return NULL;
}
memset(p, 0, (uint32)total_size);
#endif
memory_inst->module_type = Wasm_Module_AoT;
/* Initialize heap info */
module_inst->heap_data.ptr = p;
memory_inst->heap_data.ptr = p;
p += heap_size;
module_inst->heap_data_end.ptr = p;
module_inst->heap_data_size = heap_size;
module_inst->heap_base_offset = -(int32)heap_size;
memory_inst->heap_data_end.ptr = p;
memory_inst->heap_data_size = heap_size;
memory_inst->heap_base_offset = -(int32)heap_size;
if (heap_size > 0) {
if (!(heap_handle = mem_allocator_create(module_inst->heap_data.ptr,
if (!(heap_handle = mem_allocator_create(memory_inst->heap_data.ptr,
heap_size))) {
set_error_buf(error_buf, error_buf_size,
"AOT module instantiate failed: init app heap failed.");
"AOT module instantiate failed:"
"init app heap failed.");
goto fail1;
}
module_inst->heap_handle.ptr = heap_handle;
memory_inst->heap_handle.ptr = heap_handle;
}
/* Init memory info */
module_inst->memory_data.ptr = p;
p += (uint32)memory_data_size;
module_inst->memory_data_end.ptr = p;
module_inst->memory_data_size = (uint32)memory_data_size;
module_inst->mem_cur_page_count = module->mem_init_page_count;
module_inst->mem_max_page_count = module->mem_max_page_count;
memory_inst->memory_data.ptr = p;
#if WASM_ENABLE_SHARED_MEMORY != 0
if (is_shared_memory) {
p += (uint32)max_memory_data_size;
}
else
#endif
{
p += (uint32)memory_data_size;
}
memory_inst->memory_data_end.ptr = p;
memory_inst->memory_data_size = (uint32)memory_data_size;
memory_inst->mem_cur_page_count = memory->mem_init_page_count;
memory_inst->mem_max_page_count = memory->mem_max_page_count;
module_inst->mem_bound_check_heap_base = (int64)module_inst->heap_base_offset;
module_inst->mem_bound_check_1byte = (int64)module_inst->memory_data_size - 1;
module_inst->mem_bound_check_2bytes = (int64)module_inst->memory_data_size - 2;
module_inst->mem_bound_check_4bytes = (int64)module_inst->memory_data_size - 4;
module_inst->mem_bound_check_8bytes = (int64)module_inst->memory_data_size - 8;
memory_inst->mem_bound_check_heap_base = memory_inst->heap_base_offset;
memory_inst->mem_bound_check_1byte =
(int64)memory_inst->memory_data_size - 1;
memory_inst->mem_bound_check_2bytes =
(int64)memory_inst->memory_data_size - 2;
memory_inst->mem_bound_check_4bytes =
(int64)memory_inst->memory_data_size - 4;
memory_inst->mem_bound_check_8bytes =
(int64)memory_inst->memory_data_size - 8;
#if WASM_ENABLE_SHARED_MEMORY != 0
if (is_shared_memory) {
memory_inst->is_shared = true;
if (!shared_memory_set_memory_inst((WASMModuleCommon *)module,
(WASMMemoryInstanceCommon *)memory_inst)) {
set_error_buf(error_buf, error_buf_size,
"Instantiate memory failed:"
"allocate memory failed.");
goto fail2;
}
}
#endif
return memory_inst;
#if WASM_ENABLE_SHARED_MEMORY != 0
fail2:
if (heap_size > 0) {
mem_allocator_destroy(memory_inst->heap_handle.ptr);
memory_inst->heap_handle.ptr = NULL;
}
#endif
fail1:
#ifndef OS_ENABLE_HW_BOUND_CHECK
wasm_runtime_free(memory_inst->heap_data.ptr);
#else
os_munmap(mapped_mem, map_size);
#endif
memory_inst->heap_data.ptr = NULL;
return NULL;
}
static AOTMemoryInstance*
aot_get_default_memory(AOTModuleInstance *module_inst)
{
if (module_inst->memories.ptr)
return ((AOTMemoryInstance **)module_inst->memories.ptr)[0];
else
return NULL;
}
static bool
memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
uint32 heap_size, char *error_buf, uint32 error_buf_size)
{
uint32 global_index, global_data_offset, base_offset, length;
uint32 i, memory_count = module->memory_count;
AOTMemoryInstance *memories, *memory_inst;
AOTMemInitData *data_seg;
uint64 total_size;
module_inst->memory_count = memory_count;
total_size = sizeof(AOTPointer) * (uint64)memory_count;
if (!(module_inst->memories.ptr =
runtime_malloc(total_size, error_buf, error_buf_size))) {
return false;
}
memories = module_inst->global_table_data.memory_instances;
for (i = 0; i < memory_count; i++, memories++) {
memory_inst =
memory_instantiate(module_inst, module,
memories, &module->memories[i],
heap_size, error_buf, error_buf_size);
if (!memory_inst) {
return false;
}
((AOTMemoryInstance **)module_inst->memories.ptr)[i] = memory_inst;
}
/* Get default memory instance */
memory_inst = aot_get_default_memory(module_inst);
for (i = 0; i < module->mem_init_data_count; i++) {
data_seg = module->mem_init_data_list[i];
@ -255,47 +417,34 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
}
/* Copy memory data */
bh_assert(module_inst->memory_data.ptr);
bh_assert(memory_inst->memory_data.ptr);
/* Check memory data */
/* check offset since length might negative */
if (base_offset > module_inst->memory_data_size) {
if (base_offset > memory_inst->memory_data_size) {
LOG_DEBUG("base_offset(%d) > memory_data_size(%d)", base_offset,
module_inst->memory_data_size);
memory_inst->memory_data_size);
set_error_buf(error_buf, error_buf_size,
"data segment does not fit");
goto fail2;
return false;
}
/* check offset + length(could be zero) */
length = data_seg->byte_count;
if (base_offset + length > module_inst->memory_data_size) {
if (base_offset + length > memory_inst->memory_data_size) {
LOG_DEBUG("base_offset(%d) + length(%d) > memory_data_size(%d)",
base_offset, length, module_inst->memory_data_size);
base_offset, length, memory_inst->memory_data_size);
set_error_buf(error_buf, error_buf_size,
"data segment does not fit");
goto fail2;
return false;
}
memcpy((uint8*)module_inst->memory_data.ptr + base_offset,
data_seg->bytes, length);
bh_memcpy_s((uint8*)memory_inst->memory_data.ptr + base_offset,
memory_inst->memory_data_size - base_offset,
data_seg->bytes, length);
}
return true;
fail2:
if (heap_size > 0) {
mem_allocator_destroy(module_inst->heap_handle.ptr);
module_inst->heap_handle.ptr = NULL;
}
fail1:
#ifndef OS_ENABLE_HW_BOUND_CHECK
wasm_runtime_free(module_inst->heap_data.ptr);
#else
os_munmap(mapped_mem, map_size);
#endif
module_inst->heap_data.ptr = NULL;
return false;
}
static bool
@ -349,6 +498,64 @@ init_func_type_indexes(AOTModuleInstance *module_inst, AOTModule *module,
return true;
}
static bool
create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module,
char *error_buf, uint32 error_buf_size)
{
AOTExport *exports = module->exports;
AOTFunctionInstance *export_func;
uint64 size;
uint32 i, func_index, ftype_index;
for (i = 0; i < module->export_count; i++) {
if (exports[i].kind == EXPORT_KIND_FUNC)
module_inst->export_func_count++;
}
if (module_inst->export_func_count > 0) {
/* Allocate memory */
size = sizeof(AOTFunctionInstance)
* (uint64)module_inst->export_func_count;
if (!(module_inst->export_funcs.ptr = export_func =
runtime_malloc(size, error_buf, error_buf_size))) {
return false;
}
for (i = 0; i < module->export_count; i++) {
if (exports[i].kind == EXPORT_KIND_FUNC) {
export_func->func_name = exports[i].name;
export_func->func_index = exports[i].index;
if (export_func->func_index < module->import_func_count) {
export_func->is_import_func = true;
export_func->u.func_import =
&module->import_funcs[export_func->func_index];
}
else {
export_func->is_import_func = false;
func_index = export_func->func_index
- module->import_func_count;
ftype_index = module->func_type_indexes[func_index];
export_func->u.func.func_type =
module->func_types[ftype_index];
export_func->u.func.func_ptr =
module->func_ptrs[func_index];
}
export_func++;
}
}
}
return true;
}
static bool
create_exports(AOTModuleInstance *module_inst, AOTModule *module,
char *error_buf, uint32 error_buf_size)
{
return create_export_funcs(module_inst, module,
error_buf, error_buf_size);
}
static bool
execute_post_inst_function(AOTModuleInstance *module_inst)
{
@ -386,6 +593,22 @@ execute_start_function(AOTModuleInstance *module_inst)
return !aot_get_exception(module_inst);
}
#if WASM_ENABLE_BULK_MEMORY != 0
static bool
execute_memory_init_function(AOTModuleInstance *module_inst)
{
AOTFunctionInstance *memory_init_func =
aot_lookup_function(module_inst, "__wasm_call_ctors", "()");
if (!memory_init_func)
/* Not found */
return true;
return aot_create_exec_env_and_call_function(module_inst, memory_init_func,
0, NULL);
}
#endif
AOTModuleInstance*
aot_instantiate(AOTModule *module, bool is_sub_inst,
uint32 stack_size, uint32 heap_size,
@ -394,8 +617,13 @@ aot_instantiate(AOTModule *module, bool is_sub_inst,
AOTModuleInstance *module_inst;
uint32 module_inst_struct_size =
offsetof(AOTModuleInstance, global_table_data.bytes);
uint64 table_data_size = (uint64)module->table_size * sizeof(uint32);
uint64 module_inst_mem_inst_size =
(uint64)module->memory_count * sizeof(AOTMemoryInstance);
uint32 table_size = module->table_count > 0 ?
module->tables[0].table_init_size : 0;
uint64 table_data_size = (uint64)table_size * sizeof(uint32);
uint64 total_size = (uint64)module_inst_struct_size
+ module_inst_mem_inst_size
+ module->global_data_size
+ table_data_size;
uint8 *p;
@ -418,7 +646,8 @@ aot_instantiate(AOTModule *module, bool is_sub_inst,
module_inst->aot_module.ptr = module;
/* Initialize global info */
p = (uint8*)module_inst + module_inst_struct_size;
p = (uint8*)module_inst + module_inst_struct_size +
module_inst_mem_inst_size;
module_inst->global_data.ptr = p;
module_inst->global_data_size = module->global_data_size;
if (!global_instantiate(module_inst, module, error_buf, error_buf_size))
@ -427,15 +656,15 @@ aot_instantiate(AOTModule *module, bool is_sub_inst,
/* Initialize table info */
p += module->global_data_size;
module_inst->table_data.ptr = p;
module_inst->table_size = module->table_size;
module_inst->table_size = table_size;
/* Set all elements to -1 to mark them as uninitialized elements */
memset(module_inst->table_data.ptr, -1, (uint32)table_data_size);
if (!table_instantiate(module_inst, module, error_buf, error_buf_size))
goto fail;
/* Initialize memory space */
if (!memory_instantiate(module_inst, module, heap_size,
error_buf, error_buf_size))
if (!memories_instantiate(module_inst, module, heap_size,
error_buf, error_buf_size))
goto fail;
/* Initialize function pointers */
@ -446,19 +675,24 @@ aot_instantiate(AOTModule *module, bool is_sub_inst,
if (!init_func_type_indexes(module_inst, module, error_buf, error_buf_size))
goto fail;
#if WASM_ENABLE_LIBC_WASI != 0
if (heap_size > 0
&& !wasm_runtime_init_wasi((WASMModuleInstanceCommon*)module_inst,
module->wasi_args.dir_list,
module->wasi_args.dir_count,
module->wasi_args.map_dir_list,
module->wasi_args.map_dir_count,
module->wasi_args.env,
module->wasi_args.env_count,
module->wasi_args.argv,
module->wasi_args.argc,
error_buf, error_buf_size))
if (!create_exports(module_inst, module, error_buf, error_buf_size))
goto fail;
#if WASM_ENABLE_LIBC_WASI != 0
if (!is_sub_inst) {
if (heap_size > 0
&& !wasm_runtime_init_wasi((WASMModuleInstanceCommon*)module_inst,
module->wasi_args.dir_list,
module->wasi_args.dir_count,
module->wasi_args.map_dir_list,
module->wasi_args.map_dir_count,
module->wasi_args.env,
module->wasi_args.env_count,
module->wasi_args.argv,
module->wasi_args.argc,
error_buf, error_buf_size))
goto fail;
}
#endif
/* Initialize the thread related data */
@ -478,6 +712,25 @@ aot_instantiate(AOTModule *module, bool is_sub_inst,
goto fail;
}
#if WASM_ENABLE_BULK_MEMORY != 0
#if WASM_ENABLE_LIBC_WASI != 0
if (!module->is_wasi_module) {
#endif
/* Only execute the memory init function for main instance because
the data segments will be dropped once initialized.
*/
if (!is_sub_inst) {
if (!execute_memory_init_function(module_inst)) {
set_error_buf(error_buf, error_buf_size,
module_inst->cur_exception);
goto fail;
}
}
#if WASM_ENABLE_LIBC_WASI != 0
}
#endif
#endif
return module_inst;
fail:
@ -498,17 +751,11 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst)
wasm_runtime_destroy_wasi((WASMModuleInstanceCommon*)module_inst);
#endif
if (module_inst->heap_handle.ptr)
mem_allocator_destroy(module_inst->heap_handle.ptr);
if (module_inst->memories.ptr)
memories_deinstantiate(module_inst);
if (module_inst->heap_data.ptr) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
wasm_runtime_free(module_inst->heap_data.ptr);
#else
os_munmap((uint8*)module_inst->memory_data.ptr - 2 * (uint64)BH_GB,
8 * (uint64)BH_GB);
#endif
}
if (module_inst->export_funcs.ptr)
wasm_runtime_free(module_inst->export_funcs.ptr);
if (module_inst->func_ptrs.ptr)
wasm_runtime_free(module_inst->func_ptrs.ptr);
@ -524,11 +771,12 @@ aot_lookup_function(const AOTModuleInstance *module_inst,
const char *name, const char *signature)
{
uint32 i;
AOTModule *module = (AOTModule*)module_inst->aot_module.ptr;
AOTFunctionInstance *export_funcs = (AOTFunctionInstance *)
module_inst->export_funcs.ptr;
for (i = 0; i < module->export_func_count; i++)
if (!strcmp(module->export_funcs[i].func_name, name))
return &module->export_funcs[i];
for (i = 0; i < module_inst->export_func_count; i++)
if (!strcmp(export_funcs[i].func_name, name))
return &export_funcs[i];
(void)signature;
return NULL;
}
@ -565,6 +813,7 @@ static void
aot_signal_handler(void *sig_addr)
{
AOTModuleInstance *module_inst;
AOTMemoryInstance *memory_inst;
WASMJmpBuf *jmpbuf_node;
uint8 *mapped_mem_start_addr, *mapped_mem_end_addr;
uint8 *stack_min_addr;
@ -577,17 +826,22 @@ aot_signal_handler(void *sig_addr)
&& (jmpbuf_node = aot_exec_env->jmpbuf_stack_top)) {
/* Get mapped mem info of current instance */
module_inst = (AOTModuleInstance *)aot_exec_env->module_inst;
mapped_mem_start_addr = (uint8*)module_inst->memory_data.ptr
- 2 * (uint64)BH_GB;
mapped_mem_end_addr = (uint8*)module_inst->memory_data.ptr
+ 6 * (uint64)BH_GB;
/* Get the default memory instance */
memory_inst = aot_get_default_memory(module_inst);
if (memory_inst) {
mapped_mem_start_addr = (uint8*)memory_inst->memory_data.ptr
- 2 * (uint64)BH_GB;
mapped_mem_end_addr = (uint8*)memory_inst->memory_data.ptr
+ 6 * (uint64)BH_GB;
}
/* Get stack info of current thread */
page_size = os_getpagesize();
stack_min_addr = get_stack_min_addr(aot_exec_env, page_size);
if (mapped_mem_start_addr <= (uint8*)sig_addr
&& (uint8*)sig_addr < mapped_mem_end_addr) {
if (memory_inst
&& (mapped_mem_start_addr <= (uint8*)sig_addr
&& (uint8*)sig_addr < mapped_mem_end_addr)) {
/* The address which causes segmentation fault is inside
aot instance's guard regions */
aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS);
@ -721,7 +975,7 @@ aot_call_function(WASMExecEnv *exec_env,
unsigned argc, uint32 argv[])
{
AOTModuleInstance *module_inst = (AOTModuleInstance*)exec_env->module_inst;
AOTFuncType *func_type = function->func_type;
AOTFuncType *func_type = function->u.func.func_type;
uint32 result_count = func_type->result_count;
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
bool ret;
@ -758,7 +1012,7 @@ aot_call_function(WASMExecEnv *exec_env,
cell_num += wasm_value_type_cell_num(ext_ret_types[i]);
}
ret = invoke_native_internal(exec_env, function->func_ptr,
ret = invoke_native_internal(exec_env, function->u.func.func_ptr,
func_type, NULL, NULL, argv1, argc, argv);
if (!ret || aot_get_exception(module_inst)) {
if (argv1 != argv1_buf)
@ -789,7 +1043,7 @@ aot_call_function(WASMExecEnv *exec_env,
return true;
}
else {
ret = invoke_native_internal(exec_env, function->func_ptr,
ret = invoke_native_internal(exec_env, function->u.func.func_ptr,
func_type, NULL, NULL, argv, argc, argv);
return ret && !aot_get_exception(module_inst) ? true : false;
}
@ -894,24 +1148,26 @@ int32
aot_module_malloc(AOTModuleInstance *module_inst, uint32 size,
void **p_native_addr)
{
uint8 *addr = mem_allocator_malloc(module_inst->heap_handle.ptr, size);
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
uint8 *addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size);
if (!addr) {
aot_set_exception(module_inst, "out of memory");
return 0;
}
if (p_native_addr)
*p_native_addr = addr;
return (int32)(addr - (uint8*)module_inst->memory_data.ptr);
return (int32)(addr - (uint8*)memory_inst->memory_data.ptr);
}
void
aot_module_free(AOTModuleInstance *module_inst, int32 ptr)
{
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
if (ptr) {
uint8 *addr = (uint8*)module_inst->memory_data.ptr + ptr;
if ((uint8*)module_inst->heap_data.ptr < addr
&& addr < (uint8*)module_inst->memory_data.ptr)
mem_allocator_free(module_inst->heap_handle.ptr, addr);
uint8 *addr = (uint8*)memory_inst->memory_data.ptr + ptr;
if ((uint8*)memory_inst->heap_data.ptr < addr
&& addr < (uint8*)memory_inst->memory_data.ptr)
mem_allocator_free(memory_inst->heap_handle.ptr, addr);
}
}
@ -934,13 +1190,14 @@ bool
aot_validate_app_addr(AOTModuleInstance *module_inst,
int32 app_offset, uint32 size)
{
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
/* integer overflow check */
if(app_offset + (int32)size < app_offset) {
goto fail;
}
if (module_inst->heap_base_offset <= app_offset
&& app_offset + (int32)size <= (int32)module_inst->memory_data_size) {
if (memory_inst->heap_base_offset <= app_offset
&& app_offset + (int32)size <= (int32)memory_inst->memory_data_size) {
return true;
}
fail:
@ -953,15 +1210,16 @@ aot_validate_native_addr(AOTModuleInstance *module_inst,
void *native_ptr, uint32 size)
{
uint8 *addr = (uint8*)native_ptr;
int32 memory_data_size = (int32)module_inst->memory_data_size;
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
int32 memory_data_size = (int32)memory_inst->memory_data_size;
/* integer overflow check */
if (addr + size < addr) {
goto fail;
}
if ((uint8*)module_inst->heap_data.ptr <= addr
&& addr + size <= (uint8*)module_inst->memory_data.ptr
if ((uint8*)memory_inst->heap_data.ptr <= addr
&& addr + size <= (uint8*)memory_inst->memory_data.ptr
+ memory_data_size) {
return true;
}
@ -973,11 +1231,12 @@ fail:
void *
aot_addr_app_to_native(AOTModuleInstance *module_inst, int32 app_offset)
{
int32 memory_data_size = (int32)module_inst->memory_data_size;
uint8 *addr = (uint8 *)module_inst->memory_data.ptr + app_offset;
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
int32 memory_data_size = (int32)memory_inst->memory_data_size;
uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + app_offset;
if ((uint8*)module_inst->heap_data.ptr <= addr
&& addr < (uint8*)module_inst->memory_data.ptr
if ((uint8*)memory_inst->heap_data.ptr <= addr
&& addr < (uint8*)memory_inst->memory_data.ptr
+ memory_data_size)
return addr;
return NULL;
@ -987,12 +1246,13 @@ int32
aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr)
{
uint8 *addr = (uint8*)native_ptr;
int32 memory_data_size = (int32)module_inst->memory_data_size;
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
int32 memory_data_size = (int32)memory_inst->memory_data_size;
if ((uint8*)module_inst->heap_data.ptr <= addr
&& addr < (uint8*)module_inst->memory_data.ptr
if ((uint8*)memory_inst->heap_data.ptr <= addr
&& addr < (uint8*)memory_inst->memory_data.ptr
+ memory_data_size)
return (int32)(addr - (uint8*)module_inst->memory_data.ptr);
return (int32)(addr - (uint8*)memory_inst->memory_data.ptr);
return 0;
}
@ -1002,12 +1262,13 @@ aot_get_app_addr_range(AOTModuleInstance *module_inst,
int32 *p_app_start_offset,
int32 *p_app_end_offset)
{
int32 memory_data_size = (int32)module_inst->memory_data_size;
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
int32 memory_data_size = (int32)memory_inst->memory_data_size;
if (module_inst->heap_base_offset <= app_offset
if (memory_inst->heap_base_offset <= app_offset
&& app_offset < memory_data_size) {
if (p_app_start_offset)
*p_app_start_offset = module_inst->heap_base_offset;
*p_app_start_offset = memory_inst->heap_base_offset;
if (p_app_end_offset)
*p_app_end_offset = memory_data_size;
return true;
@ -1022,15 +1283,16 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst,
uint8 **p_native_end_addr)
{
uint8 *addr = (uint8*)native_ptr;
int32 memory_data_size = (int32)module_inst->memory_data_size;
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
int32 memory_data_size = (int32)memory_inst->memory_data_size;
if ((uint8*)module_inst->heap_data.ptr <= addr
&& addr < (uint8*)module_inst->memory_data.ptr
if ((uint8*)memory_inst->heap_data.ptr <= addr
&& addr < (uint8*)memory_inst->memory_data.ptr
+ memory_data_size) {
if (p_native_start_addr)
*p_native_start_addr = (uint8*)module_inst->heap_data.ptr;
*p_native_start_addr = (uint8*)memory_inst->heap_data.ptr;
if (p_native_end_addr)
*p_native_end_addr = (uint8*)module_inst->memory_data.ptr
*p_native_end_addr = (uint8*)memory_inst->memory_data.ptr
+ memory_data_size;
return true;
}
@ -1041,18 +1303,19 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst,
bool
aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
{
uint8 *heap_data_old = module_inst->heap_data.ptr, *heap_data;
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
uint8 *heap_data_old = memory_inst->heap_data.ptr, *heap_data;
uint32 num_bytes_per_page =
((AOTModule*)module_inst->aot_module.ptr)->num_bytes_per_page;
uint32 cur_page_count = module_inst->mem_cur_page_count;
uint32 max_page_count = module_inst->mem_max_page_count;
((AOTModule*)module_inst->aot_module.ptr)->memories[0].num_bytes_per_page;
uint32 cur_page_count = memory_inst->mem_cur_page_count;
uint32 max_page_count = memory_inst->mem_max_page_count;
uint32 total_page_count = cur_page_count + inc_page_count;
uint64 memory_data_size = (uint64)num_bytes_per_page * total_page_count;
uint32 heap_size = (uint32)((uint8*)module_inst->memory_data.ptr
- (uint8*)module_inst->heap_data.ptr);
uint32 total_size_old = heap_size + module_inst->memory_data_size;
uint32 heap_size = (uint32)((uint8*)memory_inst->memory_data.ptr
- (uint8*)memory_inst->heap_data.ptr);
uint32 total_size_old = heap_size + memory_inst->memory_data_size;
uint64 total_size = heap_size + memory_data_size;
void *heap_handle_old = module_inst->heap_handle.ptr;
void *heap_handle_old = memory_inst->heap_handle.ptr;
if (inc_page_count <= 0)
/* No need to enlarge memory */
@ -1069,16 +1332,25 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
return false;
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (memory_inst->is_shared) {
/* For shared memory, we have reserved the maximum spaces during
instantiate, only change the cur_page_count here */
memory_inst->mem_cur_page_count = total_page_count;
return true;
}
#endif
if (heap_size > 0) {
/* Destroy heap's lock firstly, if its memory is re-allocated,
we cannot access its lock again. */
mem_allocator_destroy_lock(module_inst->heap_handle.ptr);
mem_allocator_destroy_lock(memory_inst->heap_handle.ptr);
}
if (!(heap_data = wasm_runtime_realloc(heap_data_old, (uint32)total_size))) {
if (!(heap_data = wasm_runtime_malloc((uint32)total_size))) {
if (heap_size > 0) {
/* Restore heap's lock if memory re-alloc failed */
mem_allocator_reinit_lock(module_inst->heap_handle.ptr);
mem_allocator_reinit_lock(memory_inst->heap_handle.ptr);
}
aot_set_exception(module_inst, "fail to enlarge memory.");
return false;
@ -1091,39 +1363,40 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
memset(heap_data + total_size_old,
0, (uint32)total_size - total_size_old);
module_inst->heap_data.ptr = heap_data;
module_inst->heap_data_end.ptr = heap_data + heap_size;
memory_inst->heap_data.ptr = heap_data;
memory_inst->heap_data_end.ptr = heap_data + heap_size;
if (heap_size > 0) {
module_inst->heap_handle.ptr = (uint8*)heap_handle_old
memory_inst->heap_handle.ptr = (uint8*)heap_handle_old
+ (heap_data - heap_data_old);
if (mem_allocator_migrate(module_inst->heap_handle.ptr,
if (mem_allocator_migrate(memory_inst->heap_handle.ptr,
heap_handle_old) != 0) {
aot_set_exception(module_inst, "fail to enlarge memory.");
return false;
}
}
module_inst->mem_cur_page_count = total_page_count;
module_inst->memory_data_size = (uint32)memory_data_size;
module_inst->memory_data.ptr = (uint8*)heap_data + heap_size;
module_inst->memory_data_end.ptr = (uint8*)module_inst->memory_data.ptr
memory_inst->mem_cur_page_count = total_page_count;
memory_inst->memory_data_size = (uint32)memory_data_size;
memory_inst->memory_data.ptr = (uint8*)heap_data + heap_size;
memory_inst->memory_data_end.ptr = (uint8*)memory_inst->memory_data.ptr
+ (uint32)memory_data_size;
module_inst->mem_bound_check_1byte = module_inst->memory_data_size - 1;
module_inst->mem_bound_check_2bytes = module_inst->memory_data_size - 2;
module_inst->mem_bound_check_4bytes = module_inst->memory_data_size - 4;
module_inst->mem_bound_check_8bytes = module_inst->memory_data_size - 8;
memory_inst->mem_bound_check_1byte = memory_inst->memory_data_size - 1;
memory_inst->mem_bound_check_2bytes = memory_inst->memory_data_size - 2;
memory_inst->mem_bound_check_4bytes = memory_inst->memory_data_size - 4;
memory_inst->mem_bound_check_8bytes = memory_inst->memory_data_size - 8;
return true;
}
#else
bool
aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
{
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
uint32 num_bytes_per_page =
((AOTModule*)module_inst->aot_module.ptr)->num_bytes_per_page;
uint32 cur_page_count = module_inst->mem_cur_page_count;
uint32 max_page_count = module_inst->mem_max_page_count;
((AOTModule*)module_inst->aot_module.ptr)->memories[0].num_bytes_per_page;
uint32 cur_page_count = memory_inst->mem_cur_page_count;
uint32 max_page_count = memory_inst->mem_max_page_count;
uint32 total_page_count = cur_page_count + inc_page_count;
uint64 memory_data_size = (uint64)num_bytes_per_page * total_page_count;
@ -1137,24 +1410,24 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
return false;
}
if (os_mprotect(module_inst->memory_data.ptr, memory_data_size,
if (os_mprotect(memory_inst->memory_data.ptr, memory_data_size,
MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) {
aot_set_exception(module_inst, "fail to enlarge memory.");
return false;
}
memset(module_inst->memory_data_end.ptr, 0,
memset(memory_inst->memory_data_end.ptr, 0,
num_bytes_per_page * inc_page_count);
module_inst->mem_cur_page_count = total_page_count;
module_inst->memory_data_size = (uint32)memory_data_size;
module_inst->memory_data_end.ptr = (uint8*)module_inst->memory_data.ptr
memory_inst->mem_cur_page_count = total_page_count;
memory_inst->memory_data_size = (uint32)memory_data_size;
memory_inst->memory_data_end.ptr = (uint8*)memory_inst->memory_data.ptr
+ (uint32)memory_data_size;
module_inst->mem_bound_check_1byte = module_inst->memory_data_size - 1;
module_inst->mem_bound_check_2bytes = module_inst->memory_data_size - 2;
module_inst->mem_bound_check_4bytes = module_inst->memory_data_size - 4;
module_inst->mem_bound_check_8bytes = module_inst->memory_data_size - 8;
memory_inst->mem_bound_check_1byte = memory_inst->memory_data_size - 1;
memory_inst->mem_bound_check_2bytes = memory_inst->memory_data_size - 2;
memory_inst->mem_bound_check_4bytes = memory_inst->memory_data_size - 4;
memory_inst->mem_bound_check_8bytes = memory_inst->memory_data_size - 8;
return true;
}
#endif
@ -1389,6 +1662,7 @@ bool
aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index,
uint32 offset, uint32 len, uint32 dst)
{
AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
AOTModule *aot_module;
uint8 *data = NULL;
uint8 *maddr;
@ -1416,7 +1690,7 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index,
maddr = aot_addr_app_to_native(module_inst, dst);
bh_memcpy_s(maddr, module_inst->memory_data_size - dst,
bh_memcpy_s(maddr, memory_inst->memory_data_size - dst,
data + offset, len);
return true;
}
@ -1441,3 +1715,68 @@ aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index)
return true;
}
#endif /* WASM_ENABLE_BULK_MEMORY */
#if WASM_ENABLE_THREAD_MGR != 0
bool
aot_set_aux_stack(WASMExecEnv *exec_env,
uint32 start_offset, uint32 size)
{
AOTModuleInstance *module_inst =
(AOTModuleInstance*)exec_env->module_inst;
AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;
uint32 stack_top_idx =
module->llvm_aux_stack_global_index;
uint32 data_end =
module->llvm_aux_data_end;
uint32 stack_bottom =
module->llvm_aux_stack_bottom;
bool is_stack_before_data =
stack_bottom < data_end ? true : false;
/* Check the aux stack space, currently we don't allocate space in heap */
if ((is_stack_before_data && (size > start_offset))
|| ((!is_stack_before_data) && (start_offset - data_end < size)))
return false;
if ((stack_bottom != (uint32)-1) && (stack_top_idx != (uint32)-1)) {
/* The aux stack top is a wasm global,
set the initial value for the global */
uint32 global_offset =
module->globals[stack_top_idx].data_offset;
uint8 *global_addr = module_inst->global_data.ptr + global_offset;
*(int32*)global_addr = start_offset;
/* The aux stack boundary is a constant value,
set the value to exec_env */
exec_env->aux_stack_boundary = start_offset - size;
return true;
}
return false;
}
bool
aot_get_aux_stack(WASMExecEnv *exec_env,
uint32 *start_offset, uint32 *size)
{
AOTModuleInstance *module_inst =
(AOTModuleInstance*)exec_env->module_inst;
AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;
/* The aux stack information is resolved in loader
and store in module */
uint32 stack_bottom = module->llvm_aux_stack_bottom;
uint32 total_aux_stack_size = module->llvm_aux_stack_size;
if (stack_bottom != 0 && total_aux_stack_size != 0) {
if (start_offset)
*start_offset = stack_bottom;
if (size)
*size = total_aux_stack_size;
return true;
}
return false;
}
#endif