diff --git a/core/app-mgr/app-manager/module_wasm_app.c b/core/app-mgr/app-manager/module_wasm_app.c index 4d30c4a0..a84e760d 100644 --- a/core/app-mgr/app-manager/module_wasm_app.c +++ b/core/app-mgr/app-manager/module_wasm_app.c @@ -13,13 +13,14 @@ #include "event.h" #include "watchdog.h" #include "runtime_lib.h" +#include "wasm.h" #if WASM_ENABLE_AOT != 0 #include "aot_export.h" #endif #if WASM_ENABLE_INTERP != 0 || WASM_ENABLE_JIT != 0 /* Wasm bytecode file 4 version bytes */ -static uint8 wasm_bytecode_version[] = { +static uint8 wasm_bytecode_version[4] = { (uint8) 0x01, (uint8) 0x00, (uint8) 0x00, @@ -29,12 +30,13 @@ static uint8 wasm_bytecode_version[] = { #if WASM_ENABLE_AOT != 0 /* Wasm aot file 4 version bytes */ -static uint8 wasm_aot_version[] = { - (uint8) 0x01, +static uint8 wasm_aot_version[4] = { + (uint8) 0x02, (uint8) 0x00, (uint8) 0x00, (uint8) 0x00 }; +#endif static union { int a; @@ -42,8 +44,6 @@ static union { } __ue = { .a = 1 }; #define is_little_endian() (__ue.b == 1) -#endif - /* Wasm App Install Request Receiving Phase */ typedef enum wasm_app_install_req_recv_phase_t { Phase_Req_Ver, @@ -163,14 +163,6 @@ module_interface wasm_app_module_interface = { wasm_app_module_on_install_request_byte_arrive }; -static unsigned -align_uint(unsigned v, unsigned b) -{ - unsigned m = b - 1; - return (v + m) & ~m; -} - -#if WASM_ENABLE_AOT != 0 static void exchange_uint32(uint8 *p_data) { @@ -182,7 +174,6 @@ exchange_uint32(uint8 *p_data) *(p_data + 1) = *(p_data + 2); *(p_data + 2) = value; } -#endif static wasm_function_inst_t app_manager_lookup_function(const wasm_module_inst_t module_inst, @@ -546,7 +537,21 @@ cleanup_app_resource(module_data *m_data) static bool wasm_app_module_init(void) { - /* wasm runtime is already initialized by main func */ + uint32 version; + +#if WASM_ENABLE_INTERP != 0 || WASM_ENABLE_JIT != 0 + version = WASM_CURRENT_VERSION; + if (!is_little_endian()) + exchange_uint32((uint8 *)&version); + bh_memcpy_s(wasm_bytecode_version, 4, &version, 4); +#endif + +#if WASM_ENABLE_AOT != 0 + version = AOT_CURRENT_VERSION; + if (!is_little_endian()) + exchange_uint32((uint8 *)&version); + bh_memcpy_s(wasm_aot_version, 4, &version, 4); +#endif return true; } diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index f5bd2bb7..312f8c3d 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1039,10 +1039,13 @@ load_init_data_section(const uint8 *buf, const uint8 *buf_end, return false; } - read_uint32(p, p_end, module->llvm_aux_data_end); - read_uint32(p, p_end, module->llvm_aux_stack_bottom); - read_uint32(p, p_end, module->llvm_aux_stack_size); - read_uint32(p, p_end, module->llvm_aux_stack_global_index); + read_uint32(p, p_end, module->aux_data_end_global_index); + read_uint32(p, p_end, module->aux_data_end); + read_uint32(p, p_end, module->aux_heap_base_global_index); + read_uint32(p, p_end, module->aux_heap_base); + read_uint32(p, p_end, module->aux_stack_top_global_index); + read_uint32(p, p_end, module->aux_stack_bottom); + read_uint32(p, p_end, module->aux_stack_size); if (!load_object_data_sections_info(&p, p_end, module, error_buf, error_buf_size)) @@ -1613,6 +1616,9 @@ load_from_sections(AOTModule *module, AOTSection *sections, AOTSection *section = sections; const uint8 *buf, *buf_end; uint32 last_section_type = (uint32)-1, section_type; + uint32 i, func_index, func_type_index; + AOTFuncType *func_type; + AOTExport *exports; while (section) { buf = section->section_body; @@ -1670,6 +1676,42 @@ load_from_sections(AOTModule *module, AOTSection *sections, return false; } + /* Resolve malloc and free function */ + module->malloc_func_index = (uint32)-1; + module->free_func_index = (uint32)-1; + + exports = module->exports; + for (i = 0; i < module->export_count; i++) { + if (exports[i].kind == EXPORT_KIND_FUNC + && exports[i].index >= module->import_func_count) { + if (!strcmp(exports[i].name, "malloc")) { + func_index = exports[i].index - module->import_func_count; + func_type_index = module->func_type_indexes[func_index]; + func_type = module->func_types[func_type_index]; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == VALUE_TYPE_I32 + && func_type->types[1] == VALUE_TYPE_I32) { + module->malloc_func_index = func_index; + LOG_VERBOSE("Found malloc function, index: %u", + exports[i].index); + } + } + else if (!strcmp(exports[i].name, "free")) { + func_index = exports[i].index - module->import_func_count; + func_type_index = module->func_type_indexes[func_index]; + func_type = module->func_types[func_type_index]; + if (func_type->param_count == 1 + && func_type->result_count == 0 + && func_type->types[0] == VALUE_TYPE_I32) { + module->free_func_index = func_index; + LOG_VERBOSE("Found free function, index: %u", + exports[i].index); + } + } + } + } + /* Flush data cache before executing AOT code, * otherwise unpredictable behavior can occur. */ os_dcache_flush(); @@ -2018,14 +2060,17 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, - module->import_func_count]; } } - else { - module->start_function = NULL; - } - module->llvm_aux_data_end = comp_data->llvm_aux_data_end; - module->llvm_aux_stack_bottom = comp_data->llvm_aux_stack_bottom; - module->llvm_aux_stack_size = comp_data->llvm_aux_stack_size; - module->llvm_aux_stack_global_index = comp_data->llvm_aux_stack_global_index; + module->malloc_func_index = comp_data->malloc_func_index; + module->free_func_index = comp_data->free_func_index; + + module->aux_data_end_global_index = comp_data->aux_data_end_global_index; + module->aux_data_end = comp_data->aux_data_end; + module->aux_heap_base_global_index = comp_data->aux_heap_base_global_index; + module->aux_heap_base = comp_data->aux_heap_base; + module->aux_stack_top_global_index = comp_data->aux_stack_top_global_index; + module->aux_stack_bottom = comp_data->aux_stack_bottom; + module->aux_stack_size = comp_data->aux_stack_size; module->code = NULL; module->code_size = 0; diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 107b5f93..79235d61 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -176,9 +176,9 @@ memories_deinstantiate(AOTModuleInstance *module_inst) if (memory_inst->heap_data.ptr) { #ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(memory_inst->heap_data.ptr); + wasm_runtime_free(memory_inst->memory_data.ptr); #else - os_munmap((uint8*)memory_inst->memory_data.ptr - 2 * (uint64)BH_GB, + os_munmap((uint8*)memory_inst->memory_data.ptr, 8 * (uint64)BH_GB); #endif } @@ -193,19 +193,26 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint32 heap_size, char *error_buf, uint32 error_buf_size) { void *heap_handle; - uint64 memory_data_size = (uint64)memory->num_bytes_per_page - * memory->mem_init_page_count; - uint64 total_size = heap_size + memory_data_size; - uint8 *p; + uint32 num_bytes_per_page = memory->num_bytes_per_page; + uint32 init_page_count = memory->mem_init_page_count; + uint32 max_page_count = memory->mem_max_page_count; + uint32 inc_page_count, aux_heap_base, global_idx; + uint32 bytes_of_last_page, bytes_to_page_end; + uint32 heap_offset = num_bytes_per_page *init_page_count; + uint64 total_size; + uint8 *p, *global_addr; +#ifdef OS_ENABLE_HW_BOUND_CHECK + uint8 *mapped_mem; + uint64 map_size = 8 * (uint64)BH_GB; + uint64 page_size = os_getpagesize(); +#endif #if WASM_ENABLE_SHARED_MEMORY != 0 - AOTMemoryInstance *shared_memory_instance; bool is_shared_memory = memory->memory_flags & 0x02 ? true : false; - uint64 max_memory_data_size = (uint64)memory->num_bytes_per_page - * memory->mem_max_page_count; /* Shared memory */ if (is_shared_memory) { + AOTMemoryInstance *shared_memory_instance; WASMSharedMemNode *node = wasm_module_get_shared_memory((WASMModuleCommon *)module); /* If the memory of this module has been instantiated, @@ -219,16 +226,94 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, (AOTMemoryInstance *)shared_memory_get_memory_inst(node); bh_assert(shared_memory_instance); - /* Set the shared memory flag, so the runtime will get the - actual memory inst through module_inst->memories array */ - memory_inst->is_shared = true; (void)ref_count; return shared_memory_instance; } -#ifndef OS_ENABLE_HW_BOUND_CHECK - /* Allocate max page for shared memory */ - total_size = heap_size + max_memory_data_size; + } #endif + + if (heap_size > 0 + && module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } + + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count*/ + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + if (num_bytes_per_page < heap_size) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return NULL; + } + } + else if (heap_size > 0) { + if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base < num_bytes_per_page + * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = (heap_size - bytes_to_page_end + + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; + } + + /* Adjust __heap_base global value */ + global_idx = module->aux_heap_base_global_index + - module->import_global_count; + global_addr = (uint8*)module_inst->global_data.ptr + + module->globals[global_idx].data_offset; + *(uint32 *)global_addr = aux_heap_base; + LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = (heap_size + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = num_bytes_per_page * init_page_count; + heap_size = num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > 65536) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return NULL; + } + if (max_page_count > 65536) + max_page_count = 65536; + } + + LOG_VERBOSE("Memory instantiate:"); + LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", + num_bytes_per_page, init_page_count, max_page_count); + LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); + + total_size = (uint64)num_bytes_per_page * init_page_count; +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + /* Allocate max page for shared memory */ + total_size = (uint64)num_bytes_per_page * max_page_count; } #endif @@ -238,24 +323,21 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return NULL; } #else - uint8 *mapped_mem; - uint64 map_size = 8 * (uint64)BH_GB; + total_size = (total_size + page_size - 1) & ~(page_size - 1); - /* Totally 8G is mapped, the opcode load/store address range is -2G to 6G: - * ea = i + memarg.offset - * i is i32, the range is -2G to 2G - * memarg.offset is u32, the range is 0 to 4G - * so the range of ea is -2G to 6G + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G */ if (total_size >= UINT32_MAX - || !(mapped_mem = os_mmap(NULL, map_size, - MMAP_PROT_NONE, MMAP_MAP_NONE))) { + || !(p = mapped_mem = os_mmap(NULL, map_size, + MMAP_PROT_NONE, MMAP_MAP_NONE))) { set_error_buf(error_buf, error_buf_size, "AOT module instantiate failed: mmap memory failed."); return NULL; } - p = mapped_mem + 2 * (uint64)BH_GB - heap_size; if (os_mprotect(p, total_size, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { set_error_buf(error_buf, error_buf_size, "AOT module instantiate failed: mprotec memory failed."); @@ -263,15 +345,21 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return NULL; } memset(p, 0, (uint32)total_size); -#endif +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ memory_inst->module_type = Wasm_Module_AoT; + memory_inst->num_bytes_per_page = num_bytes_per_page; + memory_inst->cur_page_count = init_page_count; + memory_inst->max_page_count = max_page_count; + + /* Init memory info */ + memory_inst->memory_data.ptr = p; + memory_inst->memory_data_end.ptr = p + (uint32)total_size; + memory_inst->memory_data_size = (uint32)total_size; + /* Initialize heap info */ - memory_inst->heap_data.ptr = p; - p += heap_size; - memory_inst->heap_data_end.ptr = p; - memory_inst->heap_data_size = heap_size; - memory_inst->heap_base_offset = -(int32)heap_size; + memory_inst->heap_data.ptr = p + heap_offset; + memory_inst->heap_data_end.ptr = p + heap_offset + heap_size; if (heap_size > 0) { if (!(heap_handle = mem_allocator_create(memory_inst->heap_data.ptr, heap_size))) { @@ -283,31 +371,20 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, memory_inst->heap_handle.ptr = heap_handle; } - /* Init memory info */ - memory_inst->memory_data.ptr = p; -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (is_shared_memory) { - p += (uint32)max_memory_data_size; + if (total_size > 0) { + if (sizeof(uintptr_t) == sizeof(uint64)) { + memory_inst->mem_bound_check_1byte.u64 = total_size - 1; + memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; + memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; + memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; + } + else { + memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; + memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; + memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; + memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; + } } - else -#endif - { - p += (uint32)memory_data_size; - } - memory_inst->memory_data_end.ptr = p; - memory_inst->memory_data_size = (uint32)memory_data_size; - memory_inst->mem_cur_page_count = memory->mem_init_page_count; - memory_inst->mem_max_page_count = memory->mem_max_page_count; - - memory_inst->mem_bound_check_heap_base = memory_inst->heap_base_offset; - memory_inst->mem_bound_check_1byte = - (int64)memory_inst->memory_data_size - 1; - memory_inst->mem_bound_check_2bytes = - (int64)memory_inst->memory_data_size - 2; - memory_inst->mem_bound_check_4bytes = - (int64)memory_inst->memory_data_size - 4; - memory_inst->mem_bound_check_8bytes = - (int64)memory_inst->memory_data_size - 8; #if WASM_ENABLE_SHARED_MEMORY != 0 if (is_shared_memory) { @@ -333,11 +410,11 @@ fail2: #endif fail1: #ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(memory_inst->heap_data.ptr); + wasm_runtime_free(memory_inst->memory_data.ptr); #else os_munmap(mapped_mem, map_size); #endif - memory_inst->heap_data.ptr = NULL; + memory_inst->memory_data.ptr = NULL; return NULL; } @@ -632,9 +709,6 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, heap_size = align_uint(heap_size, 8); if (heap_size > APP_HEAP_SIZE_MAX) heap_size = APP_HEAP_SIZE_MAX; -#ifdef OS_ENABLE_HW_BOUND_CHECK - heap_size = align_uint(heap_size, os_getpagesize()); -#endif /* Allocate module instance, global data, table data and heap data */ if (!(module_inst = runtime_malloc(total_size, @@ -829,10 +903,9 @@ aot_signal_handler(void *sig_addr) /* Get the default memory instance */ memory_inst = aot_get_default_memory(module_inst); if (memory_inst) { - mapped_mem_start_addr = (uint8*)memory_inst->memory_data.ptr - - 2 * (uint64)BH_GB; + mapped_mem_start_addr = (uint8*)memory_inst->memory_data.ptr; mapped_mem_end_addr = (uint8*)memory_inst->memory_data.ptr - + 6 * (uint64)BH_GB; + + 8 * (uint64)BH_GB; } /* Get stack info of current thread */ @@ -1147,12 +1220,82 @@ aot_clear_exception(AOTModuleInstance *module_inst) module_inst->cur_exception[0] = '\0'; } +static bool +execute_malloc_function(AOTModuleInstance *module_inst, + AOTFunctionInstance *malloc_func, + uint32 size, uint32 *p_result) +{ + uint32 argv[2]; + bool ret; + + argv[0] = size; +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (aot_exec_env != NULL) { + bh_assert(aot_exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + ret = aot_call_function(aot_exec_env, malloc_func, 1, argv); + } + else +#endif + { + ret = aot_create_exec_env_and_call_function + (module_inst, malloc_func, 1, argv); + } + + if (ret) + *p_result = argv[0]; + return ret; +} + +static bool +execute_free_function(AOTModuleInstance *module_inst, + AOTFunctionInstance *free_func, + uint32 offset) +{ + uint32 argv[2]; + + argv[0] = offset; +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (aot_exec_env != NULL) { + bh_assert(aot_exec_env->module_inst + == (WASMModuleInstanceCommon *)module_inst); + return aot_call_function(aot_exec_env, free_func, 1, argv); + } + else +#endif + { + return aot_create_exec_env_and_call_function + (module_inst, free_func, 1, argv); + } +} + int32 aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, void **p_native_addr) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size); + AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + uint8 *addr = NULL; + uint32 offset = 0; + + if (memory_inst->heap_handle.ptr) { + addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size); + } + else if (module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1) { + AOTFunctionInstance *malloc_func = + aot_lookup_function(module_inst, "malloc", "(i)i"); + + bh_assert(malloc_func); + if (!execute_malloc_function(module_inst, malloc_func, + size, &offset)) { + return 0; + } + addr = offset + ? (uint8*)memory_inst->memory_data.ptr + offset + : NULL; + } + if (!addr) { aot_set_exception(module_inst, "out of memory"); return 0; @@ -1166,11 +1309,25 @@ void aot_module_free(AOTModuleInstance *module_inst, int32 ptr) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + if (ptr) { - uint8 *addr = (uint8*)memory_inst->memory_data.ptr + ptr; - if ((uint8*)memory_inst->heap_data.ptr < addr - && addr < (uint8*)memory_inst->memory_data.ptr) + uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + ptr; + if (memory_inst->heap_handle.ptr + &&(uint8 *)memory_inst->heap_data.ptr < addr + && addr < (uint8 *)memory_inst->heap_data_end.ptr) { mem_allocator_free(memory_inst->heap_handle.ptr, addr); + } + else if (module->malloc_func_index != (uint32)-1 + && module->free_func_index != (uint32)-1 + && (uint8 *)memory_inst->memory_data.ptr <= addr + && addr < (uint8 *)memory_inst->memory_data_end.ptr) { + AOTFunctionInstance *free_func = + aot_lookup_function(module_inst, "free", "(i)i"); + + bh_assert(free_func); + execute_free_function(module_inst, free_func, (uint32)ptr); + } } } @@ -1184,7 +1341,7 @@ aot_module_dup_data(AOTModuleInstance *module_inst, if (buffer_offset != 0) { buffer = aot_addr_app_to_native(module_inst, buffer_offset); - memcpy(buffer, src, size); + bh_memcpy_s(buffer, size, src, size); } return buffer_offset; } @@ -1194,13 +1351,12 @@ aot_validate_app_addr(AOTModuleInstance *module_inst, int32 app_offset, uint32 size) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + /* integer overflow check */ - if(app_offset + (int32)size < app_offset) { + if((uint32)app_offset + size < (uint32)app_offset) { goto fail; } - - if (memory_inst->heap_base_offset <= app_offset - && app_offset + (int32)size <= (int32)memory_inst->memory_data_size) { + if ((uint32)app_offset + size <= memory_inst->memory_data_size) { return true; } fail: @@ -1212,20 +1368,17 @@ bool aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr, uint32 size) { - uint8 *addr = (uint8*)native_ptr; + uint8 *addr = (uint8 *)native_ptr; AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - int32 memory_data_size = (int32)memory_inst->memory_data_size; /* integer overflow check */ if (addr + size < addr) { goto fail; } - if ((uint8*)memory_inst->heap_data.ptr <= addr - && addr + size <= (uint8*)memory_inst->memory_data.ptr - + memory_data_size) { + if ((uint8 *)memory_inst->memory_data.ptr <= addr + && addr + size <= (uint8 *)memory_inst->memory_data_end.ptr) return true; - } fail: aot_set_exception(module_inst, "out of bounds memory access"); return false; @@ -1235,12 +1388,10 @@ void * aot_addr_app_to_native(AOTModuleInstance *module_inst, int32 app_offset) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - int32 memory_data_size = (int32)memory_inst->memory_data_size; - uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + app_offset; + uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + (uint32)app_offset; - if ((uint8*)memory_inst->heap_data.ptr <= addr - && addr < (uint8*)memory_inst->memory_data.ptr - + memory_data_size) + if ((uint8 *)memory_inst->memory_data.ptr <= addr + && addr < (uint8 *)memory_inst->memory_data_end.ptr) return addr; return NULL; } @@ -1248,14 +1399,12 @@ aot_addr_app_to_native(AOTModuleInstance *module_inst, int32 app_offset) int32 aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr) { - uint8 *addr = (uint8*)native_ptr; + uint8 *addr = (uint8 *)native_ptr; AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - int32 memory_data_size = (int32)memory_inst->memory_data_size; - if ((uint8*)memory_inst->heap_data.ptr <= addr - && addr < (uint8*)memory_inst->memory_data.ptr - + memory_data_size) - return (int32)(addr - (uint8*)memory_inst->memory_data.ptr); + if ((uint8 *)memory_inst->memory_data.ptr <= addr + && addr < (uint8 *)memory_inst->memory_data_end.ptr) + return (int32)(addr - (uint8 *)memory_inst->memory_data.ptr); return 0; } @@ -1266,14 +1415,13 @@ aot_get_app_addr_range(AOTModuleInstance *module_inst, int32 *p_app_end_offset) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - int32 memory_data_size = (int32)memory_inst->memory_data_size; + uint32 memory_data_size = memory_inst->memory_data_size; - if (memory_inst->heap_base_offset <= app_offset - && app_offset < memory_data_size) { + if ((uint32)app_offset < memory_data_size) { if (p_app_start_offset) - *p_app_start_offset = memory_inst->heap_base_offset; + *p_app_start_offset = 0; if (p_app_end_offset) - *p_app_end_offset = memory_data_size; + *p_app_end_offset = (int32)memory_data_size; return true; } return false; @@ -1285,18 +1433,15 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 **p_native_start_addr, uint8 **p_native_end_addr) { - uint8 *addr = (uint8*)native_ptr; + uint8 *addr = (uint8 *)native_ptr; AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - int32 memory_data_size = (int32)memory_inst->memory_data_size; - if ((uint8*)memory_inst->heap_data.ptr <= addr - && addr < (uint8*)memory_inst->memory_data.ptr - + memory_data_size) { + if ((uint8 *)memory_inst->memory_data.ptr <= addr + && addr < (uint8 *)memory_inst->memory_data_end.ptr) { if (p_native_start_addr) - *p_native_start_addr = (uint8*)memory_inst->heap_data.ptr; + *p_native_start_addr = (uint8 *)memory_inst->memory_data.ptr; if (p_native_end_addr) - *p_native_end_addr = (uint8*)memory_inst->memory_data.ptr - + memory_data_size; + *p_native_end_addr = (uint8 *)memory_inst->memory_data_end.ptr; return true; } return false; @@ -1307,17 +1452,17 @@ bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint8 *heap_data_old = memory_inst->heap_data.ptr, *heap_data; - uint32 num_bytes_per_page = - ((AOTModule*)module_inst->aot_module.ptr)->memories[0].num_bytes_per_page; - uint32 cur_page_count = memory_inst->mem_cur_page_count; - uint32 max_page_count = memory_inst->mem_max_page_count; + uint32 num_bytes_per_page = memory_inst->num_bytes_per_page; + uint32 cur_page_count = memory_inst->cur_page_count; + uint32 max_page_count = memory_inst->max_page_count; uint32 total_page_count = cur_page_count + inc_page_count; - uint64 memory_data_size = (uint64)num_bytes_per_page * total_page_count; - uint32 heap_size = (uint32)((uint8*)memory_inst->memory_data.ptr - - (uint8*)memory_inst->heap_data.ptr); - uint32 total_size_old = heap_size + memory_inst->memory_data_size; - uint64 total_size = heap_size + memory_data_size; + uint32 total_size_old = memory_inst->memory_data_size; + uint64 total_size = (uint64)num_bytes_per_page * total_page_count; + uint32 heap_size = (uint32)((uint8 *)memory_inst->heap_data_end.ptr + - (uint8 *)memory_inst->heap_data.ptr); + uint8 *memory_data_old = (uint8 *)memory_inst->memory_data.ptr; + uint8 *heap_data_old = (uint8 *)memory_inst->heap_data.ptr; + uint8 *memory_data, *heap_data; void *heap_handle_old = memory_inst->heap_handle.ptr; if (inc_page_count <= 0) @@ -1339,7 +1484,7 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) if (memory_inst->is_shared) { /* For shared memory, we have reserved the maximum spaces during instantiate, only change the cur_page_count here */ - memory_inst->mem_cur_page_count = total_page_count; + memory_inst->cur_page_count = total_page_count; return true; } #endif @@ -1349,8 +1494,9 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) we cannot access its lock again. */ mem_allocator_destroy_lock(memory_inst->heap_handle.ptr); } - if (!(heap_data = wasm_runtime_realloc(heap_data_old, (uint32)total_size))) { - if (!(heap_data = wasm_runtime_malloc((uint32)total_size))) { + if (!(memory_data = wasm_runtime_realloc(memory_data_old, + (uint32)total_size))) { + if (!(memory_data = wasm_runtime_malloc((uint32)total_size))) { if (heap_size > 0) { /* Restore heap's lock if memory re-alloc failed */ mem_allocator_reinit_lock(memory_inst->heap_handle.ptr); @@ -1358,20 +1504,22 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) aot_set_exception(module_inst, "fail to enlarge memory."); return false; } - bh_memcpy_s(heap_data, (uint32)total_size, - heap_data_old, total_size_old); - wasm_runtime_free(heap_data_old); + bh_memcpy_s(memory_data, (uint32)total_size, + memory_data_old, total_size_old); + wasm_runtime_free(memory_data_old); } - memset(heap_data + total_size_old, + memset(memory_data + total_size_old, 0, (uint32)total_size - total_size_old); - memory_inst->heap_data.ptr = heap_data; - memory_inst->heap_data_end.ptr = heap_data + heap_size; + memory_inst->cur_page_count = total_page_count; + memory_inst->memory_data_size = (uint32)total_size; + memory_inst->memory_data.ptr = memory_data; + memory_inst->memory_data_end.ptr = memory_data + total_size; if (heap_size > 0) { - memory_inst->heap_handle.ptr = (uint8*)heap_handle_old - + (heap_data - heap_data_old); + memory_inst->heap_handle.ptr = (uint8 *)heap_handle_old + + (memory_data - memory_data_old); if (mem_allocator_migrate(memory_inst->heap_handle.ptr, heap_handle_old) != 0) { aot_set_exception(module_inst, "fail to enlarge memory."); @@ -1379,29 +1527,34 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) } } - memory_inst->mem_cur_page_count = total_page_count; - memory_inst->memory_data_size = (uint32)memory_data_size; - memory_inst->memory_data.ptr = (uint8*)heap_data + heap_size; - memory_inst->memory_data_end.ptr = (uint8*)memory_inst->memory_data.ptr - + (uint32)memory_data_size; + heap_data = heap_data_old + (memory_data - memory_data_old); + memory_inst->heap_data.ptr = heap_data; + memory_inst->heap_data_end.ptr = heap_data + heap_size; - memory_inst->mem_bound_check_1byte = memory_inst->memory_data_size - 1; - memory_inst->mem_bound_check_2bytes = memory_inst->memory_data_size - 2; - memory_inst->mem_bound_check_4bytes = memory_inst->memory_data_size - 4; - memory_inst->mem_bound_check_8bytes = memory_inst->memory_data_size - 8; + if (sizeof(uintptr_t) == sizeof(uint64)) { + memory_inst->mem_bound_check_1byte.u64 = total_size - 1; + memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; + memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; + memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; + } + else { + memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; + memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; + memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; + memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; + } return true; } -#else +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint32 num_bytes_per_page = - ((AOTModule*)module_inst->aot_module.ptr)->memories[0].num_bytes_per_page; - uint32 cur_page_count = memory_inst->mem_cur_page_count; - uint32 max_page_count = memory_inst->mem_max_page_count; + uint32 num_bytes_per_page = memory_inst->num_bytes_per_page; + uint32 cur_page_count = memory_inst->cur_page_count; + uint32 max_page_count = memory_inst->max_page_count; uint32 total_page_count = cur_page_count + inc_page_count; - uint64 memory_data_size = (uint64)num_bytes_per_page * total_page_count; + uint64 total_size = (uint64)num_bytes_per_page * total_page_count; if (inc_page_count <= 0) /* No need to enlarge memory */ @@ -1413,8 +1566,9 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) return false; } - if (os_mprotect(memory_inst->memory_data.ptr, memory_data_size, - MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { + if (os_mprotect(memory_inst->memory_data_end.ptr, + num_bytes_per_page * inc_page_count, + MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { aot_set_exception(module_inst, "fail to enlarge memory."); return false; } @@ -1422,18 +1576,26 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) memset(memory_inst->memory_data_end.ptr, 0, num_bytes_per_page * inc_page_count); - memory_inst->mem_cur_page_count = total_page_count; - memory_inst->memory_data_size = (uint32)memory_data_size; - memory_inst->memory_data_end.ptr = (uint8*)memory_inst->memory_data.ptr - + (uint32)memory_data_size; + memory_inst->cur_page_count = total_page_count; + memory_inst->memory_data_size = (uint32)total_size; + memory_inst->memory_data_end.ptr = (uint8 *)memory_inst->memory_data.ptr + + (uint32)total_size; - memory_inst->mem_bound_check_1byte = memory_inst->memory_data_size - 1; - memory_inst->mem_bound_check_2bytes = memory_inst->memory_data_size - 2; - memory_inst->mem_bound_check_4bytes = memory_inst->memory_data_size - 4; - memory_inst->mem_bound_check_8bytes = memory_inst->memory_data_size - 8; + if (sizeof(uintptr_t) == sizeof(uint64)) { + memory_inst->mem_bound_check_1byte.u64 = total_size - 1; + memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; + memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; + memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; + } + else { + memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; + memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; + memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; + memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; + } return true; } -#endif +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ bool aot_is_wasm_type_equal(AOTModuleInstance *module_inst, @@ -1728,21 +1890,17 @@ aot_set_aux_stack(WASMExecEnv *exec_env, (AOTModuleInstance*)exec_env->module_inst; AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; - uint32 stack_top_idx = - module->llvm_aux_stack_global_index; - uint32 data_end = - module->llvm_aux_data_end; - uint32 stack_bottom = - module->llvm_aux_stack_bottom; - bool is_stack_before_data = - stack_bottom < data_end ? true : false; + uint32 stack_top_idx = module->aux_stack_top_global_index; + uint32 data_end = module->aux_data_end; + uint32 stack_bottom = module->aux_stack_bottom; + bool is_stack_before_data = stack_bottom < data_end ? true : false; /* Check the aux stack space, currently we don't allocate space in heap */ if ((is_stack_before_data && (size > start_offset)) || ((!is_stack_before_data) && (start_offset - data_end < size))) return false; - if ((stack_bottom != (uint32)-1) && (stack_top_idx != (uint32)-1)) { + if (stack_top_idx != (uint32)-1) { /* The aux stack top is a wasm global, set the initial value for the global */ uint32 global_offset = @@ -1769,8 +1927,8 @@ aot_get_aux_stack(WASMExecEnv *exec_env, /* The aux stack information is resolved in loader and store in module */ - uint32 stack_bottom = module->llvm_aux_stack_bottom; - uint32 total_aux_stack_size = module->llvm_aux_stack_size; + uint32 stack_bottom = module->aux_stack_bottom; + uint32 total_aux_stack_size = module->aux_stack_size; if (stack_bottom != 0 && total_aux_stack_size != 0) { if (start_offset) diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 95b54a46..8f7a774a 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -147,6 +147,9 @@ typedef struct AOTModule { /* start function, point to AOTed/JITed function */ void *start_function; + uint32 malloc_func_index; + uint32 free_func_index; + /* AOTed code, NULL for JIT mode */ void *code; uint32 code_size; @@ -163,10 +166,25 @@ typedef struct AOTModule { /* constant string set */ HashMap *const_str_set; - uint32 llvm_aux_data_end; - uint32 llvm_aux_stack_bottom; - uint32 llvm_aux_stack_size; - uint32 llvm_aux_stack_global_index; + /* the index of auxiliary __data_end global, + -1 means unexported */ + uint32 aux_data_end_global_index; + /* auxiliary __data_end exported by wasm app */ + uint32 aux_data_end; + + /* the index of auxiliary __heap_base global, + -1 means unexported */ + uint32 aux_heap_base_global_index; + /* auxiliary __heap_base exported by wasm app */ + uint32 aux_heap_base; + + /* the index of auxiliary stack top global, + -1 means unexported */ + uint32 aux_stack_top_global_index; + /* auxiliary stack bottom resolved */ + uint32 aux_stack_bottom; + /* auxiliary stack size resolved */ + uint32 aux_stack_size; /* is jit mode or not */ bool is_jit_mode; @@ -188,31 +206,34 @@ typedef union { void *ptr; } AOTPointer; +typedef union { + uint64 u64; + uint32 u32[2]; +} MemBound; + typedef struct AOTMemoryInstance { uint32 module_type; /* shared memory flag */ bool is_shared; + /* memory space info */ - uint32 mem_cur_page_count; - uint32 mem_max_page_count; + uint32 num_bytes_per_page; + uint32 cur_page_count; + uint32 max_page_count; uint32 memory_data_size; - uint32 __padding__; AOTPointer memory_data; AOTPointer memory_data_end; /* heap space info */ - int32 heap_base_offset; - uint32 heap_data_size; AOTPointer heap_data; AOTPointer heap_data_end; AOTPointer heap_handle; /* boundary check constants for aot code */ - int64 mem_bound_check_heap_base; - int64 mem_bound_check_1byte; - int64 mem_bound_check_2bytes; - int64 mem_bound_check_4bytes; - int64 mem_bound_check_8bytes; + MemBound mem_bound_check_1byte; + MemBound mem_bound_check_2bytes; + MemBound mem_bound_check_4bytes; + MemBound mem_bound_check_8bytes; } AOTMemoryInstance; typedef struct AOTModuleInstance { diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index 35cf56c5..04a97e2f 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -375,6 +375,10 @@ aot_create_comp_data(WASMModule *module) } memset(comp_data->memories, 0, size); + if (!(module->import_memory_count + module->memory_count)) { + comp_data->memories[0].num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; + } + /* Set memory page count */ for (i = 0; i < module->import_memory_count + module->memory_count; i++) { if (i < module->import_memory_count) { @@ -487,13 +491,19 @@ aot_create_comp_data(WASMModule *module) && !(comp_data->funcs = aot_create_funcs(module))) goto fail; - /* Create llvm aux stack informations */ - comp_data->llvm_aux_stack_global_index = module->llvm_aux_stack_global_index; - comp_data->llvm_aux_data_end = module->llvm_aux_data_end; - comp_data->llvm_aux_stack_bottom = module->llvm_aux_stack_bottom; - comp_data->llvm_aux_stack_size = module->llvm_aux_stack_size; + /* Create aux data/heap/stack information */ + comp_data->aux_data_end_global_index = module->aux_data_end_global_index; + comp_data->aux_data_end = module->aux_data_end; + comp_data->aux_heap_base_global_index = module->aux_heap_base_global_index; + comp_data->aux_heap_base = module->aux_heap_base; + comp_data->aux_stack_top_global_index = module->aux_stack_top_global_index; + comp_data->aux_stack_bottom = module->aux_stack_bottom; + comp_data->aux_stack_size = module->aux_stack_size; comp_data->start_func_index = module->start_function; + comp_data->malloc_func_index = module->malloc_function; + comp_data->free_func_index = module->free_function; + comp_data->wasm_module = module; return comp_data; diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index abf86d85..ccceb75f 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -201,14 +201,19 @@ typedef struct AOTCompData { uint32 func_count; AOTFunc **funcs; - uint32 start_func_index; - uint32 addr_data_size; uint32 global_data_size; - uint32 llvm_aux_data_end; - uint32 llvm_aux_stack_bottom; - uint32 llvm_aux_stack_size; - uint32 llvm_aux_stack_global_index; + uint32 start_func_index; + uint32 malloc_func_index; + uint32 free_func_index; + + uint32 aux_data_end_global_index; + uint32 aux_data_end; + uint32 aux_heap_base_global_index; + uint32 aux_heap_base; + uint32 aux_stack_top_global_index; + uint32 aux_stack_bottom; + uint32 aux_stack_size; WASMModule *wasm_module; } AOTCompData; diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index c631259c..d813300e 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -779,9 +779,10 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_MISC_PREFIX: { - if (frame_ip < frame_ip_end) { - opcode = *frame_ip++; - } + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + opcode = (uint32)opcode1; switch (opcode) { case WASM_OP_I32_TRUNC_SAT_S_F32: diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 8b9c39bf..1d27d4f3 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -420,9 +420,8 @@ get_init_data_section_size(AOTCompData *comp_data, AOTObjectData *obj_data) size = align_uint(size, 4); size += (uint32)sizeof(uint32) * 2; - /* llvm aux data end + llvm aux stack bottom - + llvm aux stack size + llvm stack global index */ - size += sizeof(uint32) * 4; + /* aux data/heap/stack data */ + size += sizeof(uint32) * 7; size += get_object_data_section_info_size(obj_data); return size; @@ -1190,10 +1189,13 @@ aot_emit_init_data_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, EMIT_U32(comp_data->func_count); EMIT_U32(comp_data->start_func_index); - EMIT_U32(comp_data->llvm_aux_data_end); - EMIT_U32(comp_data->llvm_aux_stack_bottom); - EMIT_U32(comp_data->llvm_aux_stack_size); - EMIT_U32(comp_data->llvm_aux_stack_global_index); + EMIT_U32(comp_data->aux_data_end_global_index); + EMIT_U32(comp_data->aux_data_end); + EMIT_U32(comp_data->aux_heap_base_global_index); + EMIT_U32(comp_data->aux_heap_base); + EMIT_U32(comp_data->aux_stack_top_global_index); + EMIT_U32(comp_data->aux_stack_bottom); + EMIT_U32(comp_data->aux_stack_size); if (!aot_emit_object_data_section_info(buf, buf_end, &offset, obj_data)) return false; diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index cf80e630..77151394 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -70,29 +70,36 @@ get_memory_check_bound(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return mem_check_bound; } +static LLVMValueRef +get_memory_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); + static LLVMValueRef check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 offset, uint32 bytes) { LLVMValueRef offset_const = I32_CONST(offset); - LLVMValueRef addr, maddr, offset1, cmp, cmp1, cmp2; + LLVMValueRef addr, maddr, offset1, cmp1, cmp2, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMBasicBlockRef check_succ; AOTValue *aot_value; + bool is_target_64bit; #if WASM_ENABLE_SHARED_MEMORY != 0 bool is_shared_memory = comp_ctx->comp_data->memories[0].memory_flags & 0x02; #endif + is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) + ? true : false; + CHECK_LLVM_CONST(offset_const); /* Get memory base address and memory data size */ + if (func_ctx->mem_space_unchanged #if WASM_ENABLE_SHARED_MEMORY != 0 - if (func_ctx->mem_space_unchanged || is_shared_memory) { -#else - if (func_ctx->mem_space_unchanged) { + || is_shared_memory #endif + ) { mem_base_addr = func_ctx->mem_info[0].mem_base_addr; } else { @@ -111,15 +118,15 @@ check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* return addres directly if constant offset and inside memory space */ if (LLVMIsConstant(addr)) { - int64 mem_offset = (int64)LLVMConstIntGetSExtValue(addr) + (int64)offset; + uint64 mem_offset = (uint64)LLVMConstIntGetZExtValue(addr) + + (uint64)offset; uint32 num_bytes_per_page = comp_ctx->comp_data->memories[0].num_bytes_per_page; uint32 init_page_count = comp_ctx->comp_data->memories[0].mem_init_page_count; - int64 mem_data_size = num_bytes_per_page * init_page_count; - if (mem_data_size > 0 - && mem_offset >= 0 - && mem_offset <= mem_data_size - bytes) { + uint64 mem_data_size = num_bytes_per_page * init_page_count; + + if (mem_offset + bytes <= mem_data_size) { /* inside memory space */ offset1 = I32_CONST((uint32)mem_offset); CHECK_LLVM_CONST(offset1); @@ -132,12 +139,14 @@ check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, - I64_TYPE, "offset_i64")) - || !(addr = LLVMBuildSExt(comp_ctx->builder, addr, - I64_TYPE, "addr_i64"))) { - aot_set_last_error("llvm build extend i32 to i64 failed."); - goto fail; + if (is_target_64bit) { + if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, + I64_TYPE, "offset_i64")) + || !(addr = LLVMBuildZExt(comp_ctx->builder, addr, + I64_TYPE, "addr_i64"))) { + aot_set_last_error("llvm build zero extend failed."); + goto fail; + } } /* offset1 = offset + addr; */ @@ -147,16 +156,41 @@ check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, && !(aot_value->is_local && aot_checked_addr_list_find(func_ctx, aot_value->local_idx, offset, bytes))) { + uint32 init_page_count = + comp_ctx->comp_data->memories[0].mem_init_page_count; + if (init_page_count == 0) { + LLVMValueRef mem_size; + + if (!(mem_size = get_memory_size(comp_ctx, func_ctx))) { + goto fail; + } + BUILD_ICMP(LLVMIntEQ, mem_size, I32_ZERO, cmp, "is_zero"); + ADD_BASIC_BLOCK(check_succ, "check_mem_size_succ"); + LLVMMoveBasicBlockAfter(check_succ, block_curr); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, + true, cmp, check_succ)) { + goto fail; + } + + SET_BUILD_POS(check_succ); + block_curr = check_succ; + } + if (!(mem_check_bound = get_memory_check_bound(comp_ctx, func_ctx, bytes))) { goto fail; } - BUILD_ICMP(LLVMIntSGT, - func_ctx->mem_info[0].mem_bound_check_heap_base, - offset1, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntSGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + if (is_target_64bit) { + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); + } + else { + /* Check integer overflow */ + BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); + BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + } /* Add basic blocks */ ADD_BASIC_BLOCK(check_succ, "check_succ"); diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 753f67e2..cb024e5f 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -190,6 +190,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 memory_count; WASMModule *module = comp_ctx->comp_data->wasm_module; WASMFunction *func = module->functions[func_index]; + LLVMTypeRef bound_check_type; bool mem_space_unchanged = (!func->has_op_memory_grow && !func->has_op_func_call) || (!module->possible_memory_grow); #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -264,7 +265,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build in bounds gep failed"); return false; } - offset = I32_CONST(offsetof(AOTMemoryInstance, mem_cur_page_count)); + offset = I32_CONST(offsetof(AOTMemoryInstance, cur_page_count)); if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = LLVMBuildInBoundsGEP(comp_ctx->builder, shared_mem_addr, &offset, 1, "mem_cur_page_offset"))) { @@ -284,7 +285,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) - + offsetof(AOTMemoryInstance, mem_cur_page_count)); + + offsetof(AOTMemoryInstance, cur_page_count)); if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, &offset, 1, "mem_cur_page_offset"))) { @@ -339,6 +340,9 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } #endif + bound_check_type = (comp_ctx->pointer_size == sizeof(uint64)) + ? INT64_PTR_TYPE : INT32_PTR_TYPE; + /* Load memory bound check constants */ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte) - offsetof(AOTMemoryInstance, memory_data.ptr)); @@ -351,7 +355,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_1byte, - INT64_PTR_TYPE, "bound_check_1byte_ptr"))) { + bound_check_type, "bound_check_1byte_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } @@ -376,7 +380,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_2bytes, - INT64_PTR_TYPE, "bound_check_2bytes_ptr"))) { + bound_check_type, "bound_check_2bytes_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } @@ -401,7 +405,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_4bytes, - INT64_PTR_TYPE, "bound_check_4bytes_ptr"))) { + bound_check_type, "bound_check_4bytes_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } @@ -426,7 +430,7 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_8bytes, - INT64_PTR_TYPE, "bound_check_8bytes_ptr"))) { + bound_check_type, "bound_check_8bytes_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } @@ -440,30 +444,6 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - /* Load bound_check_heap_base */ - offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_heap_base) - - offsetof(AOTMemoryInstance, memory_data.ptr)); - if (!(func_ctx->mem_info[0].mem_bound_check_heap_base = - LLVMBuildInBoundsGEP(comp_ctx->builder, mem_info_base, - &offset, 1, "bound_check_heap_base_offset"))) { - aot_set_last_error("llvm build in bounds gep failed"); - return false; - } - if (!(func_ctx->mem_info[0].mem_bound_check_heap_base = - LLVMBuildBitCast(comp_ctx->builder, - func_ctx->mem_info[0].mem_bound_check_heap_base, - INT64_PTR_TYPE, "bound_check_heap_base_tmp"))) { - aot_set_last_error("llvm build bit cast failed"); - return false; - } - if (!(func_ctx->mem_info[0].mem_bound_check_heap_base = - LLVMBuildLoad(comp_ctx->builder, - func_ctx->mem_info[0].mem_bound_check_heap_base, - "bound_check_heap_base"))) { - aot_set_last_error("llvm build load failed"); - return false; - } - return true; } @@ -1033,6 +1013,7 @@ aot_create_comp_context(AOTCompData *comp_data, char triple_buf[32] = {0}; uint32 opt_level, size_level; LLVMCodeModel code_model; + LLVMTargetDataRef target_data_ref; /* Initialize LLVM environment */ LLVMInitializeAllTargetInfos(); @@ -1280,6 +1261,14 @@ aot_create_comp_context(AOTCompData *comp_data, } } + if (!(target_data_ref = + LLVMCreateTargetDataLayout(comp_ctx->target_machine))) { + aot_set_last_error("create LLVM target data layout failed."); + goto fail; + } + comp_ctx->pointer_size = LLVMPointerSize(target_data_ref); + LLVMDisposeTargetData(target_data_ref); + comp_ctx->optimize = true; if (option->output_format == AOT_LLVMIR_UNOPT_FILE) comp_ctx->optimize = false; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 82df590e..3658b5fd 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -102,7 +102,6 @@ typedef struct AOTCheckedAddr { typedef struct AOTMemInfo { LLVMValueRef mem_base_addr; LLVMValueRef mem_cur_page_count_addr; - LLVMValueRef mem_bound_check_heap_base; LLVMValueRef mem_bound_check_1byte; LLVMValueRef mem_bound_check_2bytes; LLVMValueRef mem_bound_check_4bytes; @@ -190,6 +189,7 @@ typedef struct AOTCompContext { LLVMTargetMachineRef target_machine; char *target_cpu; char target_arch[16]; + unsigned pointer_size; /* LLVM execution engine required by JIT */ LLVMExecutionEngineRef exec_engine; diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index fd7f03a8..9fa2eda9 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -325,15 +325,30 @@ typedef struct WASMModule { WASMDataSeg **data_segments; uint32 start_function; - /* __data_end global exported by llvm */ - uint32 llvm_aux_data_end; - /* auxiliary stack bottom, or __heap_base global exported by llvm */ - uint32 llvm_aux_stack_bottom; - /* auxiliary stack size */ - uint32 llvm_aux_stack_size; - /* the index of a global exported by llvm, which is - auxiliary stack top pointer */ - uint32 llvm_aux_stack_global_index; + /* the index of auxiliary __data_end global, + -1 means unexported */ + uint32 aux_data_end_global_index; + /* auxiliary __data_end exported by wasm app */ + uint32 aux_data_end; + + /* the index of auxiliary __heap_base global, + -1 means unexported */ + uint32 aux_heap_base_global_index; + /* auxiliary __heap_base exported by wasm app */ + uint32 aux_heap_base; + + /* the index of auxiliary stack top global, + -1 means unexported */ + uint32 aux_stack_top_global_index; + /* auxiliary stack bottom resolved */ + uint32 aux_stack_bottom; + /* auxiliary stack size resolved */ + uint32 aux_stack_size; + + /* the index of malloc/free function, + -1 means unexported */ + uint32 malloc_function; + uint32 free_function; /* Whether there is possible memory grow, e.g. memory.grow opcode */ bool possible_memory_grow; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 569f0466..ab37459f 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -227,9 +227,8 @@ LOAD_I16(void *addr) #endif /* WASM_CPU_SUPPORTS_UNALIGNED_64BIT_ACCESS != 0 */ #define CHECK_MEMORY_OVERFLOW(bytes) do { \ - int64 offset1 = (int64)(uint32)offset + (int64)(int32)addr; \ - if (heap_base_offset <= offset1 \ - && offset1 <= (int64)linear_mem_size - bytes) \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + if (offset1 + bytes <= (uint64)linear_mem_size) \ /* If offset1 is in valid range, maddr must also be in valid range, \ no need to check it again. */ \ maddr = memory->memory_data + offset1; \ @@ -238,8 +237,8 @@ LOAD_I16(void *addr) } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) do { \ - uint64 offset1 = (int32)(start); \ - if (offset1 + bytes <= linear_mem_size) \ + uint64 offset1 = (uint32)(start); \ + if (offset1 + bytes <= (uint64)linear_mem_size) \ /* App heap space is not valid space for bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ @@ -1063,7 +1062,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMInterpFrame *prev_frame) { WASMMemoryInstance *memory = module->default_memory; - int32 heap_base_offset = memory ? memory->heap_base_offset : 0; uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0; uint8 *global_data = module->global_data; uint32 linear_mem_size = memory ? num_bytes_per_page * memory->cur_page_count : 0; @@ -1579,8 +1577,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_LOAD): HANDLE_OP (WASM_OP_F32_LOAD): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1594,8 +1591,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD): HANDLE_OP (WASM_OP_F64_LOAD): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1608,8 +1604,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_LOAD8_S): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1622,8 +1617,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_LOAD8_U): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1636,8 +1630,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_LOAD16_S): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1650,8 +1643,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_LOAD16_U): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1664,8 +1656,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD8_S): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1678,8 +1669,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD8_U): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1692,8 +1682,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD16_S): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1706,8 +1695,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD16_U): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1720,8 +1708,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD32_S): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; opcode = *(frame_ip - 1); read_leb_uint32(frame_ip, frame_ip_end, flags); @@ -1735,8 +1722,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_LOAD32_U): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1751,8 +1737,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_STORE): HANDLE_OP (WASM_OP_F32_STORE): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1767,8 +1752,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_STORE): HANDLE_OP (WASM_OP_F64_STORE): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); read_leb_uint32(frame_ip, frame_ip_end, offset); @@ -1784,8 +1768,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I32_STORE8): HANDLE_OP (WASM_OP_I32_STORE16): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; uint32 sval; opcode = *(frame_ip - 1); @@ -1811,8 +1794,7 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_I64_STORE16): HANDLE_OP (WASM_OP_I64_STORE32): { - uint32 offset, flags; - int32 addr; + uint32 offset, flags, addr; uint64 sval; opcode = *(frame_ip - 1); @@ -2665,9 +2647,12 @@ label_pop_csp_n: HANDLE_OP (WASM_OP_MISC_PREFIX): { - opcode = *frame_ip++; - switch (opcode) - { + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + opcode = (uint8)opcode1; + + switch (opcode) { case WASM_OP_I32_TRUNC_SAT_S_F32: DEF_OP_TRUNC_SAT_F32(-2147483904.0f, 2147483648.0f, true, true); @@ -2787,8 +2772,7 @@ label_pop_csp_n: #if WASM_ENABLE_SHARED_MEMORY != 0 HANDLE_OP (WASM_OP_ATOMIC_PREFIX): { - uint32 offset, align; - int32 addr; + uint32 offset, align, addr; opcode = *frame_ip++; diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 166dc107..ce39dc3b 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -229,9 +229,8 @@ LOAD_I16(void *addr) #endif /* WASM_CPU_SUPPORTS_UNALIGNED_64BIT_ACCESS != 0 */ #define CHECK_MEMORY_OVERFLOW(bytes) do { \ - int64 offset1 = (int64)(uint32)offset + (int64)(int32)addr; \ - if (heap_base_offset <= offset1 \ - && offset1 <= (int64)linear_mem_size - bytes) \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + if (offset1 + bytes <= (uint64)linear_mem_size) \ /* If offset1 is in valid range, maddr must also be in valid range,\ no need to check it again. */ \ maddr = memory->memory_data + offset1; \ @@ -239,13 +238,13 @@ LOAD_I16(void *addr) goto out_of_bounds; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) do { \ - uint64 offset1 = (int32)(start); \ - if (offset1 + bytes <= linear_mem_size) \ - /* App heap space is not valid space for bulk memory operation */ \ - maddr = memory->memory_data + offset1; \ - else \ - goto out_of_bounds; \ +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) do { \ + uint64 offset1 = (uint32)(start); \ + if (offset1 + bytes <= linear_mem_size) \ + /* App heap space is not valid space for bulk memory operation */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ } while (0) #define CHECK_ATOMIC_MEMORY_ACCESS(align) do { \ @@ -1163,7 +1162,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMInterpFrame *prev_frame) { WASMMemoryInstance *memory = module->default_memory; - int32 heap_base_offset = memory ? memory->heap_base_offset : 0; uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0; uint8 *global_data = module->global_data; uint32 linear_mem_size = memory ? num_bytes_per_page * memory->cur_page_count : 0; @@ -2768,8 +2766,7 @@ recover_br_info: #if WASM_ENABLE_SHARED_MEMORY != 0 HANDLE_OP (WASM_OP_ATOMIC_PREFIX): { - uint32 offset; - int32 addr; + uint32 offset, addr; GET_OPCODE(); diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 175f4d03..f9f8d101 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -52,14 +52,14 @@ check_buf1(const uint8 *buf, const uint8 *buf_end, uint32 length, #define CHECK_BUF(buf, buf_end, length) do { \ if (!check_buf(buf, buf_end, length, \ error_buf, error_buf_size)) { \ - return false; \ + goto fail; \ } \ } while (0) #define CHECK_BUF1(buf, buf_end, length) do { \ if (!check_buf1(buf, buf_end, length, \ error_buf, error_buf_size)) { \ - return false; \ + goto fail; \ } \ } while (0) @@ -90,6 +90,8 @@ skip_leb(const uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, *p_buf += offset; return true; +fail: + return false; } #define skip_leb_int64(p, p_end) do { \ @@ -184,6 +186,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, fail_integer_too_large: set_error_buf(error_buf, error_buf_size, "WASM module load failed: integer too large"); +fail: return false; } @@ -195,7 +198,7 @@ fail_integer_too_large: uint64 res64; \ if (!read_leb((uint8**)&p, p_end, 64, true, &res64,\ error_buf, error_buf_size)) \ - return false; \ + goto fail; \ res = (int64)res64; \ } while (0) @@ -203,7 +206,7 @@ fail_integer_too_large: uint64 res64; \ if (!read_leb((uint8**)&p, p_end, 32, false, &res64,\ error_buf, error_buf_size)) \ - return false; \ + goto fail; \ res = (uint32)res64; \ } while (0) @@ -211,7 +214,7 @@ fail_integer_too_large: uint64 res64; \ if (!read_leb((uint8**)&p, p_end, 32, true, &res64,\ error_buf, error_buf_size)) \ - return false; \ + goto fail; \ res = (int32)res64; \ } while (0) @@ -335,19 +338,19 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, /* i32.const */ case INIT_EXPR_TYPE_I32_CONST: if (type != VALUE_TYPE_I32) - goto fail; + goto fail_type_mismatch; read_leb_int32(p, p_end, init_expr->u.i32); break; /* i64.const */ case INIT_EXPR_TYPE_I64_CONST: if (type != VALUE_TYPE_I64) - goto fail; + goto fail_type_mismatch; read_leb_int64(p, p_end, init_expr->u.i64); break; /* f32.const */ case INIT_EXPR_TYPE_F32_CONST: if (type != VALUE_TYPE_F32) - goto fail; + goto fail_type_mismatch; CHECK_BUF(p, p_end, 4); p_float = (uint8*)&init_expr->u.f32; for (i = 0; i < sizeof(float32); i++) @@ -356,7 +359,7 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, /* f64.const */ case INIT_EXPR_TYPE_F64_CONST: if (type != VALUE_TYPE_F64) - goto fail; + goto fail_type_mismatch; CHECK_BUF(p, p_end, 8); p_float = (uint8*)&init_expr->u.f64; for (i = 0; i < sizeof(float64); i++) @@ -367,19 +370,20 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, read_leb_uint32(p, p_end, init_expr->u.global_index); break; default: - goto fail; + goto fail_type_mismatch; } CHECK_BUF(p, p_end, 1); end_byte = read_uint8(p); if (end_byte != 0x0b) - goto fail; + goto fail_type_mismatch; *p_buf = p; return true; -fail: +fail_type_mismatch: set_error_buf(error_buf, error_buf_size, "WASM module load failed: type mismatch or " "constant expression required."); +fail: return false; } @@ -472,6 +476,8 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load type section success.\n"); return true; +fail: + return false; } #if WASM_ENABLE_MULTI_MODULE != 0 @@ -804,6 +810,8 @@ load_function_import(const WASMModule *parent_module, WASMModule *sub_module, function->import_func_linked = is_built_in_module ? NULL : linked_func; #endif return true; +fail: + return false; } static bool @@ -905,6 +913,8 @@ load_table_import(WASMModule *sub_module, const char *sub_module_name, table->flags = declare_max_size_flag; table->max_size = declare_max_size; return true; +fail: + return false; } unsigned @@ -1034,6 +1044,8 @@ load_memory_import(WASMModule *sub_module, const char *sub_module_name, *p_buf = p; return true; +fail: + return false; } static bool @@ -1107,13 +1119,15 @@ load_global_import(const WASMModule *parent_module, global->type = declare_type; global->is_mutable = is_mutable; return true; +fail: + return false; } static bool load_table(const uint8 **p_buf, const uint8 *buf_end, WASMTable *table, char *error_buf, uint32 error_buf_size) { - const uint8 *p = *p_buf, *p_end = buf_end; + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; CHECK_BUF(p, p_end, 1); /* 0x70 */ @@ -1123,32 +1137,40 @@ load_table(const uint8 **p_buf, const uint8 *buf_end, WASMTable *table, return false; } + p_org = p; read_leb_uint32(p, p_end, table->flags); + if (p - p_org > 1) { + set_error_buf(error_buf, error_buf_size, + "integer representation too long"); + return false; + } + if (table->flags > 1) { + set_error_buf(error_buf, error_buf_size, "integer too large"); + return false; + } + read_leb_uint32(p, p_end, table->init_size); - if (table->flags & 1) { + if (table->flags == 0) { + table->max_size = 0x10000; + } + else if (table->flags == 1) { read_leb_uint32(p, p_end, table->max_size); if (!check_table_max_size(table->init_size, table->max_size, error_buf, error_buf_size)) return false; } - else - table->max_size = 0x10000; - - if ((table->flags & 1) && table->init_size > table->max_size) { - set_error_buf(error_buf, error_buf_size, - "size minimum must not be greater than maximum"); - return false; - } *p_buf = p; return true; +fail: + return false; } static bool load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, char *error_buf, uint32 error_buf_size) { - const uint8 *p = *p_buf, *p_end = buf_end; + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; uint32 pool_size = wasm_runtime_memory_pool_size(); #if WASM_ENABLE_APP_FRAMEWORK != 0 uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT @@ -1157,7 +1179,25 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE; #endif + p_org = p; read_leb_uint32(p, p_end, memory->flags); + if (p - p_org > 1) { + set_error_buf(error_buf, error_buf_size, + "integer representation too long"); + return false; + } +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (memory->flags > 1) { + set_error_buf(error_buf, error_buf_size, "integer too large"); + return false; + } +#else + if (memory->flags > 3 || memory->flags == 2) { + set_error_buf(error_buf, error_buf_size, "integer too large"); + return false; + } +#endif + read_leb_uint32(p, p_end, memory->init_page_count); if (!check_memory_init_size(memory->init_page_count, error_buf, error_buf_size)) @@ -1172,14 +1212,17 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, if (memory->max_page_count > max_page_count) memory->max_page_count = max_page_count; } - else + else { /* Limit the maximum memory size to max_page_count */ memory->max_page_count = max_page_count; + } memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; *p_buf = p; return true; +fail: + return false; } #if WASM_ENABLE_MULTI_MODULE != 0 @@ -1434,7 +1477,7 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, default: set_error_buf(error_buf, error_buf_size, - "Load import section failed: invalid import type."); + "Load import section failed: invalid import kind"); return false; } } @@ -1454,7 +1497,7 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, p = p_old; - // TODO: move it out of the loop + /* TODO: move it out of the loop */ /* insert "env", "wasi_unstable" and "wasi_snapshot_preview1" to const str list */ if (!const_str_list_insert((uint8*)"env", 3, module, error_buf, error_buf_size) || !const_str_list_insert((uint8*)"wasi_unstable", 13, module, @@ -1566,7 +1609,7 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, default: set_error_buf(error_buf, error_buf_size, "Load import section failed: " - "invalid import type."); + "invalid import kind"); return false; } import->kind = kind; @@ -1598,6 +1641,8 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, (void)u32; (void)type_index; return true; +fail: + return false; } static bool @@ -1779,6 +1824,8 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, LOG_VERBOSE("Load function section success.\n"); return true; +fail: + return false; } static bool @@ -1820,6 +1867,8 @@ load_table_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load table section success.\n"); return true; +fail: + return false; } static bool @@ -1861,6 +1910,8 @@ load_memory_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load memory section success.\n"); return true; +fail: + return false; } static bool @@ -1926,6 +1977,8 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load global section success.\n"); return true; +fail: + return false; } static bool @@ -2029,6 +2082,8 @@ load_export_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load export section success.\n"); return true; +fail: + return false; } static bool @@ -2102,6 +2157,8 @@ load_table_segment_section(const uint8 *buf, const uint8 *buf_end, WASMModule *m LOG_VERBOSE("Load table segment section success.\n"); return true; +fail: + return false; } static bool @@ -2217,6 +2274,8 @@ check_mem_index: LOG_VERBOSE("Load data segment section success.\n"); return true; +fail: + return false; } #if WASM_ENABLE_BULK_MEMORY != 0 @@ -2238,6 +2297,8 @@ load_datacount_section(const uint8 *buf, const uint8 *buf_end, WASMModule *modul LOG_VERBOSE("Load datacount section success.\n"); return true; +fail: + return false; } #endif @@ -2268,6 +2329,8 @@ load_code_section(const uint8 *buf, const uint8 *buf_end, LOG_VERBOSE("Load code segment section success.\n"); return true; +fail: + return false; } static bool @@ -2311,6 +2374,8 @@ load_start_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load start section success.\n"); return true; +fail: + return false; } static bool @@ -2344,6 +2409,8 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, LOG_VERBOSE("Load custom section success.\n"); return true; +fail: + return false; } @@ -2367,12 +2434,14 @@ load_from_sections(WASMModule *module, WASMSection *sections, WASMSection *section = sections; const uint8 *buf, *buf_end, *buf_code = NULL, *buf_code_end = NULL, *buf_func = NULL, *buf_func_end = NULL; - WASMGlobal *llvm_data_end_global = NULL, *llvm_heap_base_global = NULL; - WASMGlobal *llvm_stack_top_global = NULL, *global; - uint32 llvm_data_end = UINT32_MAX, llvm_heap_base = UINT32_MAX; - uint32 llvm_stack_top = UINT32_MAX, global_index, i; - uint32 stack_top_global_index = UINT32_MAX; + WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; + WASMGlobal *aux_stack_top_global = NULL, *global; + uint32 aux_data_end = (uint32)-1, aux_heap_base = (uint32)-1; + uint32 aux_stack_top = (uint32)-1, global_index, func_index, i; + uint32 aux_data_end_global_index = (uint32)-1; + uint32 aux_heap_base_global_index = (uint32)-1; BlockAddr *block_addr_cache; + WASMType *func_type; uint64 total_size; /* Find code and function sections if have */ @@ -2481,7 +2550,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, } wasm_runtime_free(block_addr_cache); - /* Resolve llvm auxiliary data/stack/heap info and reset memory info */ + module->aux_data_end_global_index = (uint32)-1; + module->aux_heap_base_global_index = (uint32)-1; + module->aux_stack_top_global_index = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ export = module->exports; for (i = 0; i < module->export_count; i++, export++) { if (export->kind == EXPORT_KIND_GLOBAL) { @@ -2492,10 +2565,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, && !global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { - llvm_heap_base_global = global; - llvm_heap_base = global->init_expr.u.i32; - LOG_VERBOSE("found llvm __heap_base global, value: %d\n", - llvm_heap_base); + aux_heap_base_global = global; + aux_heap_base = global->init_expr.u.i32; + aux_heap_base_global_index = export->index; + LOG_VERBOSE("Found aux __heap_base global, value: %d", + aux_heap_base); } } else if (!strcmp(export->name, "__data_end")) { @@ -2505,12 +2579,13 @@ load_from_sections(WASMModule *module, WASMSection *sections, && !global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { - llvm_data_end_global = global; - llvm_data_end = global->init_expr.u.i32; - LOG_VERBOSE("found llvm __data_end global, value: %d\n", - llvm_data_end); + aux_data_end_global = global; + aux_data_end = global->init_expr.u.i32; + aux_data_end_global_index = export->index; + LOG_VERBOSE("Found aux __data_end global, value: %d", + aux_data_end); - llvm_data_end = align_uint(llvm_data_end, 16); + aux_data_end = align_uint(aux_data_end, 16); } } @@ -2530,64 +2605,97 @@ load_from_sections(WASMModule *module, WASMSection *sections, [3] heap_base <-- 3 [4] dso_handle */ - if (llvm_data_end_global && llvm_heap_base_global) { + if (aux_data_end_global && aux_heap_base_global + && aux_data_end <= aux_heap_base) { + module->aux_data_end_global_index = aux_data_end_global_index; + module->aux_data_end = aux_data_end; + module->aux_heap_base_global_index = aux_heap_base_global_index; + module->aux_heap_base = aux_heap_base; + /* Resolve aux stack top global */ - for (global_index = 0; global_index < module->global_count; global_index++) { + for (global_index = 0; global_index < module->global_count; + global_index++) { global = module->globals + global_index; - if (global != llvm_data_end_global - && global != llvm_heap_base_global + if (global->is_mutable /* heap_base and data_end is + not mutable */ && global->type == VALUE_TYPE_I32 - && global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST - && (global->init_expr.u.i32 <= - llvm_heap_base_global->init_expr.u.i32 - && llvm_data_end_global->init_expr.u.i32 <= - llvm_heap_base_global->init_expr.u.i32)) { - llvm_stack_top_global = global; - llvm_stack_top = global->init_expr.u.i32; - stack_top_global_index = global_index; - LOG_VERBOSE("found llvm stack top global, " - "value: %d, global index: %d\n", - llvm_stack_top, global_index); + && (uint32)global->init_expr.u.i32 <= aux_heap_base) { + aux_stack_top_global = global; + aux_stack_top = (uint32)global->init_expr.u.i32; + module->aux_stack_top_global_index = + module->import_global_count + global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = aux_stack_top > aux_data_end + ? aux_stack_top - aux_data_end + : aux_stack_top; + LOG_VERBOSE("Found aux stack top global, value: %d, " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); break; } } - - module->llvm_aux_data_end = llvm_data_end; - module->llvm_aux_stack_bottom = llvm_stack_top; - module->llvm_aux_stack_size = llvm_stack_top > llvm_data_end - ? llvm_stack_top - llvm_data_end - : llvm_stack_top; - module->llvm_aux_stack_global_index = stack_top_global_index; - LOG_VERBOSE("aux stack bottom: %d, size: %d\n", - module->llvm_aux_stack_bottom, - module->llvm_aux_stack_size); break; } } } + module->malloc_function = (uint32)-1; + module->free_function = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ + export = module->exports; + for (i = 0; i < module->export_count; i++, export++) { + if (export->kind == EXPORT_KIND_FUNC) { + if (!strcmp(export->name, "malloc") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == VALUE_TYPE_I32 + && func_type->types[1] == VALUE_TYPE_I32) { + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, index: %u", + export->index); + } + } + else if (!strcmp(export->name, "free") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 0 + && func_type->types[0] == VALUE_TYPE_I32) { + module->free_function = export->index; + LOG_VERBOSE("Found free function, index: %u", + export->index); + } + } + } + } + if (!module->possible_memory_grow) { - if (llvm_data_end_global - && llvm_heap_base_global - && llvm_stack_top_global - && llvm_stack_top <= llvm_heap_base) { - WASMMemoryImport *memory_import; - WASMMemory *memory; + WASMMemoryImport *memory_import; + WASMMemory *memory; + + if (aux_data_end_global + && aux_heap_base_global + && aux_stack_top_global) { uint64 init_memory_size; - uint32 shrunk_memory_size = llvm_heap_base > llvm_data_end - ? llvm_heap_base : llvm_data_end; + uint32 shrunk_memory_size = align_uint(aux_heap_base, 8); + if (module->import_memory_count) { memory_import = &module->import_memories[0].u.memory; init_memory_size = (uint64)memory_import->num_bytes_per_page * memory_import->init_page_count; - if (llvm_heap_base <= init_memory_size - && llvm_data_end <= init_memory_size) { + if (shrunk_memory_size <= init_memory_size) { /* Reset memory info to decrease memory usage */ memory_import->num_bytes_per_page = shrunk_memory_size; memory_import->init_page_count = 1; - LOG_VERBOSE("reset import memory size to %d\n", + LOG_VERBOSE("Shrink import memory size to %d", shrunk_memory_size); } } @@ -2595,15 +2703,31 @@ load_from_sections(WASMModule *module, WASMSection *sections, memory = &module->memories[0]; init_memory_size = (uint64)memory->num_bytes_per_page * memory->init_page_count; - if (llvm_heap_base <= init_memory_size - && llvm_data_end <= init_memory_size) { + if (shrunk_memory_size <= init_memory_size) { /* Reset memory info to decrease memory usage */ memory->num_bytes_per_page = shrunk_memory_size; memory->init_page_count = 1; - LOG_VERBOSE("reset memory size to %d\n", shrunk_memory_size); + LOG_VERBOSE("Shrink memory size to %d", shrunk_memory_size); } } } + +#if WASM_ENABLE_MULTI_MODULE == 0 + if (module->import_memory_count) { + memory_import = &module->import_memories[0].u.memory; + /* Memory init page count cannot be larger than 65536, we don't + check integer overflow again. */ + memory_import->num_bytes_per_page *= memory_import->init_page_count; + memory_import->init_page_count = memory_import->max_page_count = 1; + } + if (module->memory_count) { + /* Memory init page count cannot be larger than 65536, we don't + check integer overflow again. */ + memory = &module->memories[0]; + memory->num_bytes_per_page *= memory->init_page_count; + memory->init_page_count = memory->max_page_count = 1; + } +#endif } return true; @@ -2759,6 +2883,8 @@ create_sections(const uint8 *buf, uint32 size, } return true; +fail: + return false; } static void @@ -2819,6 +2945,8 @@ load(const uint8 *buf, uint32 size, WASMModule *module, destroy_sections(section_list); return true; +fail: + return false; } WASMModule* @@ -2834,7 +2962,7 @@ wasm_loader_load(const uint8 *buf, uint32 size, char *error_buf, uint32 error_bu goto fail; } - LOG_VERBOSE("Load module success"); + LOG_VERBOSE("Load module success.\n"); return module; fail: @@ -3275,8 +3403,11 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, break; case WASM_OP_MISC_PREFIX: { - opcode = read_uint8(p); - switch (opcode) { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + + switch (opcode1) { case WASM_OP_I32_TRUNC_SAT_S_F32: case WASM_OP_I32_TRUNC_SAT_U_F32: case WASM_OP_I32_TRUNC_SAT_S_F64: @@ -3340,6 +3471,8 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, (void)u8; return false; +fail: + return false; } #define REF_I32 VALUE_TYPE_I32 @@ -6032,8 +6165,8 @@ handle_op_block_and_loop: || global_type == VALUE_TYPE_F64) { *p_org = WASM_OP_SET_GLOBAL_64; } - else if (module->llvm_aux_stack_size > 0 - && global_idx == module->llvm_aux_stack_global_index) { + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { *p_org = WASM_OP_SET_GLOBAL_AUX_STACK; } #endif @@ -6043,8 +6176,8 @@ handle_op_block_and_loop: skip_label(); emit_label(WASM_OP_SET_GLOBAL_64); } - else if (module->llvm_aux_stack_size > 0 - && global_idx == module->llvm_aux_stack_global_index) { + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { skip_label(); emit_label(WASM_OP_SET_GLOBAL_AUX_STACK); } @@ -6462,11 +6595,13 @@ handle_op_block_and_loop: case WASM_OP_MISC_PREFIX: { - opcode = read_uint8(p); + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); #if WASM_ENABLE_FAST_INTERP != 0 - emit_byte(loader_ctx, opcode); + emit_byte(loader_ctx, ((uint8)opcode1)); #endif - switch (opcode) + switch (opcode1) { case WASM_OP_I32_TRUNC_SAT_S_F32: case WASM_OP_I32_TRUNC_SAT_U_F32: @@ -6574,7 +6709,7 @@ fail_data_cnt_sec_require: if (error_buf != NULL) snprintf(error_buf, error_buf_size, "WASM module load failed: " - "invalid opcode 0xfc %02x.", opcode); + "invalid opcode 0xfc %02x.", opcode1); goto fail; break; } diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 483ee126..a56c2a38 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -555,24 +555,27 @@ static bool load_table(const uint8 **p_buf, const uint8 *buf_end, WASMTable *table, char *error_buf, uint32 error_buf_size) { - const uint8 *p = *p_buf, *p_end = buf_end; + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; CHECK_BUF(p, p_end, 1); /* 0x70 */ table->elem_type = read_uint8(p); bh_assert(TABLE_ELEM_TYPE_ANY_FUNC == table->elem_type); + p_org = p; read_leb_uint32(p, p_end, table->flags); + bh_assert(p - p_org <= 1); + bh_assert(table->flags <= 1); + (void)p_org; + read_leb_uint32(p, p_end, table->init_size); - if (table->flags & 1) { + if (table->flags == 0) { + table->max_size = 0x10000; + } + else if (table->flags == 1) { read_leb_uint32(p, p_end, table->max_size); bh_assert(table->init_size <= table->max_size); } - else - table->max_size = 0x10000; - - bh_assert(!((table->flags & 1) - && table->init_size > table->max_size)); *p_buf = p; return true; @@ -582,7 +585,7 @@ static bool load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, char *error_buf, uint32 error_buf_size) { - const uint8 *p = *p_buf, *p_end = buf_end; + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; uint32 pool_size = wasm_runtime_memory_pool_size(); #if WASM_ENABLE_APP_FRAMEWORK != 0 uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT @@ -591,7 +594,15 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE; #endif + p_org = p; read_leb_uint32(p, p_end, memory->flags); + bh_assert(p - p_org <= 1); +#if WASM_ENABLE_SHARED_MEMORY == 0 + bh_assert(memory->flags <= 1); +#else + bh_assert(memory->flags <= 3 && memory->flags != 2); +#endif + read_leb_uint32(p, p_end, memory->init_page_count); bh_assert(memory->init_page_count <= 65536); @@ -602,9 +613,10 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, if (memory->max_page_count > max_page_count) memory->max_page_count = max_page_count; } - else + else { /* Limit the maximum memory size to max_page_count */ memory->max_page_count = max_page_count; + } memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; @@ -707,7 +719,7 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, p = p_old; - // TODO: move it out of the loop + /* TODO: move it out of the loop */ /* insert "env", "wasi_unstable" and "wasi_snapshot_preview1" to const str list */ if (!const_str_list_insert((uint8*)"env", 3, module, error_buf, error_buf_size) || !const_str_list_insert((uint8*)"wasi_unstable", 13, module, @@ -1436,12 +1448,14 @@ load_from_sections(WASMModule *module, WASMSection *sections, WASMSection *section = sections; const uint8 *buf, *buf_end, *buf_code = NULL, *buf_code_end = NULL, *buf_func = NULL, *buf_func_end = NULL; - WASMGlobal *llvm_data_end_global = NULL, *llvm_heap_base_global = NULL; - WASMGlobal *llvm_stack_top_global = NULL, *global; - uint32 llvm_data_end = UINT32_MAX, llvm_heap_base = UINT32_MAX; - uint32 llvm_stack_top = UINT32_MAX, global_index, i; - uint32 stack_top_global_index = UINT32_MAX; + WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; + WASMGlobal *aux_stack_top_global = NULL, *global; + uint32 aux_data_end = (uint32)-1, aux_heap_base = (uint32)-1; + uint32 aux_stack_top = (uint32)-1, global_index, func_index, i; + uint32 aux_data_end_global_index = (uint32)-1; + uint32 aux_heap_base_global_index = (uint32)-1; BlockAddr *block_addr_cache; + WASMType *func_type; uint64 total_size; /* Find code and function sections if have */ @@ -1550,7 +1564,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, } wasm_runtime_free(block_addr_cache); - /* Resolve llvm auxiliary data/stack/heap info and reset memory info */ + module->aux_data_end_global_index = (uint32)-1; + module->aux_heap_base_global_index = (uint32)-1; + module->aux_stack_top_global_index = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ export = module->exports; for (i = 0; i < module->export_count; i++, export++) { if (export->kind == EXPORT_KIND_GLOBAL) { @@ -1561,10 +1579,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, && !global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { - llvm_heap_base_global = global; - llvm_heap_base = global->init_expr.u.i32; - LOG_VERBOSE("found llvm __heap_base global, value: %d\n", - llvm_heap_base); + aux_heap_base_global = global; + aux_heap_base = global->init_expr.u.i32; + aux_heap_base_global_index = export->index; + LOG_VERBOSE("Found aux __heap_base global, value: %d", + aux_heap_base); } } else if (!strcmp(export->name, "__data_end")) { @@ -1574,89 +1593,155 @@ load_from_sections(WASMModule *module, WASMSection *sections, && !global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { - llvm_data_end_global = global; - llvm_data_end = global->init_expr.u.i32; - LOG_VERBOSE("found llvm __data_end global, value: %d\n", - llvm_data_end); + aux_data_end_global = global; + aux_data_end = global->init_expr.u.i32; + aux_data_end_global_index = export->index; + LOG_VERBOSE("Found aux __data_end global, value: %d", + aux_data_end); - llvm_data_end = align_uint(llvm_data_end, 16); + aux_data_end = align_uint(aux_data_end, 16); } } - if (llvm_data_end_global && llvm_heap_base_global) { + /* For module compiled with -pthread option, the global is: + [0] stack_top <-- 0 + [1] tls_pointer + [2] tls_size + [3] data_end <-- 3 + [4] global_base + [5] heap_base <-- 5 + [6] dso_handle + + For module compiled without -pthread option: + [0] stack_top <-- 0 + [1] data_end <-- 1 + [2] global_base + [3] heap_base <-- 3 + [4] dso_handle + */ + if (aux_data_end_global && aux_heap_base_global + && aux_data_end <= aux_heap_base) { + module->aux_data_end_global_index = aux_data_end_global_index; + module->aux_data_end = aux_data_end; + module->aux_heap_base_global_index = aux_heap_base_global_index; + module->aux_heap_base = aux_heap_base; + /* Resolve aux stack top global */ - for (global_index = 0; global_index < module->global_count; global_index++) { + for (global_index = 0; global_index < module->global_count; + global_index++) { global = module->globals + global_index; - if (global != llvm_data_end_global - && global != llvm_heap_base_global + if (global->is_mutable /* heap_base and data_end is + not mutable */ && global->type == VALUE_TYPE_I32 - && global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST - && (global->init_expr.u.i32 <= - llvm_heap_base_global->init_expr.u.i32 - && llvm_data_end_global->init_expr.u.i32 <= - llvm_heap_base_global->init_expr.u.i32)) { - llvm_stack_top_global = global; - llvm_stack_top = global->init_expr.u.i32; - stack_top_global_index = global_index; - LOG_VERBOSE("found llvm stack top global, " - "value: %d, global index: %d\n", - llvm_stack_top, global_index); + && (uint32)global->init_expr.u.i32 <= aux_heap_base) { + aux_stack_top_global = global; + aux_stack_top = (uint32)global->init_expr.u.i32; + module->aux_stack_top_global_index = + module->import_global_count + global_index; + module->aux_stack_bottom = aux_stack_top; + module->aux_stack_size = aux_stack_top > aux_data_end + ? aux_stack_top - aux_data_end + : aux_stack_top; + LOG_VERBOSE("Found aux stack top global, value: %d, " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); break; } } - - module->llvm_aux_data_end = llvm_data_end; - module->llvm_aux_stack_bottom = llvm_stack_top; - module->llvm_aux_stack_size = llvm_stack_top > llvm_data_end - ? llvm_stack_top - llvm_data_end - : llvm_stack_top; - module->llvm_aux_stack_global_index = stack_top_global_index; - LOG_VERBOSE("aux stack bottom: %d, size: %d\n", - module->llvm_aux_stack_bottom, - module->llvm_aux_stack_size); break; } } } + module->malloc_function = (uint32)-1; + module->free_function = (uint32)-1; + + /* Resolve auxiliary data/stack/heap info and reset memory info */ + export = module->exports; + for (i = 0; i < module->export_count; i++, export++) { + if (export->kind == EXPORT_KIND_FUNC) { + if (!strcmp(export->name, "malloc") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 1 + && func_type->types[0] == VALUE_TYPE_I32 + && func_type->types[1] == VALUE_TYPE_I32) { + module->malloc_function = export->index; + LOG_VERBOSE("Found malloc function, index: %u", + export->index); + } + } + else if (!strcmp(export->name, "free") + && export->index >= module->import_function_count) { + func_index = export->index - module->import_function_count; + func_type = module->functions[func_index]->func_type; + if (func_type->param_count == 1 + && func_type->result_count == 0 + && func_type->types[0] == VALUE_TYPE_I32) { + module->free_function = export->index; + LOG_VERBOSE("Found free function, index: %u", + export->index); + } + } + } + } + if (!module->possible_memory_grow) { - if (llvm_data_end_global - && llvm_heap_base_global - && llvm_stack_top_global - && llvm_stack_top <= llvm_heap_base) { - WASMMemoryImport *memory_import; - WASMMemory *memory; + WASMMemoryImport *memory_import; + WASMMemory *memory; + + if (aux_data_end_global + && aux_heap_base_global + && aux_stack_top_global) { uint64 init_memory_size; - uint32 shrunk_memory_size = llvm_heap_base > llvm_data_end - ? llvm_heap_base : llvm_data_end; + uint32 shrunk_memory_size = align_uint(aux_heap_base, 8); + if (module->import_memory_count) { memory_import = &module->import_memories[0].u.memory; init_memory_size = (uint64)memory_import->num_bytes_per_page * memory_import->init_page_count; - if (llvm_heap_base <= init_memory_size - && llvm_data_end <= init_memory_size) { + if (shrunk_memory_size <= init_memory_size) { /* Reset memory info to decrease memory usage */ memory_import->num_bytes_per_page = shrunk_memory_size; memory_import->init_page_count = 1; - LOG_VERBOSE("reset import memory size to %d\n", + LOG_VERBOSE("Shrink import memory size to %d", shrunk_memory_size); } } if (module->memory_count) { memory = &module->memories[0]; init_memory_size = (uint64)memory->num_bytes_per_page * - memory->init_page_count; - if (llvm_heap_base <= init_memory_size - && llvm_data_end <= init_memory_size) { + memory->init_page_count; + if (shrunk_memory_size <= init_memory_size) { /* Reset memory info to decrease memory usage */ memory->num_bytes_per_page = shrunk_memory_size; memory->init_page_count = 1; - LOG_VERBOSE("reset memory size to %d\n", shrunk_memory_size); + LOG_VERBOSE("Shrink memory size to %d", shrunk_memory_size); } } } + +#if WASM_ENABLE_MULTI_MODULE == 0 + if (module->import_memory_count) { + memory_import = &module->import_memories[0].u.memory; + /* Memory init page count cannot be larger than 65536, we don't + check integer overflow again. */ + memory_import->num_bytes_per_page *= memory_import->init_page_count; + memory_import->init_page_count = memory_import->max_page_count = 1; + } + if (module->memory_count) { + /* Memory init page count cannot be larger than 65536, we don't + check integer overflow again. */ + memory = &module->memories[0]; + memory->num_bytes_per_page *= memory->init_page_count; + memory->init_page_count = memory->max_page_count = 1; + } +#endif } return true; @@ -1878,7 +1963,7 @@ wasm_loader_load(const uint8 *buf, uint32 size, char *error_buf, uint32 error_bu goto fail; } - LOG_VERBOSE("Load module success"); + LOG_VERBOSE("Load module success.\n"); return module; fail: @@ -2295,8 +2380,11 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, break; case WASM_OP_MISC_PREFIX: { - opcode = read_uint8(p); - switch (opcode) { + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); + + switch (opcode1) { case WASM_OP_I32_TRUNC_SAT_S_F32: case WASM_OP_I32_TRUNC_SAT_U_F32: case WASM_OP_I32_TRUNC_SAT_S_F64: @@ -4869,8 +4957,8 @@ handle_op_block_and_loop: || global_type == VALUE_TYPE_F64) { *p_org = WASM_OP_SET_GLOBAL_64; } - else if (module->llvm_aux_stack_size > 0 - && global_idx == module->llvm_aux_stack_global_index) { + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { *p_org = WASM_OP_SET_GLOBAL_AUX_STACK; } #endif @@ -4880,8 +4968,8 @@ handle_op_block_and_loop: skip_label(); emit_label(WASM_OP_SET_GLOBAL_64); } - else if (module->llvm_aux_stack_size > 0 - && global_idx == module->llvm_aux_stack_global_index) { + else if (module->aux_stack_size > 0 + && global_idx == module->aux_stack_top_global_index) { skip_label(); emit_label(WASM_OP_SET_GLOBAL_AUX_STACK); } @@ -5289,12 +5377,13 @@ handle_op_block_and_loop: case WASM_OP_MISC_PREFIX: { - opcode = read_uint8(p); + uint32 opcode1; + + read_leb_uint32(p, p_end, opcode1); #if WASM_ENABLE_FAST_INTERP != 0 - emit_byte(loader_ctx, opcode); + emit_byte(loader_ctx, ((uint8)opcode1)); #endif - switch (opcode) - { + switch (opcode1) { case WASM_OP_I32_TRUNC_SAT_S_F32: case WASM_OP_I32_TRUNC_SAT_U_F32: POP_AND_PUSH(VALUE_TYPE_F32, VALUE_TYPE_I32); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 42829c7a..9e1c427c 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -122,11 +122,13 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 heap_size, uint32 flags, char *error_buf, uint32 error_buf_size) { + WASMModule *module = module_inst->module; WASMMemoryInstance *memory; - uint64 heap_and_inst_size = offsetof(WASMMemoryInstance, base_addr) + - (uint64)heap_size; - uint64 total_size = heap_and_inst_size + - num_bytes_per_page * (uint64)init_page_count; + uint64 total_size, memory_data_size; + uint32 heap_offset = num_bytes_per_page * init_page_count; + uint32 inc_page_count, aux_heap_base, global_idx; + uint32 bytes_of_last_page, bytes_to_page_end; + uint8 *global_addr; #if WASM_ENABLE_SHARED_MEMORY != 0 bool is_shared_memory = flags & 0x02 ? true : false; @@ -149,12 +151,96 @@ memory_instantiate(WASMModuleInstance *module_inst, (void)ref_count; return memory; } + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + + if (heap_size > 0 + && module_inst->module->malloc_function != (uint32)-1 + && module_inst->module->free_function != (uint32)-1) { + /* Disable app heap, use malloc/free function exported + by wasm app to allocate/free memory instead */ + heap_size = 0; + } + + if (init_page_count == max_page_count && init_page_count == 1) { + /* If only one page and at most one page, we just append + the app heap to the end of linear memory, enlarge the + num_bytes_per_page, and don't change the page count*/ + heap_offset = num_bytes_per_page; + num_bytes_per_page += heap_size; + if (num_bytes_per_page < heap_size) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return NULL; + } + } + else if (heap_size > 0) { + if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base < num_bytes_per_page + * init_page_count) { + /* Insert app heap before __heap_base */ + aux_heap_base = module->aux_heap_base; + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + inc_page_count = (heap_size - bytes_to_page_end + + num_bytes_per_page - 1) / num_bytes_per_page; + heap_offset = aux_heap_base; + aux_heap_base += heap_size; + + bytes_of_last_page = aux_heap_base % num_bytes_per_page; + if (bytes_of_last_page == 0) + bytes_of_last_page = num_bytes_per_page; + bytes_to_page_end = num_bytes_per_page - bytes_of_last_page; + if (bytes_to_page_end < 1 * BH_KB) { + aux_heap_base += 1 * BH_KB; + inc_page_count++; + } + + /* Adjust __heap_base global value */ + global_idx = module->aux_heap_base_global_index; + global_addr = module_inst->global_data + + module_inst->globals[global_idx].data_offset; + *(uint32 *)global_addr = aux_heap_base; + LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); + } + else { + /* Insert app heap before new page */ + inc_page_count = (heap_size + num_bytes_per_page - 1) + / num_bytes_per_page; + heap_offset = num_bytes_per_page * init_page_count; + heap_size = num_bytes_per_page * inc_page_count; + if (heap_size > 0) + heap_size -= 1 * BH_KB; + } + init_page_count += inc_page_count; + max_page_count += inc_page_count; + if (init_page_count > 65536) { + set_error_buf(error_buf, error_buf_size, + "memory size must be at most 65536 pages (4GiB)"); + return NULL; + } + if (max_page_count > 65536) + max_page_count = 65536; + } + + LOG_VERBOSE("Memory instantiate:"); + LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", + num_bytes_per_page, init_page_count, max_page_count); + LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); + + memory_data_size = (uint64)num_bytes_per_page * init_page_count; +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { /* Allocate max page for shared memory */ - total_size = heap_and_inst_size + - num_bytes_per_page * (uint64)max_page_count; + memory_data_size = (uint64)num_bytes_per_page * max_page_count; } #endif + total_size = offsetof(WASMMemoryInstance, memory_data) + + memory_data_size; + /* Allocate memory space, addr data and global data */ if (!(memory = runtime_malloc(total_size, error_buf, error_buf_size))) { @@ -166,37 +252,28 @@ memory_instantiate(WASMModuleInstance *module_inst, memory->cur_page_count = init_page_count; memory->max_page_count = max_page_count; - memory->heap_data = memory->base_addr; - memory->memory_data = memory->heap_data + heap_size; -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (is_shared_memory) { - memory->end_addr = memory->memory_data + - num_bytes_per_page * memory->max_page_count; - } - else -#endif - { - memory->end_addr = memory->memory_data + - num_bytes_per_page * memory->cur_page_count; - } + memory->heap_data = memory->memory_data + heap_offset; + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data_end = memory->memory_data + (uint32)memory_data_size; - bh_assert(memory->end_addr - (uint8*)memory == (uint32)total_size); + bh_assert(memory->memory_data_end - (uint8*)memory == (uint32)total_size); /* Initialize heap */ if (heap_size > 0 && !(memory->heap_handle = mem_allocator_create(memory->heap_data, heap_size))) { - wasm_runtime_free(memory); - return NULL; + set_error_buf(error_buf, error_buf_size, + "Instantiate memory failed: " + "init app heap failed."); + goto fail1; } - memory->heap_base_offset = -(int32)heap_size; - #if WASM_ENABLE_SHARED_MEMORY != 0 if (0 != os_mutex_init(&memory->mem_lock)) { - mem_allocator_destroy(memory->heap_handle); - wasm_runtime_free(memory); - return NULL; + set_error_buf(error_buf, error_buf_size, + "Instantiate memory failed: " + "init mutex failed."); + goto fail2; } if (is_shared_memory) { memory->is_shared = true; @@ -204,16 +281,23 @@ memory_instantiate(WASMModuleInstance *module_inst, (WASMModuleCommon *)module_inst->module, (WASMMemoryInstanceCommon *)memory)) { set_error_buf(error_buf, error_buf_size, - "Instantiate memory failed:" + "Instantiate memory failed: " "allocate memory failed."); - os_mutex_destroy(&memory->mem_lock); - mem_allocator_destroy(memory->heap_handle); - wasm_runtime_free(memory); - return NULL; + goto fail3; } } #endif return memory; +#if WASM_ENABLE_SHARED_MEMORY != 0 +fail3: + os_mutex_destroy(&memory->mem_lock); +fail2: + if (heap_size > 0) + mem_allocator_destroy(memory->heap_handle); +#endif +fail1: + wasm_runtime_free(memory); + return NULL; } /** @@ -275,12 +359,7 @@ memories_instantiate(const WASMModule *module, module_inst, num_bytes_per_page, init_page_count, max_page_count, actual_heap_size, flags, error_buf, error_buf_size))) { - set_error_buf(error_buf, error_buf_size, - "Instantiate memory failed: " - "allocate memory failed."); - memories_deinstantiate( - module_inst, - memories, memory_count); + memories_deinstantiate(module_inst, memories, memory_count); return NULL; } } @@ -295,12 +374,7 @@ memories_instantiate(const WASMModule *module, module->memories[i].max_page_count, heap_size, module->memories[i].flags, error_buf, error_buf_size))) { - set_error_buf(error_buf, error_buf_size, - "Instantiate memory failed: " - "allocate memory failed."); - memories_deinstantiate( - module_inst, - memories, memory_count); + memories_deinstantiate(module_inst, memories, memory_count); return NULL; } #if WASM_ENABLE_MULTI_MODULE != 0 @@ -316,9 +390,6 @@ memories_instantiate(const WASMModule *module, if (!(memory = memories[mem_index++] = memory_instantiate(module_inst, 0, 0, 0, heap_size, 0, error_buf, error_buf_size))) { - set_error_buf(error_buf, error_buf_size, - "Instantiate memory failed: " - "allocate memory failed.\n"); memories_deinstantiate(module_inst, memories, memory_count); return NULL; } @@ -914,6 +985,34 @@ execute_start_function(WASMModuleInstance *module_inst) return wasm_create_exec_env_and_call_function(module_inst, func, 0, NULL); } +static bool +execute_malloc_function(WASMModuleInstance *module_inst, + WASMFunctionInstance *malloc_func, + uint32 size, uint32 *p_result) +{ + uint32 argv[2]; + bool ret; + + argv[0] = size; + ret = wasm_create_exec_env_and_call_function + (module_inst, malloc_func, 1, argv); + if (ret) + *p_result = argv[0]; + return ret; +} + +static bool +execute_free_function(WASMModuleInstance *module_inst, + WASMFunctionInstance *free_func, + uint32 offset) +{ + uint32 argv[2]; + + argv[0] = offset; + return wasm_create_exec_env_and_call_function + (module_inst, free_func, 1, argv); +} + #if WASM_ENABLE_MULTI_MODULE != 0 static bool sub_module_instantiate(WASMModule *module, WASMModuleInstance *module_inst, @@ -1251,6 +1350,28 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, table_seg->func_indexes, (uint32)(length * sizeof(uint32))); } + /* module instance type */ + module_inst->module_type = Wasm_Module_Bytecode; + + /* Initialize the thread related data */ + if (stack_size == 0) + stack_size = DEFAULT_WASM_STACK_SIZE; +#if WASM_ENABLE_SPEC_TEST != 0 + if (stack_size < 48 *1024) + stack_size = 48 * 1024; +#endif + module_inst->default_wasm_stack_size = stack_size; + + if (module->malloc_function != (uint32)-1) { + module_inst->malloc_function = + &module_inst->functions[module->malloc_function]; + } + + if (module->free_function != (uint32)-1) { + module_inst->free_function = + &module_inst->functions[module->free_function]; + } + #if WASM_ENABLE_LIBC_WASI != 0 /* The sub-instance will get the wasi_ctx from main-instance */ if (!is_sub_inst) { @@ -1278,18 +1399,6 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, &module_inst->functions[module->start_function]; } - /* module instance type */ - module_inst->module_type = Wasm_Module_Bytecode; - - /* Initialize the thread related data */ - if (stack_size == 0) - stack_size = DEFAULT_WASM_STACK_SIZE; -#if WASM_ENABLE_SPEC_TEST != 0 - if (stack_size < 48 *1024) - stack_size = 48 * 1024; -#endif - module_inst->default_wasm_stack_size = stack_size; - /* Execute __post_instantiate function */ if (!execute_post_inst_function(module_inst) || !execute_start_function(module_inst)) { @@ -1467,7 +1576,22 @@ wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size, void **p_native_addr) { WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = mem_allocator_malloc(memory->heap_handle, size); + uint8 *addr = NULL; + uint32 offset = 0; + + if (memory->heap_handle) { + addr = mem_allocator_malloc(memory->heap_handle, size); + } + else if (module_inst->malloc_function + && module_inst->free_function) { + if (!execute_malloc_function(module_inst, + module_inst->malloc_function, + size, &offset)) { + return 0; + } + addr = offset ? memory->memory_data + offset : NULL; + } + if (!addr) { wasm_set_exception(module_inst, "out of memory"); return 0; @@ -1482,9 +1606,21 @@ wasm_module_free(WASMModuleInstance *module_inst, int32 ptr) { if (ptr) { WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = memory->memory_data + ptr; - if (memory->heap_data < addr && addr < memory->memory_data) + uint8 *addr = memory->memory_data + (uint32)ptr; + + if (memory->heap_handle + && memory->heap_data <= addr + && addr < memory->heap_data_end) { mem_allocator_free(memory->heap_handle, addr); + } + else if (module_inst->malloc_function + && module_inst->free_function + && memory->memory_data <= addr + && addr < memory->memory_data_end) { + execute_free_function(module_inst, + module_inst->free_function, + (uint32)ptr); + } } } @@ -1507,16 +1643,15 @@ wasm_validate_app_addr(WASMModuleInstance *module_inst, int32 app_offset, uint32 size) { WASMMemoryInstance *memory = module_inst->default_memory; - int32 memory_data_size = - (int32)(memory->num_bytes_per_page * memory->cur_page_count); + uint32 memory_data_size = + memory->num_bytes_per_page * memory->cur_page_count; /* integer overflow check */ - if (app_offset + (int32)size < app_offset) { + if ((uint32)app_offset + size < (uint32)app_offset) { goto fail; } - if (memory->heap_base_offset <= app_offset - && app_offset + (int32)size <= memory_data_size) { + if ((uint32)app_offset + size <= memory_data_size) { return true; } fail: @@ -1528,17 +1663,16 @@ bool wasm_validate_native_addr(WASMModuleInstance *module_inst, void *native_ptr, uint32 size) { - uint8 *addr = (uint8*)native_ptr; + uint8 *addr = (uint8 *)native_ptr; WASMMemoryInstance *memory = module_inst->default_memory; - int32 memory_data_size = - (int32)(memory->num_bytes_per_page * memory->cur_page_count); + /* integer overflow check */ if (addr + size < addr) { goto fail; } - if (memory->heap_data <= addr - && addr + size <= memory->memory_data + memory_data_size) { + if (memory->memory_data <= addr + && addr + size <= memory->memory_data_end) { return true; } fail: @@ -1552,11 +1686,9 @@ wasm_addr_app_to_native(WASMModuleInstance *module_inst, { WASMMemoryInstance *memory = module_inst->default_memory; uint8 *addr = memory->memory_data + app_offset; - int32 memory_data_size = - (int32)(memory->num_bytes_per_page * memory->cur_page_count); - if (memory->heap_data <= addr - && addr < memory->memory_data + memory_data_size) + if (memory->memory_data <= addr + && addr < memory->memory_data_end) return addr; return NULL; } @@ -1566,12 +1698,10 @@ wasm_addr_native_to_app(WASMModuleInstance *module_inst, void *native_ptr) { WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8*)native_ptr; - int32 memory_data_size = - (int32)(memory->num_bytes_per_page * memory->cur_page_count); + uint8 *addr = (uint8 *)native_ptr; - if (memory->heap_data <= addr - && addr < memory->memory_data + memory_data_size) + if (memory->memory_data <= addr + && addr < memory->memory_data_end) return (int32)(addr - memory->memory_data); return 0; } @@ -1583,13 +1713,12 @@ wasm_get_app_addr_range(WASMModuleInstance *module_inst, int32 *p_app_end_offset) { WASMMemoryInstance *memory = module_inst->default_memory; - int32 memory_data_size = - (int32)(memory->num_bytes_per_page * memory->cur_page_count); + uint32 memory_data_size = + memory->num_bytes_per_page * memory->cur_page_count; - if (memory->heap_base_offset <= app_offset - && app_offset < memory_data_size) { + if ((uint32)app_offset < memory_data_size) { if (p_app_start_offset) - *p_app_start_offset = memory->heap_base_offset; + *p_app_start_offset = 0; if (p_app_end_offset) *p_app_end_offset = memory_data_size; return true; @@ -1604,16 +1733,14 @@ wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 **p_native_end_addr) { WASMMemoryInstance *memory = module_inst->default_memory; - uint8 *addr = (uint8*)native_ptr; - int32 memory_data_size = - (int32)(memory->num_bytes_per_page * memory->cur_page_count); + uint8 *addr = (uint8 *)native_ptr; - if (memory->heap_data <= addr - && addr < memory->memory_data + memory_data_size) { + if (memory->memory_data <= addr + && addr < memory->memory_data_end) { if (p_native_start_addr) - *p_native_start_addr = memory->heap_data; + *p_native_start_addr = memory->memory_data; if (p_native_end_addr) - *p_native_end_addr = memory->memory_data + memory_data_size; + *p_native_end_addr = memory->memory_data_end; return true; } return false; @@ -1623,13 +1750,13 @@ bool wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) { WASMMemoryInstance *memory = module->default_memory, *new_memory; - uint32 heap_size = memory->memory_data - memory->heap_data; - uint32 total_size_old = memory->end_addr - (uint8*)memory; + uint32 heap_size = memory->heap_data_end - memory->heap_data; + uint32 total_size_old = memory->memory_data_end - (uint8 *)memory; uint32 total_page_count = inc_page_count + memory->cur_page_count; - uint64 total_size = offsetof(WASMMemoryInstance, base_addr) - + (uint64)heap_size + uint64 total_size = offsetof(WASMMemoryInstance, memory_data) + memory->num_bytes_per_page * (uint64)total_page_count; void *heap_handle_old = memory->heap_handle; + uint8 *heap_data_old = memory->heap_data; if (inc_page_count <= 0) /* No need to enlarge memory */ @@ -1669,17 +1796,17 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) wasm_set_exception(module, "fail to enlarge memory."); return false; } - bh_memcpy_s((uint8*)new_memory, (uint32)total_size, - (uint8*)memory, total_size_old); + bh_memcpy_s((uint8 *)new_memory, (uint32)total_size, + (uint8 *)memory, total_size_old); wasm_runtime_free(memory); } - memset((uint8*)new_memory + total_size_old, + memset((uint8 *)new_memory + total_size_old, 0, (uint32)total_size - total_size_old); if (heap_size > 0) { - new_memory->heap_handle = (uint8*)heap_handle_old + - ((uint8*)new_memory - (uint8*)memory); + new_memory->heap_handle = (uint8 *)heap_handle_old + + ((uint8 *)new_memory - (uint8 *)memory); if (mem_allocator_migrate(new_memory->heap_handle, heap_handle_old) != 0) { wasm_set_exception(module, "fail to enlarge memory."); @@ -1688,10 +1815,12 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) } new_memory->cur_page_count = total_page_count; - new_memory->heap_data = new_memory->base_addr; - new_memory->memory_data = new_memory->base_addr + heap_size; - new_memory->end_addr = new_memory->memory_data + - new_memory->num_bytes_per_page * total_page_count; + new_memory->heap_data = heap_data_old + + ((uint8 *)new_memory - (uint8 *)memory); + new_memory->heap_data_end = new_memory->heap_data + heap_size; + new_memory->memory_data_end = new_memory->memory_data + + new_memory->num_bytes_per_page + * total_page_count; module->memories[0] = module->default_memory = new_memory; return true; @@ -1757,13 +1886,9 @@ wasm_set_aux_stack(WASMExecEnv *exec_env, { WASMModuleInstance *module_inst = (WASMModuleInstance*)exec_env->module_inst; - - uint32 stack_top_idx = - module_inst->module->llvm_aux_stack_global_index; - uint32 data_end = - module_inst->module->llvm_aux_data_end; - uint32 stack_bottom = - module_inst->module->llvm_aux_stack_bottom; + uint32 stack_top_idx = module_inst->module->aux_stack_top_global_index; + uint32 data_end = module_inst->module->aux_data_end; + uint32 stack_bottom = module_inst->module->aux_stack_bottom; bool is_stack_before_data = stack_bottom < data_end ? true : false; @@ -1772,7 +1897,7 @@ wasm_set_aux_stack(WASMExecEnv *exec_env, || ((!is_stack_before_data) && (start_offset - data_end < size))) return false; - if ((stack_bottom != (uint32)-1) && (stack_top_idx != (uint32)-1)) { + if (stack_top_idx != (uint32)-1) { /* The aux stack top is a wasm global, set the initial value for the global */ uint8 *global_addr = @@ -1798,9 +1923,9 @@ wasm_get_aux_stack(WASMExecEnv *exec_env, /* The aux stack information is resolved in loader and store in module */ uint32 stack_bottom = - module_inst->module->llvm_aux_stack_bottom; + module_inst->module->aux_stack_bottom; uint32 total_aux_stack_size = - module_inst->module->llvm_aux_stack_size; + module_inst->module->aux_stack_size; if (stack_bottom != 0 && total_aux_stack_size != 0) { if (start_offset) diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index fcb9a1e6..2750e69c 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -33,19 +33,13 @@ typedef struct WASMMemoryInstance { /* Maximum page count */ uint32 max_page_count; - /* Heap base offset of wasm app */ - int32 heap_base_offset; /* Heap data base address */ uint8 *heap_data; + /* Heap data end address */ + uint8 *heap_data_end; /* The heap created */ void *heap_handle; - /* Memory data */ - uint8 *memory_data; - - /* End address of memory */ - uint8 *end_addr; - #if WASM_ENABLE_MULTI_MODULE != 0 /* to indicate which module instance create it */ WASMModuleInstance *owner; @@ -56,13 +50,13 @@ typedef struct WASMMemoryInstance { korp_mutex mem_lock; #endif - /* Base address, the layout is: - heap_data + memory data - memory data init size is: num_bytes_per_page * cur_page_count + /* Memory data end address */ + uint8 *memory_data_end; + + /* Memory data begin address, the layout is: memory data + heap data Note: when memory is re-allocated, the heap data and memory data - must be copied to new memory also. - */ - uint8 base_addr[1]; + must be copied to new memory also. */ + uint8 memory_data[1]; } WASMMemoryInstance; typedef struct WASMTableInstance { @@ -188,6 +182,8 @@ typedef struct WASMModuleInstance { uint8 *global_data; WASMFunctionInstance *start_function; + WASMFunctionInstance *malloc_function; + WASMFunctionInstance *free_function; WASMModule *module; diff --git a/core/shared/platform/common/posix/posix_memmap.c b/core/shared/platform/common/posix/posix_memmap.c index 26b8d842..0bbea276 100644 --- a/core/shared/platform/common/posix/posix_memmap.c +++ b/core/shared/platform/common/posix/posix_memmap.c @@ -65,7 +65,7 @@ os_munmap(void *addr, size_t size) if (addr) { if (munmap(addr, request_size)) { - os_printf("os_munmap error addr:%p, size:0x%lx, errno:%d\n", + os_printf("os_munmap error addr:%p, size:0x%"PRIx64", errno:%d\n", addr, request_size, errno); } } diff --git a/core/shared/platform/darwin/platform_init.c b/core/shared/platform/darwin/platform_init.c index 76594b81..4f806083 100644 --- a/core/shared/platform/darwin/platform_init.c +++ b/core/shared/platform/darwin/platform_init.c @@ -16,3 +16,22 @@ bh_platform_destroy() { } +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); + ret += vprintf(format, ap); + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ + return vprintf(format, ap); +} + diff --git a/core/shared/platform/darwin/platform_internal.h b/core/shared/platform/darwin/platform_internal.h index 3aab6be0..75348df1 100644 --- a/core/shared/platform/darwin/platform_internal.h +++ b/core/shared/platform/darwin/platform_internal.h @@ -57,9 +57,6 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; -#define os_printf printf -#define os_vprintf vprintf - #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ diff --git a/core/shared/platform/linux-sgx/sgx_platform.c b/core/shared/platform/linux-sgx/sgx_platform.c index f7f5c622..abfb9911 100644 --- a/core/shared/platform/linux-sgx/sgx_platform.c +++ b/core/shared/platform/linux-sgx/sgx_platform.c @@ -152,8 +152,8 @@ int os_mprotect(void *addr, size_t size, int prot) mprot |= SGX_PROT_EXEC; st = sgx_tprotect_rsrv_mem(addr, aligned_size, mprot); if (st != SGX_SUCCESS) - os_printf("os_mprotect(addr=0x%lx, size=%u, prot=0x%x) failed.", - addr, size, prot); + os_printf("os_mprotect(addr=0x%"PRIx64", size=%u, prot=0x%x) failed.", + (uintptr_t)addr, size, prot); return (st == SGX_SUCCESS? 0:-1); } diff --git a/core/shared/platform/linux/platform_init.c b/core/shared/platform/linux/platform_init.c index 76594b81..4f806083 100644 --- a/core/shared/platform/linux/platform_init.c +++ b/core/shared/platform/linux/platform_init.c @@ -16,3 +16,22 @@ bh_platform_destroy() { } +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); + ret += vprintf(format, ap); + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ + return vprintf(format, ap); +} + diff --git a/core/shared/platform/linux/platform_internal.h b/core/shared/platform/linux/platform_internal.h index 1aa22ed4..7a15f85e 100644 --- a/core/shared/platform/linux/platform_internal.h +++ b/core/shared/platform/linux/platform_internal.h @@ -56,9 +56,6 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; -#define os_printf printf -#define os_vprintf vprintf - #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ diff --git a/core/shared/platform/vxworks/platform_init.c b/core/shared/platform/vxworks/platform_init.c index 76594b81..4f806083 100644 --- a/core/shared/platform/vxworks/platform_init.c +++ b/core/shared/platform/vxworks/platform_init.c @@ -16,3 +16,22 @@ bh_platform_destroy() { } +int +os_printf(const char *format, ...) +{ + int ret = 0; + va_list ap; + + va_start(ap, format); + ret += vprintf(format, ap); + va_end(ap); + + return ret; +} + +int +os_vprintf(const char *format, va_list ap) +{ + return vprintf(format, ap); +} + diff --git a/core/shared/platform/vxworks/platform_internal.h b/core/shared/platform/vxworks/platform_internal.h index 701ab644..a8479361 100644 --- a/core/shared/platform/vxworks/platform_internal.h +++ b/core/shared/platform/vxworks/platform_internal.h @@ -55,9 +55,6 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; -#define os_printf printf -#define os_vprintf vprintf - #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ diff --git a/core/shared/utils/bh_log.h b/core/shared/utils/bh_log.h index 8f7a179d..6a5ead46 100644 --- a/core/shared/utils/bh_log.h +++ b/core/shared/utils/bh_log.h @@ -54,7 +54,7 @@ bh_log(LogLevel log_level, const char *file, int line, const char *fmt, ...); #if BH_DEBUG == 1 #define LOG_DEBUG(...) bh_log(BH_LOG_LEVEL_DEBUG, __FILE__, __LINE__, __VA_ARGS__) #else -#define LOG_DEBUG(...) /* do nothing */ +#define LOG_DEBUG(...) (void)0 #endif void diff --git a/samples/wasm-c-api/src/callback.c b/samples/wasm-c-api/src/callback.c index df2e6559..da3e7491 100644 --- a/samples/wasm-c-api/src/callback.c +++ b/samples/wasm-c-api/src/callback.c @@ -68,7 +68,11 @@ int main(int argc, const char* argv[]) { // Load binary. printf("Loading binary...\n"); +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_INTERP == 0 + FILE* file = fopen("callback.aot", "rb"); +#else FILE* file = fopen("callback.wasm", "rb"); +#endif if (!file) { printf("> Error loading module!\n"); return 1;