christoph/wamr
1
Fork 0
Code Releases Activity

Implement memory64 for classic interpreter (#3266)

Browse Source 
Adding a new cmake flag (cache variable) `WAMR_BUILD_MEMORY64` to enable
the memory64 feature, it can only be enabled on the 64-bit platform/target and
can only use software boundary check. And when it is enabled, it can support both
i32 and i64 linear memory types. The main modifications are:

- wasm loader & mini-loader: loading and bytecode validating process 
- wasm runtime: memory instantiating process
- classic-interpreter: wasm code executing process
- Support memory64 memory in related runtime APIs
- Modify main function type check when it's memory64 wasm file
- Modify `wasm_runtime_invoke_native` and `wasm_runtime_invoke_native_raw` to
  handle registered native function pointer argument when memory64 is enabled
- memory64 classic-interpreter spec test in `test_wamr.sh` and in CI

Currently, it supports memory64 memory wasm file that uses core spec
(including bulk memory proposal) opcodes and threads opcodes.

ps.
https://github.com/bytecodealliance/wasm-micro-runtime/issues/3091
https://github.com/bytecodealliance/wasm-micro-runtime/pull/3240
https://github.com/bytecodealliance/wasm-micro-runtime/pull/3260
This commit is contained in:
Wenyong Huang
2024-04-02 15:22:07 +08:00
committed by GitHub
parent 6b0b5de1c5
commit a23fa9f86c
22 changed files with 1084 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
core/config.h
View File

@ -415,7 +415,7 @@
#else
#define DEFAULT_WASM_STACK_SIZE (12 * 1024)
#endif
/* Min auxilliary stack size of each wasm thread */
/* Min auxiliary stack size of each wasm thread */
#define WASM_THREAD_AUX_STACK_SIZE_MIN (256)
/* Default/min native stack size of each app thread */
@ -564,7 +564,7 @@
#endif
/* Support registering quick AOT/JIT function entries of some func types
to speedup the calling process of invoking the AOT/JIT functions of
to speed up the calling process of invoking the AOT/JIT functions of
these types from the host embedder */
#ifndef WASM_ENABLE_QUICK_AOT_ENTRY
#define WASM_ENABLE_QUICK_AOT_ENTRY 1
@ -578,6 +578,11 @@
#define WASM_ENABLE_AOT_INTRINSICS 1
#endif
/* Disable memory64 by default */
#ifndef WASM_ENABLE_MEMORY64
#define WASM_ENABLE_MEMORY64 0
#endif
#ifndef WASM_TABLE_MAX_SIZE
#define WASM_TABLE_MAX_SIZE 1024
#endif

7
core/iwasm/aot/aot_runtime.c
View File

@ -914,9 +914,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE);
(void)max_memory_data_size;
if (wasm_allocate_linear_memory(&p, is_shared_memory, num_bytes_per_page,
init_page_count, max_page_count,
&memory_data_size)
/* TODO: memory64 uses is_memory64 flag */
if (wasm_allocate_linear_memory(&p, is_shared_memory, false,
num_bytes_per_page, init_page_count,
max_page_count, &memory_data_size)
!= BHT_OK) {
set_error_buf(error_buf, error_buf_size,
"allocate linear memory failed");

33
core/iwasm/common/wasm_application.c
View File

@ -61,7 +61,7 @@ static union {
* Implementation of wasm_application_execute_main()
*/
static bool
check_main_func_type(const WASMFuncType *type)
check_main_func_type(const WASMFuncType *type, bool is_memory64)
{
if (!(type->param_count == 0 || type->param_count == 2)
|| type->result_count > 1) {
@ -72,7 +72,8 @@ check_main_func_type(const WASMFuncType *type)
if (type->param_count == 2
&& !(type->types[0] == VALUE_TYPE_I32
&& type->types[1] == VALUE_TYPE_I32)) {
&& type->types[1]
== (is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32))) {
LOG_ERROR(
"WASM execute application failed: invalid main function type.\n");
return false;
@ -94,14 +95,18 @@ execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[])
WASMFunctionInstanceCommon *func;
WASMFuncType *func_type = NULL;
WASMExecEnv *exec_env = NULL;
uint32 argc1 = 0, argv1[2] = { 0 };
uint32 argc1 = 0, argv1[3] = { 0 };
uint32 total_argv_size = 0;
uint64 total_size;
uint64 argv_buf_offset = 0;
int32 i;
char *argv_buf, *p, *p_end;
uint32 *argv_offsets, module_type;
bool ret, is_import_func = true;
bool ret, is_import_func = true, is_memory64 = false;
#if WASM_ENABLE_MEMORY64 != 0
WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst;
is_memory64 = wasm_module_inst->memories[0]->is_memory64;
#endif
exec_env = wasm_runtime_get_exec_env_singleton(module_inst);
if (!exec_env) {
@ -187,7 +192,7 @@ execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[])
return false;
}
if (!check_main_func_type(func_type)) {
if (!check_main_func_type(func_type, is_memory64)) {
wasm_runtime_set_exception(module_inst,
"invalid function type of main function");
return false;
@ -218,11 +223,21 @@ execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[])
p += strlen(argv[i]) + 1;
}
argc1 = 2;
argv1[0] = (uint32)argc;
/* TODO: memory64 uint64 when the memory idx is i64 */
argv1[1] =
(uint32)wasm_runtime_addr_native_to_app(module_inst, argv_offsets);
#if WASM_ENABLE_MEMORY64 != 0
if (is_memory64) {
argc1 = 3;
uint64 app_addr =
wasm_runtime_addr_native_to_app(module_inst, argv_offsets);
PUT_I64_TO_ADDR(&argv[1], app_addr);
}
else
#endif
{
argc1 = 2;
argv1[1] = (uint32)wasm_runtime_addr_native_to_app(module_inst,
argv_offsets);
}
}
ret = wasm_runtime_call_wasm(exec_env, func, argc1, argv1);

51
core/iwasm/common/wasm_memory.c
View File

@ -286,6 +286,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
{
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
WASMMemoryInstance *memory_inst;
uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE;
bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode
|| module_inst_comm->module_type == Wasm_Module_AoT);
@ -299,9 +300,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail;
}
#if WASM_ENABLE_MEMORY64 != 0
if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
#endif
/* boundary overflow check */
if (size > MAX_LINEAR_MEMORY_SIZE
|| app_offset > MAX_LINEAR_MEMORY_SIZE - size) {
if (size > max_linear_memory_size
|| app_offset > max_linear_memory_size - size) {
goto fail;
}
@ -324,7 +329,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
uint64 app_str_offset)
{
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
uint64 app_end_offset;
uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE;
char *str, *str_end;
bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode
@ -338,10 +343,14 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
&app_end_offset))
goto fail;
#if WASM_ENABLE_MEMORY64 != 0
if (module_inst->memories[0]->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
#endif
/* boundary overflow check, max start offset can only be size - 1, while end
* offset can be size */
if (app_str_offset >= MAX_LINEAR_MEMORY_SIZE
|| app_end_offset > MAX_LINEAR_MEMORY_SIZE)
if (app_str_offset >= max_linear_memory_size
|| app_end_offset > max_linear_memory_size)
goto fail;
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
@ -364,6 +373,7 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
WASMMemoryInstance *memory_inst;
uint8 *addr = (uint8 *)native_ptr;
uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE;
bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode
|| module_inst_comm->module_type == Wasm_Module_AoT);
@ -377,8 +387,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail;
}
#if WASM_ENABLE_MEMORY64 != 0
if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
#endif
/* boundary overflow check */
if (size > MAX_LINEAR_MEMORY_SIZE || (uintptr_t)addr > UINTPTR_MAX - size) {
if (size > max_linear_memory_size || (uintptr_t)addr > UINTPTR_MAX - size) {
goto fail;
}
@ -748,12 +762,13 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
goto return_func;
}
bh_assert(total_size_new <= MAX_LINEAR_MEMORY_SIZE);
bh_assert(total_size_new
<= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64));
if (full_size_mmaped) {
#ifdef BH_PLATFORM_WINDOWS
if (!os_mem_commit(memory->memory_data_end,
(uint32)(total_size_new - total_size_old),
(mem_offset_t)(total_size_new - total_size_old),
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
ret = false;
goto return_func;
@ -761,12 +776,12 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
#endif
if (os_mprotect(memory->memory_data_end,
(uint32)(total_size_new - total_size_old),
(mem_offset_t)(total_size_new - total_size_old),
MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(memory->memory_data_end,
(uint32)(total_size_new - total_size_old));
(mem_offset_t)(total_size_new - total_size_old));
#endif
ret = false;
goto return_func;
@ -895,8 +910,9 @@ wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst)
int
wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
uint64 num_bytes_per_page, uint64 init_page_count,
uint64 max_page_count, uint64 *memory_data_size)
bool is_memory64, uint64 num_bytes_per_page,
uint64 init_page_count, uint64 max_page_count,
uint64 *memory_data_size)
{
uint64 map_size, page_size;
@ -925,7 +941,16 @@ wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
page_size = os_getpagesize();
*memory_data_size = init_page_count * num_bytes_per_page;
bh_assert(*memory_data_size <= MAX_LINEAR_MEMORY_SIZE);
#if WASM_ENABLE_MEMORY64 != 0
if (is_memory64) {
bh_assert(*memory_data_size <= MAX_LINEAR_MEM64_MEMORY_SIZE);
}
else
#endif
{
bh_assert(*memory_data_size <= MAX_LINEAR_MEMORY_SIZE);
}
align_as_and_cast(*memory_data_size, page_size);
if (map_size > 0) {

5
core/iwasm/common/wasm_memory.h
View File

@ -64,8 +64,9 @@ wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst);
int
wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
uint64 num_bytes_per_page, uint64 init_page_count,
uint64 max_page_count, uint64 *memory_data_size);
bool is_memory64, uint64 num_bytes_per_page,
uint64 init_page_count, uint64 max_page_count,
uint64 *memory_data_size);
#ifdef __cplusplus
}

103
core/iwasm/common/wasm_runtime_common.c
View File

@ -3746,7 +3746,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env);
typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *);
NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr;
uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size;
uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64;
uint32 *argv_src = argv, i, argc1, ptr_len;
uint32 arg_i32;
bool ret = false;
@ -3770,9 +3770,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
case VALUE_TYPE_FUNCREF:
#endif
{
/* TODO: memory64 the data type of ptr_len and argc depends on
* mem idx type */
*(uint32 *)argv_dst = arg_i32 = *argv_src++;
/* TODO: memory64 if future there is a way for supporting
* wasm64 and wasm32 in libc at the same time, remove the
* macro control */
#if WASM_ENABLE_MEMORY64 == 0
if (signature) {
if (signature[i + 1] == '*') {
/* param is a pointer */
@ -3802,9 +3804,49 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
module, (uint64)arg_i32);
}
}
#endif
break;
}
case VALUE_TYPE_I64:
#if WASM_ENABLE_MEMORY64 != 0
{
PUT_I64_TO_ADDR((uint32 *)argv_dst,
GET_I64_FROM_ADDR(argv_src));
argv_src += 2;
arg_i64 = *argv_dst;
if (signature) {
/* TODO: memory64 pointer with length need a new symbol
* to represent type i64, with '~' still represent i32
* length */
if (signature[i + 1] == '*') {
/* param is a pointer */
if (signature[i + 2] == '~')
/* pointer with length followed */
ptr_len = *argv_src;
else
/* pointer without length followed */
ptr_len = 1;
if (!wasm_runtime_validate_app_addr(module, arg_i64,
(uint64)ptr_len))
goto fail;
*argv_dst = (uint64)wasm_runtime_addr_app_to_native(
module, arg_i64);
}
else if (signature[i + 1] == '$') {
/* param is a string */
if (!wasm_runtime_validate_app_str_addr(module,
arg_i64))
goto fail;
*argv_dst = (uint64)wasm_runtime_addr_app_to_native(
module, arg_i64);
}
}
break;
}
#endif
case VALUE_TYPE_F64:
bh_memcpy_s(argv_dst, sizeof(uint64), argv_src,
sizeof(uint32) * 2);
@ -3933,6 +3975,9 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
fail:
if (argv1 != argv_buf)
wasm_runtime_free(argv1);
#if WASM_ENABLE_MEMORY64 == 0
(void)arg_i64;
#endif
return ret;
}
@ -4195,8 +4240,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i32 = *argv_src++;
/* TODO: memory64 the data type of ptr_len and argc depends on
* mem idx type */
if (signature) {
if (signature[i + 1] == '*') {
/* param is a pointer */
@ -4572,8 +4615,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i32 = *argv++;
/* TODO: memory64 the data type of ptr_len and argc depends on
* mem idx type */
if (signature) {
if (signature[i + 1] == '*') {
/* param is a pointer */
@ -4889,8 +4930,10 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i32 = *argv_src++;
arg_i64 = arg_i32;
/* TODO: memory64 the data type of ptr_len and argc depends on
* mem idx type */
/* TODO: memory64 if future there is a way for supporting
* wasm64 and wasm32 in libc at the same time, remove the
* macro control */
#if WASM_ENABLE_MEMORY64 == 0
if (signature) {
if (signature[i + 1] == '*') {
/* param is a pointer */
@ -4918,6 +4961,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
module, (uint64)arg_i32);
}
}
#endif
if (n_ints < MAX_REG_INTS)
ints[n_ints++] = arg_i64;
else
@ -4925,6 +4969,47 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
break;
}
case VALUE_TYPE_I64:
#if WASM_ENABLE_MEMORY64 != 0
{
arg_i64 = GET_I64_FROM_ADDR(argv_src);
argv_src += 2;
if (signature) {
/* TODO: memory64 pointer with length need a new symbol
* to represent type i64, with '~' still represent i32
* length */
if (signature[i + 1] == '*') {
/* param is a pointer */
if (signature[i + 2] == '~')
/* pointer with length followed */
ptr_len = *argv_src;
else
/* pointer without length followed */
ptr_len = 1;
if (!wasm_runtime_validate_app_addr(module, arg_i64,
(uint64)ptr_len))
goto fail;
arg_i64 = (uint64)wasm_runtime_addr_app_to_native(
module, arg_i64);
}
else if (signature[i + 1] == '$') {
/* param is a string */
if (!wasm_runtime_validate_app_str_addr(module,
arg_i64))
goto fail;
arg_i64 = (uint64)wasm_runtime_addr_app_to_native(
module, arg_i64);
}
}
if (n_ints < MAX_REG_INTS)
ints[n_ints++] = arg_i64;
else
stacks[n_stacks++] = arg_i64;
break;
}
#endif
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:

4
core/iwasm/common/wasm_runtime_common.h
View File

@ -373,7 +373,7 @@ typedef struct WASMModuleCommon {
/* The following uint8[1] member is a dummy just to indicate
some module_type dependent members follow.
Typically it should be accessed by casting to the corresponding
Typically, it should be accessed by casting to the corresponding
actual module_type dependent structure, not via this member. */
uint8 module_data[1];
} WASMModuleCommon;
@ -389,7 +389,7 @@ typedef struct WASMModuleInstanceCommon {
/* The following uint8[1] member is a dummy just to indicate
some module_type dependent members follow.
Typically it should be accessed by casting to the corresponding
Typically, it should be accessed by casting to the corresponding
actual module_type dependent structure, not via this member. */
uint8 module_inst_data[1];
} WASMModuleInstanceCommon;

17
core/iwasm/interpreter/wasm.h
View File

@ -90,11 +90,22 @@ extern "C" {
*/
#define VALUE_TYPE_GC_REF 0x43
#define MAX_PAGE_COUNT_FLAG 0x01
#define SHARED_MEMORY_FLAG 0x02
#define MEMORY64_FLAG 0x04
#define DEFAULT_NUM_BYTES_PER_PAGE 65536
#define DEFAULT_MAX_PAGES 65536
#define DEFAULT_MEM64_MAX_PAGES UINT32_MAX
/* Max size of linear memory */
#define MAX_LINEAR_MEMORY_SIZE (4 * (uint64)BH_GB)
/* Roughly 274 TB */
#define MAX_LINEAR_MEM64_MEMORY_SIZE \
(DEFAULT_MEM64_MAX_PAGES * (uint64)64 * (uint64)BH_KB)
/* Macro to check memory flag and return appropriate memory size */
#define GET_MAX_LINEAR_MEMORY_SIZE(is_memory64) \
(is_memory64 ? MAX_LINEAR_MEM64_MEMORY_SIZE : MAX_LINEAR_MEMORY_SIZE)
#if WASM_ENABLE_GC == 0
typedef uintptr_t table_elem_type_t;
@ -484,6 +495,12 @@ typedef struct WASMTable {
#endif
} WASMTable;
#if WASM_ENABLE_MEMORY64 != 0
typedef uint64 mem_offset_t;
#else
typedef uint32 mem_offset_t;
#endif
typedef struct WASMMemory {
uint32 flags;
uint32 num_bytes_per_page;

391
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -46,8 +46,10 @@ typedef float64 CellType_F64;
#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory)
#endif
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0
#if WASM_ENABLE_MEMORY64 == 0
#if (!defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0)
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
@ -69,7 +71,8 @@ typedef float64 CellType_F64;
else \
goto out_of_bounds; \
} while (0)
#else
#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
@ -80,8 +83,37 @@ typedef float64 CellType_F64;
do { \
maddr = memory->memory_data + (uint32)(start); \
} while (0)
#endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
#endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
#else /* else of WASM_ENABLE_MEMORY64 == 0 */
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
/* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \
if (disable_bounds_checks \
|| (offset1 >= offset && offset1 + bytes >= offset1 \
&& offset1 + bytes <= get_linear_mem_size())) \
maddr = memory->memory_data + offset1; \
else \
goto out_of_bounds; \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint64)(start); \
/* If memory64 is enabled, offset1 + bytes can overflow */ \
if (disable_bounds_checks \
|| (offset1 + bytes >= offset1 \
&& offset1 + bytes <= get_linear_mem_size())) \
/* App heap space is not valid space for \
bulk memory operation */ \
maddr = memory->memory_data + offset1; \
else \
goto out_of_bounds; \
} while (0)
#endif /* end of WASM_ENABLE_MEMORY64 == 0 */
#define CHECK_ATOMIC_MEMORY_ACCESS() \
do { \
@ -472,6 +504,23 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame)
#define SET_LABEL_TYPE(_label_type) (void)0
#endif
#if WASM_ENABLE_MEMORY64 != 0
#define PUSH_MEM_OFFSET(value) \
do { \
if (is_memory64) { \
PUT_I64_TO_ADDR(frame_sp, value); \
frame_sp += 2; \
} \
else { \
*(int32 *)frame_sp++ = (int32)(value); \
} \
} while (0)
#else
#define PUSH_MEM_OFFSET(value) PUSH_I32(value)
#endif
#define PUSH_PAGE_COUNT(value) PUSH_MEM_OFFSET(value)
#define PUSH_CSP(_label_type, param_cell_num, cell_num, _target_addr) \
do { \
bh_assert(frame_csp < frame->csp_boundary); \
@ -501,6 +550,14 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame)
GET_REF_FROM_ADDR(frame_sp))
#endif
#if WASM_ENABLE_MEMORY64 != 0
#define POP_MEM_OFFSET() (is_memory64 ? POP_I64() : POP_I32())
#else
#define POP_MEM_OFFSET() POP_I32()
#endif
#define POP_PAGE_COUNT() POP_MEM_OFFSET()
#define POP_CSP_CHECK_OVERFLOW(n) \
do { \
bh_assert(frame_csp - n >= frame->csp_bottom); \
@ -567,51 +624,73 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame)
frame_csp = frame->csp; \
} while (0)
#define read_leb_int64(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = (int64)_val; \
if (_val & 0x40) \
/* sign extend */ \
res |= 0xFFFFFFFFFFFFFF80LL; \
p++; \
break; \
} \
uint32 _off = 0; \
res = (int64)read_leb(p, &_off, 64, true); \
p += _off; \
#define read_leb_int64(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = (int64)_val; \
if (_val & 0x40) \
/* sign extend */ \
res |= 0xFFFFFFFFFFFFFF80LL; \
p++; \
} \
else { \
uint32 _off = 0; \
res = (int64)read_leb(p, &_off, 64, true); \
p += _off; \
} \
} while (0)
#define read_leb_uint32(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = _val; \
p++; \
break; \
} \
uint32 _off = 0; \
res = (uint32)read_leb(p, &_off, 32, false); \
p += _off; \
#define read_leb_uint32(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = _val; \
p++; \
} \
else { \
uint32 _off = 0; \
res = (uint32)read_leb(p, &_off, 32, false); \
p += _off; \
} \
} while (0)
#define read_leb_int32(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = (int32)_val; \
if (_val & 0x40) \
/* sign extend */ \
res |= 0xFFFFFF80; \
p++; \
break; \
} \
uint32 _off = 0; \
res = (int32)read_leb(p, &_off, 32, true); \
p += _off; \
#define read_leb_int32(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = (int32)_val; \
if (_val & 0x40) \
/* sign extend */ \
res |= 0xFFFFFF80; \
p++; \
} \
else { \
uint32 _off = 0; \
res = (int32)read_leb(p, &_off, 32, true); \
p += _off; \
} \
} while (0)
#if WASM_ENABLE_MEMORY64 != 0
#define read_leb_mem_offset(p, p_end, res) \
do { \
uint8 _val = *p; \
if (!(_val & 0x80)) { \
res = (mem_offset_t)_val; \
p++; \
} \
else { \
uint32 _off = 0; \
res = (mem_offset_t)read_leb(p, &_off, is_memory64 ? 64 : 32, \
false); \
p += _off; \
} \
} while (0)
#else
#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res)
#endif
#if WASM_ENABLE_LABELS_AS_VALUES == 0
#define RECOVER_FRAME_IP_END() frame_ip_end = wasm_get_func_code_end(cur_func)
#else
@ -872,7 +951,7 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
uint32 readv, sval; \
\
sval = POP_I32(); \
addr = POP_I32(); \
addr = POP_MEM_OFFSET(); \
\
if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \
CHECK_MEMORY_OVERFLOW(1); \
@ -912,7 +991,7 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
uint64 readv, sval; \
\
sval = (uint64)POP_I64(); \
addr = POP_I32(); \
addr = POP_MEM_OFFSET(); \
\
if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \
CHECK_MEMORY_OVERFLOW(1); \
@ -1430,6 +1509,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
WASMStringviewIterObjectRef stringview_iter_obj;
#endif
#endif
#if WASM_ENABLE_MEMORY64 != 0
/* TODO: multi-memories for now assuming the memory idx type is consistent
* across multi-memories */
bool is_memory64 = false;
if (memory)
is_memory64 = memory->is_memory64;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
uint8 *frame_ip_orig = NULL;
@ -4087,8 +4173,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
bh_assert(global_idx < module->e->global_count);
global = globals + global_idx;
global_addr = get_global_addr(global_data, global);
/* TODO: Memory64 the data type depends on mem idx type */
aux_stack_top = (uint64)(*(uint32 *)(frame_sp - 1));
#if WASM_ENABLE_MEMORY64 != 0
if (is_memory64) {
aux_stack_top = *(uint64 *)(frame_sp - 2);
}
else
#endif
{
aux_stack_top = (uint64)(*(uint32 *)(frame_sp - 1));
}
if (aux_stack_top <= (uint64)exec_env->aux_stack_boundary) {
wasm_set_exception(module, "wasm auxiliary stack overflow");
goto got_exception;
@ -4098,8 +4191,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
"wasm auxiliary stack underflow");
goto got_exception;
}
*(int32 *)global_addr = aux_stack_top;
frame_sp--;
#if WASM_ENABLE_MEMORY64 != 0
if (is_memory64) {
*(uint64 *)global_addr = aux_stack_top;
frame_sp -= 2;
}
else
#endif
{
*(uint32 *)global_addr = aux_stack_top;
frame_sp--;
}
#if WASM_ENABLE_MEMORY_PROFILING != 0
if (module->module->aux_stack_top_global_index != (uint32)-1) {
uint32 aux_stack_used =
@ -4126,11 +4228,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_LOAD)
HANDLE_OP(WASM_OP_F32_LOAD)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(4);
PUSH_I32(LOAD_I32(maddr));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4141,11 +4244,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD)
HANDLE_OP(WASM_OP_F64_LOAD)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(8);
PUSH_I64(LOAD_I64(maddr));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4155,11 +4259,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_LOAD8_S)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(1);
PUSH_I32(sign_ext_8_32(*(int8 *)maddr));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4169,11 +4274,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_LOAD8_U)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(1);
PUSH_I32((uint32)(*(uint8 *)maddr));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4183,11 +4289,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_LOAD16_S)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(2);
PUSH_I32(sign_ext_16_32(LOAD_I16(maddr)));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4197,11 +4304,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_LOAD16_U)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(2);
PUSH_I32((uint32)(LOAD_U16(maddr)));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4211,11 +4319,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD8_S)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(1);
PUSH_I64(sign_ext_8_64(*(int8 *)maddr));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4225,11 +4334,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD8_U)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(1);
PUSH_I64((uint64)(*(uint8 *)maddr));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4239,11 +4349,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD16_S)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(2);
PUSH_I64(sign_ext_16_64(LOAD_I16(maddr)));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4253,11 +4364,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD16_U)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(2);
PUSH_I64((uint64)(LOAD_U16(maddr)));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4267,12 +4379,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD32_S)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
opcode = *(frame_ip - 1);
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(4);
PUSH_I64(sign_ext_32_64(LOAD_I32(maddr)));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4282,11 +4395,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_LOAD32_U)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
addr = POP_I32();
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(4);
PUSH_I64((uint64)(LOAD_U32(maddr)));
CHECK_READ_WATCHPOINT(addr, offset);
@ -4298,14 +4412,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_STORE)
HANDLE_OP(WASM_OP_F32_STORE)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
frame_sp--;
addr = POP_I32();
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(4);
STORE_U32(maddr, frame_sp[1]);
#if WASM_ENABLE_MEMORY64 != 0
if (is_memory64) {
STORE_U32(maddr, frame_sp[2]);
}
else
#endif
{
STORE_U32(maddr, frame_sp[1]);
}
CHECK_WRITE_WATCHPOINT(addr, offset);
(void)flags;
HANDLE_OP_END();
@ -4314,15 +4437,26 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_STORE)
HANDLE_OP(WASM_OP_F64_STORE)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
frame_sp -= 2;
addr = POP_I32();
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(8);
PUT_I64_TO_ADDR((uint32 *)maddr,
GET_I64_FROM_ADDR(frame_sp + 1));
#if WASM_ENABLE_MEMORY64 != 0
if (is_memory64) {
PUT_I64_TO_ADDR((mem_offset_t *)maddr,
GET_I64_FROM_ADDR(frame_sp + 2));
}
else
#endif
{
PUT_I64_TO_ADDR((uint32 *)maddr,
GET_I64_FROM_ADDR(frame_sp + 1));
}
CHECK_WRITE_WATCHPOINT(addr, offset);
(void)flags;
HANDLE_OP_END();
@ -4331,14 +4465,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I32_STORE8)
HANDLE_OP(WASM_OP_I32_STORE16)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
uint32 sval;
opcode = *(frame_ip - 1);
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
sval = (uint32)POP_I32();
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_I32_STORE8) {
CHECK_MEMORY_OVERFLOW(1);
@ -4357,14 +4492,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
HANDLE_OP(WASM_OP_I64_STORE16)
HANDLE_OP(WASM_OP_I64_STORE32)
{
uint32 offset, flags, addr;
uint32 flags;
mem_offset_t offset, addr;
uint64 sval;
opcode = *(frame_ip - 1);
read_leb_uint32(frame_ip, frame_ip_end, flags);
read_leb_uint32(frame_ip, frame_ip_end, offset);
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
sval = (uint64)POP_I64();
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_I64_STORE8) {
CHECK_MEMORY_OVERFLOW(1);
@ -4388,7 +4524,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
{
uint32 reserved;
read_leb_uint32(frame_ip, frame_ip_end, reserved);
PUSH_I32(memory->cur_page_count);
PUSH_PAGE_COUNT(memory->cur_page_count);
(void)reserved;
HANDLE_OP_END();
}
@ -4399,15 +4535,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
prev_page_count = memory->cur_page_count;
read_leb_uint32(frame_ip, frame_ip_end, reserved);
delta = (uint32)POP_I32();
delta = (uint32)POP_PAGE_COUNT();
if (!wasm_enlarge_memory(module, delta)) {
/* failed to memory.grow, return -1 */
PUSH_I32(-1);
PUSH_PAGE_COUNT(-1);
}
else {
/* success, return previous page count */
PUSH_I32(prev_page_count);
PUSH_PAGE_COUNT(prev_page_count);
/* update memory size, no need to update memory ptr as
it isn't changed in wasm_enlarge_memory */
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
@ -5407,7 +5543,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_BULK_MEMORY != 0
case WASM_OP_MEMORY_INIT:
{
uint32 addr, segment;
uint32 segment;
mem_offset_t addr;
uint64 bytes, offset, seg_len;
uint8 *data;
@ -5417,7 +5554,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
bytes = (uint64)(uint32)POP_I32();
offset = (uint64)(uint32)POP_I32();
addr = (uint32)POP_I32();
addr = (mem_offset_t)POP_MEM_OFFSET();
#if WASM_ENABLE_THREAD_MGR != 0
linear_mem_size = get_linear_mem_size();
@ -5460,14 +5597,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
case WASM_OP_MEMORY_COPY:
{
uint32 dst, src, len;
mem_offset_t dst, src, len;
uint8 *mdst, *msrc;
frame_ip += 2;
len = POP_I32();
src = POP_I32();
dst = POP_I32();
len = POP_MEM_OFFSET();
src = POP_MEM_OFFSET();
dst = POP_MEM_OFFSET();
#if WASM_ENABLE_THREAD_MGR != 0
linear_mem_size = get_linear_mem_size();
@ -5493,13 +5629,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
}
case WASM_OP_MEMORY_FILL:
{
uint32 dst, len;
mem_offset_t dst, len;
uint8 fill_val, *mdst;
frame_ip++;
len = POP_I32();
len = POP_MEM_OFFSET();
fill_val = POP_I32();
dst = POP_I32();
dst = POP_MEM_OFFSET();
#if WASM_ENABLE_THREAD_MGR != 0
linear_mem_size = get_linear_mem_size();
@ -5729,7 +5865,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_SHARED_MEMORY != 0
HANDLE_OP(WASM_OP_ATOMIC_PREFIX)
{
uint32 offset = 0, align = 0, addr;
mem_offset_t offset = 0, addr;
uint32 align = 0;
uint32 opcode1;
read_leb_uint32(frame_ip, frame_ip_end, opcode1);
@ -5739,7 +5876,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (opcode != WASM_OP_ATOMIC_FENCE) {
read_leb_uint32(frame_ip, frame_ip_end, align);
read_leb_uint32(frame_ip, frame_ip_end, offset);
read_leb_mem_offset(frame_ip, frame_ip_end, offset);
}
switch (opcode) {
@ -5748,7 +5885,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 notify_count, ret;
notify_count = POP_I32();
addr = POP_I32();
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(4);
CHECK_ATOMIC_MEMORY_ACCESS();
@ -5768,7 +5905,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
timeout = POP_I64();
expect = POP_I32();
addr = POP_I32();
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(4);
CHECK_ATOMIC_MEMORY_ACCESS();
@ -5792,7 +5929,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
timeout = POP_I64();
expect = POP_I64();
addr = POP_I32();
addr = POP_MEM_OFFSET();
CHECK_MEMORY_OVERFLOW(8);
CHECK_ATOMIC_MEMORY_ACCESS();
@ -5823,7 +5960,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
{
uint32 readv;
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
CHECK_MEMORY_OVERFLOW(1);
@ -5858,7 +5995,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
{
uint64 readv;
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
CHECK_MEMORY_OVERFLOW(1);
@ -5900,7 +6037,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint32 sval;
sval = (uint32)POP_I32();
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
CHECK_MEMORY_OVERFLOW(1);
@ -5934,7 +6071,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
uint64 sval;
sval = (uint64)POP_I64();
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
CHECK_MEMORY_OVERFLOW(1);
@ -5961,7 +6098,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_MEMORY_OVERFLOW(8);
CHECK_ATOMIC_MEMORY_ACCESS();
shared_memory_lock(memory);
PUT_I64_TO_ADDR((uint32 *)maddr, sval);
STORE_I64(maddr, sval);
shared_memory_unlock(memory);
}
break;
@ -5975,7 +6112,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
sval = POP_I32();
expect = POP_I32();
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) {
CHECK_MEMORY_OVERFLOW(1);
@ -6021,7 +6158,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
sval = (uint64)POP_I64();
expect = (uint64)POP_I64();
addr = POP_I32();
addr = POP_MEM_OFFSET();
if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) {
CHECK_MEMORY_OVERFLOW(1);

302
core/iwasm/interpreter/wasm_loader.c
View File

@ -45,6 +45,16 @@ set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
}
}
#if WASM_ENABLE_MEMORY64 != 0
static void
set_error_buf_mem_offset_out_of_range(char *error_buf, uint32 error_buf_size)
{
if (error_buf != NULL) {
snprintf(error_buf, error_buf_size, "offset out of range");
}
}
#endif
static void
set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...)
{
@ -102,6 +112,7 @@ check_buf1(const uint8 *buf, const uint8 *buf_end, uint32 length,
#define skip_leb_int64(p, p_end) skip_leb(p)
#define skip_leb_uint32(p, p_end) skip_leb(p)
#define skip_leb_int32(p, p_end) skip_leb(p)
#define skip_leb_mem_offset(p, p_end) skip_leb(p)
static bool
read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
@ -139,7 +150,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
}
else if (sign && maxbits == 32) {
if (shift < maxbits) {
/* Sign extend, second highest bit is the sign bit */
/* Sign extend, second-highest bit is the sign bit */
if ((uint8)byte & 0x40)
result |= (~((uint64)0)) << shift;
}
@ -154,7 +165,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
}
else if (sign && maxbits == 64) {
if (shift < maxbits) {
/* Sign extend, second highest bit is the sign bit */
/* Sign extend, second-highest bit is the sign bit */
if ((uint8)byte & 0x40)
result |= (~((uint64)0)) << shift;
}
@ -191,6 +202,21 @@ fail:
res = (int64)res64; \
} while (0)
#if WASM_ENABLE_MEMORY64 != 0
#define read_leb_mem_offset(p, p_end, res) \
do { \
uint64 res64; \
if (!read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, \
&res64, error_buf, error_buf_size)) { \
set_error_buf_mem_offset_out_of_range(error_buf, error_buf_size); \
goto fail; \
} \
res = (mem_offset_t)res64; \
} while (0)
#else
#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res)
#endif
#define read_leb_uint32(p, p_end, res) \
do { \
uint64 res64; \
@ -2582,31 +2608,92 @@ fail:
}
static bool
check_memory_init_size(uint32 init_size, char *error_buf, uint32 error_buf_size)
check_memory_init_size(bool is_memory64, uint32 init_size, char *error_buf,
uint32 error_buf_size)
{
if (init_size > DEFAULT_MAX_PAGES) {
uint32 default_max_size =
is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
if (!is_memory64 && init_size > default_max_size) {
set_error_buf(error_buf, error_buf_size,
"memory size must be at most 65536 pages (4GiB)");
return false;
}
#if WASM_ENABLE_MEMORY64 != 0
else if (is_memory64 && init_size > default_max_size) {
set_error_buf(
error_buf, error_buf_size,
"memory size must be at most 4,294,967,295 pages (274 Terabyte)");
return false;
}
#endif
return true;
}
static bool
check_memory_max_size(uint32 init_size, uint32 max_size, char *error_buf,
uint32 error_buf_size)
check_memory_max_size(bool is_memory64, uint32 init_size, uint32 max_size,
char *error_buf, uint32 error_buf_size)
{
uint32 default_max_size =
is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
if (max_size < init_size) {
set_error_buf(error_buf, error_buf_size,
"size minimum must not be greater than maximum");
return false;
}
if (max_size > DEFAULT_MAX_PAGES) {
if (!is_memory64 && max_size > default_max_size) {
set_error_buf(error_buf, error_buf_size,
"memory size must be at most 65536 pages (4GiB)");
return false;
}
#if WASM_ENABLE_MEMORY64 != 0
else if (is_memory64 && max_size > default_max_size) {
set_error_buf(
error_buf, error_buf_size,
"memory size must be at most 4,294,967,295 pages (274 Terabyte)");
return false;
}
#endif
return true;
}
static bool
check_memory_flag(const uint8 mem_flag, char *error_buf, uint32 error_buf_size)
{
/* Check whether certain features indicated by mem_flag are enabled in
* runtime */
if (mem_flag > MAX_PAGE_COUNT_FLAG) {
#if WASM_ENABLE_SHARED_MEMORY == 0
if (mem_flag & SHARED_MEMORY_FLAG) {
LOG_VERBOSE("shared memory flag was found, please enable shared "
"memory, lib-pthread or lib-wasi-threads");
set_error_buf(error_buf, error_buf_size, "invalid limits flags");
return false;
}
#endif
#if WASM_ENABLE_MEMORY64 == 0
if (mem_flag & MEMORY64_FLAG) {
LOG_VERBOSE("memory64 flag was found, please enable memory64");
set_error_buf(error_buf, error_buf_size, "invalid limits flags");
return false;
}
#endif
}
if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) {
set_error_buf(error_buf, error_buf_size, "invalid limits flags");
return false;
}
else if ((mem_flag & SHARED_MEMORY_FLAG)
&& !(mem_flag & MAX_PAGE_COUNT_FLAG)) {
set_error_buf(error_buf, error_buf_size,
"shared memory must have maximum");
return false;
}
return true;
}
@ -2616,15 +2703,16 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
const char *memory_name, WASMMemoryImport *memory,
char *error_buf, uint32 error_buf_size)
{
const uint8 *p = *p_buf, *p_end = buf_end;
const uint8 *p = *p_buf, *p_end = buf_end, *p_org;
#if WASM_ENABLE_APP_FRAMEWORK != 0
uint32 pool_size = wasm_runtime_memory_pool_size();
uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
/ DEFAULT_NUM_BYTES_PER_PAGE;
#else
uint32 max_page_count = DEFAULT_MAX_PAGES;
uint32 max_page_count;
#endif /* WASM_ENABLE_APP_FRAMEWORK */
uint32 declare_max_page_count_flag = 0;
uint32 mem_flag = 0;
bool is_memory64 = false;
uint32 declare_init_page_count = 0;
uint32 declare_max_page_count = 0;
#if WASM_ENABLE_MULTI_MODULE != 0
@ -2632,16 +2720,31 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
WASMMemory *linked_memory = NULL;
#endif
read_leb_uint32(p, p_end, declare_max_page_count_flag);
p_org = p;
read_leb_uint32(p, p_end, mem_flag);
is_memory64 = mem_flag & MEMORY64_FLAG;
if (p - p_org > 1) {
LOG_VERBOSE("integer representation too long");
set_error_buf(error_buf, error_buf_size, "invalid limits flags");
return false;
}
if (!check_memory_flag(mem_flag, error_buf, error_buf_size)) {
return false;
}
read_leb_uint32(p, p_end, declare_init_page_count);
if (!check_memory_init_size(declare_init_page_count, error_buf,
if (!check_memory_init_size(is_memory64, declare_init_page_count, error_buf,
error_buf_size)) {
return false;
}
if (declare_max_page_count_flag & 1) {
#if WASM_ENABLE_APP_FRAMEWORK == 0
max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
#endif
if (mem_flag & MAX_PAGE_COUNT_FLAG) {
read_leb_uint32(p, p_end, declare_max_page_count);
if (!check_memory_max_size(declare_init_page_count,
if (!check_memory_max_size(is_memory64, declare_init_page_count,
declare_max_page_count, error_buf,
error_buf_size)) {
return false;
@ -2664,7 +2767,7 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
#if WASM_ENABLE_LIB_WASI_THREADS != 0
/* Avoid memory import failure when wasi-threads is enabled
and the memory is shared */
if (!(declare_max_page_count_flag & 2))
if (!(mem_flag & SHARED_MEMORY_FLAG))
return false;
#else
return false;
@ -2712,7 +2815,7 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
}
/* now we believe all declaration are ok */
memory->flags = declare_max_page_count_flag;
memory->flags = mem_flag;
memory->init_page_count = declare_init_page_count;
memory->max_page_count = declare_max_page_count;
memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE;
@ -3013,53 +3116,34 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
/ DEFAULT_NUM_BYTES_PER_PAGE;
#else
uint32 max_page_count = DEFAULT_MAX_PAGES;
uint32 max_page_count;
#endif
bool is_memory64 = false;
p_org = p;
read_leb_uint32(p, p_end, memory->flags);
#if WASM_ENABLE_SHARED_MEMORY == 0
if (p - p_org > 1) {
set_error_buf(error_buf, error_buf_size,
"integer representation too long");
return false;
}
if (memory->flags > 1) {
if (memory->flags & 2) {
set_error_buf(error_buf, error_buf_size,
"shared memory flag was found, "
"please enable shared memory, lib-pthread "
"or lib-wasi-threads");
}
else {
set_error_buf(error_buf, error_buf_size, "invalid memory flags");
}
return false;
}
#else
is_memory64 = memory->flags & MEMORY64_FLAG;
if (p - p_org > 1) {
LOG_VERBOSE("integer representation too long");
set_error_buf(error_buf, error_buf_size, "invalid limits flags");
return false;
}
if (memory->flags > 3) {
set_error_buf(error_buf, error_buf_size, "invalid limits flags");
if (!check_memory_flag(memory->flags, error_buf, error_buf_size)) {
return false;
}
else if (memory->flags == 2) {
set_error_buf(error_buf, error_buf_size,
"shared memory must have maximum");
return false;
}
#endif
read_leb_uint32(p, p_end, memory->init_page_count);
if (!check_memory_init_size(memory->init_page_count, error_buf,
if (!check_memory_init_size(is_memory64, memory->init_page_count, error_buf,
error_buf_size))
return false;
#if WASM_ENABLE_APP_FRAMEWORK == 0
max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
#endif
if (memory->flags & 1) {
read_leb_uint32(p, p_end, memory->max_page_count);
if (!check_memory_max_size(memory->init_page_count,
if (!check_memory_max_size(is_memory64, memory->init_page_count,
memory->max_page_count, error_buf,
error_buf_size))
return false;
@ -4450,6 +4534,7 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
bool is_passive = false;
uint32 mem_flag;
#endif
uint8 mem_offset_type;
read_leb_uint32(p, p_end, data_seg_count);
@ -4515,11 +4600,35 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
}
#endif /* WASM_ENABLE_BULK_MEMORY */
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif
{
#if WASM_ENABLE_MEMORY64 != 0
/* This memory_flag is from memory instead of data segment */
uint8 memory_flag;
if (module->import_memory_count > 0) {
memory_flag =
module->import_memories[mem_index].u.memory.flags;
}
else {
memory_flag =
module
->memories[mem_index - module->import_memory_count]
.flags;
}
mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64
: VALUE_TYPE_I32;
#else
mem_offset_type = VALUE_TYPE_I32;
#endif
}
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif
if (!load_init_expr(module, &p, p_end, &init_expr,
VALUE_TYPE_I32, NULL, error_buf,
mem_offset_type, NULL, error_buf,
error_buf_size))
return false;
@ -6979,8 +7088,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
case WASM_OP_I64_STORE8:
case WASM_OP_I64_STORE16:
case WASM_OP_I64_STORE32:
skip_leb_uint32(p, p_end); /* align */
skip_leb_uint32(p, p_end); /* offset */
skip_leb_uint32(p, p_end); /* align */
skip_leb_mem_offset(p, p_end); /* offset */
break;
case WASM_OP_MEMORY_SIZE:
@ -7326,6 +7435,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0)
case WASM_OP_SIMD_PREFIX:
{
/* TODO: memory64 offset type changes */
uint32 opcode1;
read_leb_uint32(p, p_end, opcode1);
@ -7422,6 +7532,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
#if WASM_ENABLE_SHARED_MEMORY != 0
case WASM_OP_ATOMIC_PREFIX:
{
/* TODO: memory64 offset type changes */
uint32 opcode1;
/* atomic_op (u32_leb) + memarg (2 u32_leb) */
@ -7431,8 +7542,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
opcode = (uint8)opcode1;
if (opcode != WASM_OP_ATOMIC_FENCE) {
skip_leb_uint32(p, p_end); /* align */
skip_leb_uint32(p, p_end); /* offset */
skip_leb_uint32(p, p_end); /* align */
skip_leb_mem_offset(p, p_end); /* offset */
}
else {
/* atomic.fence doesn't have memarg */
@ -9334,6 +9445,8 @@ fail:
#define PUSH_EXTERNREF() TEMPLATE_PUSH(EXTERNREF)
#define PUSH_REF(Type) TEMPLATE_PUSH_REF(Type)
#define POP_REF(Type) TEMPLATE_POP_REF(Type)
#define PUSH_MEM_OFFSET() TEMPLATE_PUSH_REF(mem_offset_type)
#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET()
#define POP_I32() TEMPLATE_POP(I32)
#define POP_F32() TEMPLATE_POP(F32)
@ -9343,6 +9456,7 @@ fail:
#define POP_FUNCREF() TEMPLATE_POP(FUNCREF)
#define POP_EXTERNREF() TEMPLATE_POP(EXTERNREF)
#define POP_STRINGREF() TEMPLATE_POP(STRINGREF)
#define POP_MEM_OFFSET() TEMPLATE_POP_REF(mem_offset_type)
#if WASM_ENABLE_FAST_INTERP != 0
@ -10510,11 +10624,12 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
{
uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org;
uint32 param_count, local_count, global_count;
uint8 *param_types, *local_types, local_type, global_type;
uint8 *param_types, *local_types, local_type, global_type, mem_offset_type;
BlockType func_block_type;
uint16 *local_offsets, local_offset;
uint32 type_idx, func_idx, local_idx, global_idx, table_idx;
uint32 table_seg_idx, data_seg_idx, count, align, mem_offset, i;
uint32 table_seg_idx, data_seg_idx, count, align, i;
mem_offset_t mem_offset;
int32 i32_const = 0;
int64 i64_const;
uint8 opcode;
@ -10539,6 +10654,19 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n",
func->param_cell_num, func->local_cell_num, func->ret_cell_num);
#endif
#if WASM_ENABLE_MEMORY64 != 0
bool is_memory64 = false;
/* TODO: multi-memories for now assuming the memory idx type is consistent
* across multi-memories */
if (module->import_memory_count > 0)
is_memory64 = module->import_memories[0].u.memory.flags & MEMORY64_FLAG;
else if (module->memory_count > 0)
is_memory64 = module->memories[0].flags & MEMORY64_FLAG;
mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32;
#else
mem_offset_type = VALUE_TYPE_I32;
#endif
global_count = module->import_global_count + module->global_count;
@ -12730,8 +12858,8 @@ re_scan:
}
#endif
CHECK_MEMORY();
read_leb_uint32(p, p_end, align); /* align */
read_leb_uint32(p, p_end, mem_offset); /* offset */
read_leb_uint32(p, p_end, align); /* align */
read_leb_mem_offset(p, p_end, mem_offset); /* offset */
if (!check_memory_access_align(opcode, align, error_buf,
error_buf_size)) {
goto fail;
@ -12749,7 +12877,7 @@ re_scan:
case WASM_OP_I32_LOAD8_U:
case WASM_OP_I32_LOAD16_S:
case WASM_OP_I32_LOAD16_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32);
break;
case WASM_OP_I64_LOAD:
case WASM_OP_I64_LOAD8_S:
@ -12758,35 +12886,35 @@ re_scan:
case WASM_OP_I64_LOAD16_U:
case WASM_OP_I64_LOAD32_S:
case WASM_OP_I64_LOAD32_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64);
break;
case WASM_OP_F32_LOAD:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32);
break;
case WASM_OP_F64_LOAD:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64);
break;
/* store */
case WASM_OP_I32_STORE:
case WASM_OP_I32_STORE8:
case WASM_OP_I32_STORE16:
POP_I32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_I64_STORE:
case WASM_OP_I64_STORE8:
case WASM_OP_I64_STORE16:
case WASM_OP_I64_STORE32:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_F32_STORE:
POP_F32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_F64_STORE:
POP_F64();
POP_I32();
POP_MEM_OFFSET();
break;
default:
break;
@ -12802,7 +12930,7 @@ re_scan:
"zero byte expected");
goto fail;
}
PUSH_I32();
PUSH_PAGE_COUNT();
module->possible_memory_grow = true;
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
@ -12818,7 +12946,7 @@ re_scan:
"zero byte expected");
goto fail;
}
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, mem_offset_type);
module->possible_memory_grow = true;
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
@ -14179,7 +14307,7 @@ re_scan:
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -14222,9 +14350,9 @@ re_scan:
&& module->memory_count == 0)
goto fail_unknown_memory;
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
POP_MEM_OFFSET();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -14242,10 +14370,9 @@ re_scan:
&& module->memory_count == 0) {
goto fail_unknown_memory;
}
POP_I32();
POP_I32();
POP_MEM_OFFSET();
POP_I32();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -14491,6 +14618,7 @@ re_scan:
#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0)
case WASM_OP_SIMD_PREFIX:
{
/* TODO: memory64 offset type changes */
uint32 opcode1;
#if WASM_ENABLE_WAMR_COMPILER != 0
@ -15167,8 +15295,8 @@ re_scan:
#endif
if (opcode1 != WASM_OP_ATOMIC_FENCE) {
CHECK_MEMORY();
read_leb_uint32(p, p_end, align); /* align */
read_leb_uint32(p, p_end, mem_offset); /* offset */
read_leb_uint32(p, p_end, align); /* align */
read_leb_mem_offset(p, p_end, mem_offset); /* offset */
if (!check_memory_align_equal(opcode1, align, error_buf,
error_buf_size)) {
goto fail;
@ -15182,18 +15310,20 @@ re_scan:
#endif
switch (opcode1) {
case WASM_OP_ATOMIC_NOTIFY:
POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_WAIT32:
POP_I64();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_WAIT64:
POP_I64();
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_FENCE:
@ -15207,26 +15337,26 @@ re_scan:
case WASM_OP_ATOMIC_I32_LOAD:
case WASM_OP_ATOMIC_I32_LOAD8_U:
case WASM_OP_ATOMIC_I32_LOAD16_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32);
break;
case WASM_OP_ATOMIC_I32_STORE:
case WASM_OP_ATOMIC_I32_STORE8:
case WASM_OP_ATOMIC_I32_STORE16:
POP_I32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_ATOMIC_I64_LOAD:
case WASM_OP_ATOMIC_I64_LOAD8_U:
case WASM_OP_ATOMIC_I64_LOAD16_U:
case WASM_OP_ATOMIC_I64_LOAD32_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64);
break;
case WASM_OP_ATOMIC_I64_STORE:
case WASM_OP_ATOMIC_I64_STORE8:
case WASM_OP_ATOMIC_I64_STORE16:
case WASM_OP_ATOMIC_I64_STORE32:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_ATOMIC_RMW_I32_ADD:
case WASM_OP_ATOMIC_RMW_I32_ADD8_U:
@ -15246,7 +15376,9 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I32_XCHG:
case WASM_OP_ATOMIC_RMW_I32_XCHG8_U:
case WASM_OP_ATOMIC_RMW_I32_XCHG16_U:
POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_RMW_I64_ADD:
case WASM_OP_ATOMIC_RMW_I64_ADD8_U:
@ -15273,7 +15405,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I64_XCHG16_U:
case WASM_OP_ATOMIC_RMW_I64_XCHG32_U:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I64();
break;
case WASM_OP_ATOMIC_RMW_I32_CMPXCHG:
@ -15281,7 +15413,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U:
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_RMW_I64_CMPXCHG:
@ -15290,7 +15422,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U:
POP_I64();
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I64();
break;
default:

232
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -47,6 +47,7 @@ set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
#define skip_leb_int64(p, p_end) skip_leb(p)
#define skip_leb_uint32(p, p_end) skip_leb(p)
#define skip_leb_int32(p, p_end) skip_leb(p)
#define skip_leb_mem_offset(p, p_end) skip_leb(p)
static bool
is_32bit_type(uint8 type)
@ -116,7 +117,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
}
else if (sign && maxbits == 32) {
if (shift < maxbits) {
/* Sign extend, second highest bit is the sign bit */
/* Sign extend, second-highest bit is the sign bit */
if ((uint8)byte & 0x40)
result |= (~((uint64)0)) << shift;
}
@ -132,7 +133,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
}
else if (sign && maxbits == 64) {
if (shift < maxbits) {
/* Sign extend, second highest bit is the sign bit */
/* Sign extend, second-highest bit is the sign bit */
if ((uint8)byte & 0x40)
result |= (~((uint64)0)) << shift;
}
@ -180,6 +181,18 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
res = (int32)res64; \
} while (0)
#if WASM_ENABLE_MEMORY64 != 0
#define read_leb_mem_offset(p, p_end, res) \
do { \
uint64 res64; \
read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, &res64, \
error_buf, error_buf_size); \
res = (mem_offset_t)res64; \
} while (0)
#else
#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res)
#endif
static void *
loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
{
@ -683,6 +696,38 @@ load_table_import(const uint8 **p_buf, const uint8 *buf_end,
return true;
}
static bool
check_memory_flag(const uint8 mem_flag)
{
/* Check whether certain features indicated by mem_flag are enabled in
* runtime */
if (mem_flag > MAX_PAGE_COUNT_FLAG) {
#if WASM_ENABLE_SHARED_MEMORY == 0
if (mem_flag & SHARED_MEMORY_FLAG) {
LOG_VERBOSE("shared memory flag was found, please enable shared "
"memory, lib-pthread or lib-wasi-threads");
return false;
}
#endif
#if WASM_ENABLE_MEMORY64 == 0
if (mem_flag & MEMORY64_FLAG) {
LOG_VERBOSE("memory64 flag was found, please enable memory64");
return false;
}
#endif
}
if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) {
return false;
}
else if ((mem_flag & SHARED_MEMORY_FLAG)
&& !(mem_flag & MAX_PAGE_COUNT_FLAG)) {
return false;
}
return true;
}
static bool
load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
WASMModule *parent_module, const char *sub_module_name,
@ -695,20 +740,28 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
/ DEFAULT_NUM_BYTES_PER_PAGE;
#else
uint32 max_page_count = DEFAULT_MAX_PAGES;
uint32 max_page_count;
#endif /* WASM_ENABLE_APP_FRAMEWORK */
uint32 declare_max_page_count_flag = 0;
uint32 mem_flag = 0;
bool is_memory64 = false;
uint32 declare_init_page_count = 0;
uint32 declare_max_page_count = 0;
read_leb_uint32(p, p_end, declare_max_page_count_flag);
read_leb_uint32(p, p_end, declare_init_page_count);
bh_assert(declare_init_page_count <= 65536);
read_leb_uint32(p, p_end, mem_flag);
bh_assert(check_memory_flag(mem_flag));
if (declare_max_page_count_flag & 1) {
#if WASM_ENABLE_APP_FRAMEWORK == 0
is_memory64 = mem_flag & MEMORY64_FLAG;
max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
#endif
read_leb_uint32(p, p_end, declare_init_page_count);
bh_assert(declare_init_page_count <= max_page_count);
if (mem_flag & MAX_PAGE_COUNT_FLAG) {
read_leb_uint32(p, p_end, declare_max_page_count);
bh_assert(declare_init_page_count <= declare_max_page_count);
bh_assert(declare_max_page_count <= 65536);
bh_assert(declare_max_page_count <= max_page_count);
if (declare_max_page_count > max_page_count) {
declare_max_page_count = max_page_count;
}
@ -719,12 +772,13 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
}
/* now we believe all declaration are ok */
memory->flags = declare_max_page_count_flag;
memory->flags = mem_flag;
memory->init_page_count = declare_init_page_count;
memory->max_page_count = declare_max_page_count;
memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE;
*p_buf = p;
(void)check_memory_flag;
return true;
}
@ -811,26 +865,28 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
/ DEFAULT_NUM_BYTES_PER_PAGE;
#else
uint32 max_page_count = DEFAULT_MAX_PAGES;
uint32 max_page_count;
bool is_memory64 = false;
#endif
p_org = p;
read_leb_uint32(p, p_end, memory->flags);
bh_assert(p - p_org <= 1);
(void)p_org;
#if WASM_ENABLE_SHARED_MEMORY == 0
bh_assert(memory->flags <= 1);
#else
bh_assert(memory->flags <= 3 && memory->flags != 2);
bh_assert(check_memory_flag(memory->flags));
#if WASM_ENABLE_APP_FRAMEWORK == 0
is_memory64 = memory->flags & MEMORY64_FLAG;
max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
#endif
read_leb_uint32(p, p_end, memory->init_page_count);
bh_assert(memory->init_page_count <= 65536);
bh_assert(memory->init_page_count <= max_page_count);
if (memory->flags & 1) {
read_leb_uint32(p, p_end, memory->max_page_count);
bh_assert(memory->init_page_count <= memory->max_page_count);
bh_assert(memory->max_page_count <= 65536);
bh_assert(memory->max_page_count <= max_page_count);
if (memory->max_page_count > max_page_count)
memory->max_page_count = max_page_count;
}
@ -842,6 +898,7 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE;
*p_buf = p;
(void)check_memory_flag;
return true;
}
@ -1704,6 +1761,7 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
bool is_passive = false;
uint32 mem_flag;
#endif
uint8 mem_offset_type;
read_leb_uint32(p, p_end, data_seg_count);
@ -1750,11 +1808,35 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
< module->import_memory_count + module->memory_count);
#endif /* WASM_ENABLE_BULK_MEMORY */
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif /* WASM_ENABLE_BULK_MEMORY */
{
#if WASM_ENABLE_MEMORY64 != 0
/* This memory_flag is from memory instead of data segment */
uint8 memory_flag;
if (module->import_memory_count > 0) {
memory_flag =
module->import_memories[mem_index].u.memory.flags;
}
else {
memory_flag =
module
->memories[mem_index - module->import_memory_count]
.flags;
}
mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64
: VALUE_TYPE_I32;
#else
mem_offset_type = VALUE_TYPE_I32;
#endif /* WASM_ENABLE_MEMORY64 */
}
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif
if (!load_init_expr(module, &p, p_end, &init_expr,
VALUE_TYPE_I32, error_buf, error_buf_size))
mem_offset_type, error_buf, error_buf_size))
return false;
read_leb_uint32(p, p_end, data_seg_len);
@ -3532,8 +3614,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
case WASM_OP_I64_STORE8:
case WASM_OP_I64_STORE16:
case WASM_OP_I64_STORE32:
skip_leb_uint32(p, p_end); /* align */
skip_leb_uint32(p, p_end); /* offset */
skip_leb_uint32(p, p_end); /* align */
skip_leb_mem_offset(p, p_end); /* offset */
break;
case WASM_OP_MEMORY_SIZE:
@ -3748,6 +3830,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
#if WASM_ENABLE_SHARED_MEMORY != 0
case WASM_OP_ATOMIC_PREFIX:
{
/* TODO: memory64 offset type changes */
uint32 opcode1;
/* atomic_op (u32_leb) + memarg (2 u32_leb) */
@ -3757,8 +3840,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
opcode = (uint8)opcode1;
if (opcode != WASM_OP_ATOMIC_FENCE) {
skip_leb_uint32(p, p_end); /* align */
skip_leb_uint32(p, p_end); /* offset */
skip_leb_uint32(p, p_end); /* align */
skip_leb_mem_offset(p, p_end); /* offset */
}
else {
/* atomic.fence doesn't have memarg */
@ -5075,6 +5158,11 @@ fail:
goto fail; \
} while (0)
#define PUSH_MEM_OFFSET() PUSH_OFFSET_TYPE(mem_offset_type)
#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET()
#define POP_MEM_OFFSET() POP_OFFSET_TYPE(mem_offset_type)
#define POP_AND_PUSH(type_pop, type_push) \
do { \
if (!(wasm_loader_push_pop_frame_ref_offset( \
@ -5129,6 +5217,15 @@ fail:
goto fail; \
} while (0)
#define PUSH_MEM_OFFSET() \
do { \
if (!(wasm_loader_push_frame_ref(loader_ctx, mem_offset_type, \
error_buf, error_buf_size))) \
goto fail; \
} while (0)
#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET()
#define POP_I32() \
do { \
if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_I32, error_buf, \
@ -5164,6 +5261,13 @@ fail:
goto fail; \
} while (0)
#define POP_MEM_OFFSET() \
do { \
if (!(wasm_loader_pop_frame_ref(loader_ctx, mem_offset_type, \
error_buf, error_buf_size))) \
goto fail; \
} while (0)
#define POP_AND_PUSH(type_pop, type_push) \
do { \
if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 1, type_push, \
@ -5774,10 +5878,11 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
{
uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org;
uint32 param_count, local_count, global_count;
uint8 *param_types, *local_types, local_type, global_type;
uint8 *param_types, *local_types, local_type, global_type, mem_offset_type;
BlockType func_block_type;
uint16 *local_offsets, local_offset;
uint32 count, local_idx, global_idx, u32, align, mem_offset, i;
uint32 count, local_idx, global_idx, u32, align, i;
mem_offset_t mem_offset;
int32 i32, i32_const = 0;
int64 i64_const;
uint8 opcode, u8;
@ -5799,6 +5904,19 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n",
func->param_cell_num, func->local_cell_num, func->ret_cell_num);
#endif
#if WASM_ENABLE_MEMORY64 != 0
bool is_memory64 = false;
/* TODO: multi-memories for now assuming the memory idx type is consistent
* across multi-memories */
if (module->import_memory_count > 0)
is_memory64 = module->import_memories[0].u.memory.flags & MEMORY64_FLAG;
else if (module->memory_count > 0)
is_memory64 = module->memories[0].flags & MEMORY64_FLAG;
mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32;
#else
mem_offset_type = VALUE_TYPE_I32;
#endif
global_count = module->import_global_count + module->global_count;
@ -7107,8 +7225,8 @@ re_scan:
}
#endif
CHECK_MEMORY();
read_leb_uint32(p, p_end, align); /* align */
read_leb_uint32(p, p_end, mem_offset); /* offset */
read_leb_uint32(p, p_end, align); /* align */
read_leb_mem_offset(p, p_end, mem_offset); /* offset */
#if WASM_ENABLE_FAST_INTERP != 0
emit_uint32(loader_ctx, mem_offset);
#endif
@ -7122,7 +7240,7 @@ re_scan:
case WASM_OP_I32_LOAD8_U:
case WASM_OP_I32_LOAD16_S:
case WASM_OP_I32_LOAD16_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32);
break;
case WASM_OP_I64_LOAD:
case WASM_OP_I64_LOAD8_S:
@ -7131,35 +7249,35 @@ re_scan:
case WASM_OP_I64_LOAD16_U:
case WASM_OP_I64_LOAD32_S:
case WASM_OP_I64_LOAD32_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64);
break;
case WASM_OP_F32_LOAD:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32);
break;
case WASM_OP_F64_LOAD:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64);
break;
/* store */
case WASM_OP_I32_STORE:
case WASM_OP_I32_STORE8:
case WASM_OP_I32_STORE16:
POP_I32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_I64_STORE:
case WASM_OP_I64_STORE8:
case WASM_OP_I64_STORE16:
case WASM_OP_I64_STORE32:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_F32_STORE:
POP_F32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_F64_STORE:
POP_F64();
POP_I32();
POP_MEM_OFFSET();
break;
default:
break;
@ -7172,7 +7290,7 @@ re_scan:
/* reserved byte 0x00 */
bh_assert(*p == 0x00);
p++;
PUSH_I32();
PUSH_PAGE_COUNT();
module->possible_memory_grow = true;
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
@ -7185,7 +7303,7 @@ re_scan:
/* reserved byte 0x00 */
bh_assert(*p == 0x00);
p++;
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, mem_offset_type);
module->possible_memory_grow = true;
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
@ -7536,7 +7654,7 @@ re_scan:
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -7565,9 +7683,9 @@ re_scan:
+ module->memory_count
> 0);
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
POP_MEM_OFFSET();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -7582,9 +7700,9 @@ re_scan:
+ module->memory_count
> 0);
POP_MEM_OFFSET();
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -7748,8 +7866,8 @@ re_scan:
#endif
if (opcode1 != WASM_OP_ATOMIC_FENCE) {
CHECK_MEMORY();
read_leb_uint32(p, p_end, align); /* align */
read_leb_uint32(p, p_end, mem_offset); /* offset */
read_leb_uint32(p, p_end, align); /* align */
read_leb_mem_offset(p, p_end, mem_offset); /* offset */
#if WASM_ENABLE_FAST_INTERP != 0
emit_uint32(loader_ctx, mem_offset);
#endif
@ -7759,18 +7877,20 @@ re_scan:
#endif
switch (opcode1) {
case WASM_OP_ATOMIC_NOTIFY:
POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_WAIT32:
POP_I64();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_WAIT64:
POP_I64();
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_FENCE:
@ -7781,26 +7901,26 @@ re_scan:
case WASM_OP_ATOMIC_I32_LOAD:
case WASM_OP_ATOMIC_I32_LOAD8_U:
case WASM_OP_ATOMIC_I32_LOAD16_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32);
break;
case WASM_OP_ATOMIC_I32_STORE:
case WASM_OP_ATOMIC_I32_STORE8:
case WASM_OP_ATOMIC_I32_STORE16:
POP_I32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_ATOMIC_I64_LOAD:
case WASM_OP_ATOMIC_I64_LOAD8_U:
case WASM_OP_ATOMIC_I64_LOAD16_U:
case WASM_OP_ATOMIC_I64_LOAD32_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64);
break;
case WASM_OP_ATOMIC_I64_STORE:
case WASM_OP_ATOMIC_I64_STORE8:
case WASM_OP_ATOMIC_I64_STORE16:
case WASM_OP_ATOMIC_I64_STORE32:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_ATOMIC_RMW_I32_ADD:
case WASM_OP_ATOMIC_RMW_I32_ADD8_U:
@ -7820,7 +7940,9 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I32_XCHG:
case WASM_OP_ATOMIC_RMW_I32_XCHG8_U:
case WASM_OP_ATOMIC_RMW_I32_XCHG16_U:
POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_RMW_I64_ADD:
case WASM_OP_ATOMIC_RMW_I64_ADD8_U:
@ -7847,7 +7969,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I64_XCHG16_U:
case WASM_OP_ATOMIC_RMW_I64_XCHG32_U:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I64();
break;
case WASM_OP_ATOMIC_RMW_I32_CMPXCHG:
@ -7855,7 +7977,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U:
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_RMW_I64_CMPXCHG:
@ -7864,7 +7986,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U:
POP_I64();
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I64();
break;
default:

85
core/iwasm/interpreter/wasm_runtime.c
View File

@ -162,7 +162,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
char *error_buf, uint32 error_buf_size)
{
WASMModule *module = module_inst->module;
uint32 inc_page_count, global_idx;
uint32 inc_page_count, global_idx, default_max_page;
uint32 bytes_of_last_page, bytes_to_page_end;
uint64 aux_heap_base,
heap_offset = (uint64)num_bytes_per_page * init_page_count;
@ -171,7 +171,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
bool is_shared_memory = false;
#if WASM_ENABLE_SHARED_MEMORY != 0
is_shared_memory = flags & 0x02 ? true : false;
is_shared_memory = flags & SHARED_MEMORY_FLAG ? true : false;
/* shared memory */
if (is_shared_memory && parent != NULL) {
@ -186,6 +186,14 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
(void)flags;
#endif /* end of WASM_ENABLE_SHARED_MEMORY */
#if WASM_ENABLE_MEMORY64 != 0
if (flags & MEMORY64_FLAG) {
memory->is_memory64 = 1;
}
#endif
default_max_page =
memory->is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1
&& module_inst->module->free_function != (uint32)-1) {
/* Disable app heap, use malloc/free function exported
@ -195,7 +203,8 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
/* If initial memory is the largest size allowed, disallowing insert host
* managed heap */
if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) {
if (heap_size > 0
&& heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) {
set_error_buf(error_buf, error_buf_size,
"failed to insert app heap into linear memory, "
"try using `--heap-size=0` option");
@ -253,8 +262,18 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
&& global_idx < module_inst->e->global_count);
global_addr = module_inst->global_data
+ module_inst->e->globals[global_idx].data_offset;
*(uint32 *)global_addr = (uint32)aux_heap_base;
LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base);
#if WASM_ENABLE_MEMORY64 != 0
if (memory->is_memory64) {
/* For memory64, the global value should be i64 */
*(uint64 *)global_addr = aux_heap_base;
}
else
#endif
{
/* For memory32, the global value should be i32 */
*(uint32 *)global_addr = (uint32)aux_heap_base;
}
LOG_VERBOSE("Reset __heap_base global to %lu", aux_heap_base);
}
else {
/* Insert app heap before new page */
@ -267,14 +286,15 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
}
init_page_count += inc_page_count;
max_page_count += inc_page_count;
if (init_page_count > DEFAULT_MAX_PAGES) {
if (init_page_count > default_max_page) {
set_error_buf(error_buf, error_buf_size,
"failed to insert app heap into linear memory, "
"try using `--heap-size=0` option");
return NULL;
}
if (max_page_count > DEFAULT_MAX_PAGES)
max_page_count = DEFAULT_MAX_PAGES;
if (max_page_count > default_max_page)
max_page_count = default_max_page;
}
LOG_VERBOSE("Memory instantiate:");
@ -283,14 +303,16 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size);
max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE);
bh_assert(max_memory_data_size
<= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64));
(void)max_memory_data_size;
bh_assert(memory != NULL);
if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory,
num_bytes_per_page, init_page_count,
max_page_count, &memory_data_size)
memory->is_memory64, num_bytes_per_page,
init_page_count, max_page_count,
&memory_data_size)
!= BHT_OK) {
set_error_buf(error_buf, error_buf_size,
"allocate linear memory failed");
@ -1947,7 +1969,8 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
WASMGlobalInstance *globals = NULL, *global;
WASMTableInstance *first_table;
uint32 global_count, i;
uint32 base_offset, length, extra_info_offset;
uint32 length, extra_info_offset;
mem_offset_t base_offset;
uint32 module_inst_struct_size =
offsetof(WASMModuleInstance, global_table_data.bytes);
uint64 module_inst_mem_inst_size;
@ -2305,10 +2328,12 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
(uint64)memory->num_bytes_per_page * memory->cur_page_count;
bh_assert(memory_data || memory_size == 0);
bh_assert(data_seg->base_offset.init_expr_type
== INIT_EXPR_TYPE_I32_CONST
|| data_seg->base_offset.init_expr_type
== INIT_EXPR_TYPE_GET_GLOBAL);
bh_assert(
data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL
|| (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST
&& !memory->is_memory64)
|| (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I64_CONST
&& memory->is_memory64));
if (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
if (!check_global_init_expr(module,
@ -2319,17 +2344,37 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
if (!globals
|| globals[data_seg->base_offset.u.global_index].type
!= VALUE_TYPE_I32) {
!= (memory->is_memory64 ? VALUE_TYPE_I64
: VALUE_TYPE_I32)) {
set_error_buf(error_buf, error_buf_size,
"data segment does not fit");
goto fail;
}
base_offset =
globals[data_seg->base_offset.u.global_index].initial_value.i32;
#if WASM_ENABLE_MEMORY64 != 0
if (memory->is_memory64) {
base_offset =
(uint64)globals[data_seg->base_offset.u.global_index]
.initial_value.i64;
}
else
#endif
{
base_offset =
(uint32)globals[data_seg->base_offset.u.global_index]
.initial_value.i32;
}
}
else {
base_offset = (uint32)data_seg->base_offset.u.i32;
#if WASM_ENABLE_MEMORY64 != 0
if (memory->is_memory64) {
base_offset = (uint64)data_seg->base_offset.u.i64;
}
else
#endif
{
base_offset = (uint32)data_seg->base_offset.u.i32;
}
}
/* check offset */

2
core/iwasm/interpreter/wasm_runtime.h
View File

@ -103,7 +103,7 @@ struct WASMMemoryInstance {
/* Whether the memory is shared */
uint8 is_shared_memory;
/* TODO: Memory64 whether the memory has 64-bit memory addresses */
/* Whether the memory has 64-bit memory addresses */
uint8 is_memory64;
/* Reference count of the memory instance:

2
core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c
View File

@ -17,7 +17,7 @@ void
wasm_runtime_set_exception(wasm_module_inst_t module, const char *exception);
uint32
wasm_runtime_module_realloc(wasm_module_inst_t module, uint32 ptr, uint32 size,
wasm_runtime_module_realloc(wasm_module_inst_t module, uint64 ptr, uint64 size,
void **p_native_addr);
/* clang-format off */

4
core/shared/platform/common/posix/posix_memmap.c
View File

@ -65,9 +65,11 @@ os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file)
/* integer overflow */
return NULL;
#if WASM_ENABLE_MEMORY64 == 0
if (request_size > 16 * (uint64)UINT32_MAX)
/* at most 16 G is allowed */
/* at most 64 G is allowed */
return NULL;
#endif
if (prot & MMAP_PROT_READ)
map_prot |= PROT_READ;