christoph/wamr
1
Fork 0
Code Releases Activity

Implement memory64 for classic interpreter (#3266)

Browse Source 
Adding a new cmake flag (cache variable) `WAMR_BUILD_MEMORY64` to enable
the memory64 feature, it can only be enabled on the 64-bit platform/target and
can only use software boundary check. And when it is enabled, it can support both
i32 and i64 linear memory types. The main modifications are:

- wasm loader & mini-loader: loading and bytecode validating process 
- wasm runtime: memory instantiating process
- classic-interpreter: wasm code executing process
- Support memory64 memory in related runtime APIs
- Modify main function type check when it's memory64 wasm file
- Modify `wasm_runtime_invoke_native` and `wasm_runtime_invoke_native_raw` to
  handle registered native function pointer argument when memory64 is enabled
- memory64 classic-interpreter spec test in `test_wamr.sh` and in CI

Currently, it supports memory64 memory wasm file that uses core spec
(including bulk memory proposal) opcodes and threads opcodes.

ps.
https://github.com/bytecodealliance/wasm-micro-runtime/issues/3091
https://github.com/bytecodealliance/wasm-micro-runtime/pull/3240
https://github.com/bytecodealliance/wasm-micro-runtime/pull/3260
This commit is contained in:
Wenyong Huang
2024-04-02 15:22:07 +08:00
committed by GitHub
parent 6b0b5de1c5
commit a23fa9f86c
22 changed files with 1084 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -47,6 +47,7 @@ set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
#define skip_leb_int64(p, p_end) skip_leb(p)
#define skip_leb_uint32(p, p_end) skip_leb(p)
#define skip_leb_int32(p, p_end) skip_leb(p)
#define skip_leb_mem_offset(p, p_end) skip_leb(p)
static bool
is_32bit_type(uint8 type)
@ -116,7 +117,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
}
else if (sign && maxbits == 32) {
if (shift < maxbits) {
/* Sign extend, second highest bit is the sign bit */
/* Sign extend, second-highest bit is the sign bit */
if ((uint8)byte & 0x40)
result |= (~((uint64)0)) << shift;
}
@ -132,7 +133,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
}
else if (sign && maxbits == 64) {
if (shift < maxbits) {
/* Sign extend, second highest bit is the sign bit */
/* Sign extend, second-highest bit is the sign bit */
if ((uint8)byte & 0x40)
result |= (~((uint64)0)) << shift;
}
@ -180,6 +181,18 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
res = (int32)res64; \
} while (0)
#if WASM_ENABLE_MEMORY64 != 0
#define read_leb_mem_offset(p, p_end, res) \
do { \
uint64 res64; \
read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, &res64, \
error_buf, error_buf_size); \
res = (mem_offset_t)res64; \
} while (0)
#else
#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res)
#endif
static void *
loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
{
@ -683,6 +696,38 @@ load_table_import(const uint8 **p_buf, const uint8 *buf_end,
return true;
}
static bool
check_memory_flag(const uint8 mem_flag)
{
/* Check whether certain features indicated by mem_flag are enabled in
* runtime */
if (mem_flag > MAX_PAGE_COUNT_FLAG) {
#if WASM_ENABLE_SHARED_MEMORY == 0
if (mem_flag & SHARED_MEMORY_FLAG) {
LOG_VERBOSE("shared memory flag was found, please enable shared "
"memory, lib-pthread or lib-wasi-threads");
return false;
}
#endif
#if WASM_ENABLE_MEMORY64 == 0
if (mem_flag & MEMORY64_FLAG) {
LOG_VERBOSE("memory64 flag was found, please enable memory64");
return false;
}
#endif
}
if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) {
return false;
}
else if ((mem_flag & SHARED_MEMORY_FLAG)
&& !(mem_flag & MAX_PAGE_COUNT_FLAG)) {
return false;
}
return true;
}
static bool
load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
WASMModule *parent_module, const char *sub_module_name,
@ -695,20 +740,28 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
/ DEFAULT_NUM_BYTES_PER_PAGE;
#else
uint32 max_page_count = DEFAULT_MAX_PAGES;
uint32 max_page_count;
#endif /* WASM_ENABLE_APP_FRAMEWORK */
uint32 declare_max_page_count_flag = 0;
uint32 mem_flag = 0;
bool is_memory64 = false;
uint32 declare_init_page_count = 0;
uint32 declare_max_page_count = 0;
read_leb_uint32(p, p_end, declare_max_page_count_flag);
read_leb_uint32(p, p_end, declare_init_page_count);
bh_assert(declare_init_page_count <= 65536);
read_leb_uint32(p, p_end, mem_flag);
bh_assert(check_memory_flag(mem_flag));
if (declare_max_page_count_flag & 1) {
#if WASM_ENABLE_APP_FRAMEWORK == 0
is_memory64 = mem_flag & MEMORY64_FLAG;
max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
#endif
read_leb_uint32(p, p_end, declare_init_page_count);
bh_assert(declare_init_page_count <= max_page_count);
if (mem_flag & MAX_PAGE_COUNT_FLAG) {
read_leb_uint32(p, p_end, declare_max_page_count);
bh_assert(declare_init_page_count <= declare_max_page_count);
bh_assert(declare_max_page_count <= 65536);
bh_assert(declare_max_page_count <= max_page_count);
if (declare_max_page_count > max_page_count) {
declare_max_page_count = max_page_count;
}
@ -719,12 +772,13 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
}
/* now we believe all declaration are ok */
memory->flags = declare_max_page_count_flag;
memory->flags = mem_flag;
memory->init_page_count = declare_init_page_count;
memory->max_page_count = declare_max_page_count;
memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE;
*p_buf = p;
(void)check_memory_flag;
return true;
}
@ -811,26 +865,28 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
/ DEFAULT_NUM_BYTES_PER_PAGE;
#else
uint32 max_page_count = DEFAULT_MAX_PAGES;
uint32 max_page_count;
bool is_memory64 = false;
#endif
p_org = p;
read_leb_uint32(p, p_end, memory->flags);
bh_assert(p - p_org <= 1);
(void)p_org;
#if WASM_ENABLE_SHARED_MEMORY == 0
bh_assert(memory->flags <= 1);
#else
bh_assert(memory->flags <= 3 && memory->flags != 2);
bh_assert(check_memory_flag(memory->flags));
#if WASM_ENABLE_APP_FRAMEWORK == 0
is_memory64 = memory->flags & MEMORY64_FLAG;
max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES;
#endif
read_leb_uint32(p, p_end, memory->init_page_count);
bh_assert(memory->init_page_count <= 65536);
bh_assert(memory->init_page_count <= max_page_count);
if (memory->flags & 1) {
read_leb_uint32(p, p_end, memory->max_page_count);
bh_assert(memory->init_page_count <= memory->max_page_count);
bh_assert(memory->max_page_count <= 65536);
bh_assert(memory->max_page_count <= max_page_count);
if (memory->max_page_count > max_page_count)
memory->max_page_count = max_page_count;
}
@ -842,6 +898,7 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE;
*p_buf = p;
(void)check_memory_flag;
return true;
}
@ -1704,6 +1761,7 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
bool is_passive = false;
uint32 mem_flag;
#endif
uint8 mem_offset_type;
read_leb_uint32(p, p_end, data_seg_count);
@ -1750,11 +1808,35 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
< module->import_memory_count + module->memory_count);
#endif /* WASM_ENABLE_BULK_MEMORY */
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif /* WASM_ENABLE_BULK_MEMORY */
{
#if WASM_ENABLE_MEMORY64 != 0
/* This memory_flag is from memory instead of data segment */
uint8 memory_flag;
if (module->import_memory_count > 0) {
memory_flag =
module->import_memories[mem_index].u.memory.flags;
}
else {
memory_flag =
module
->memories[mem_index - module->import_memory_count]
.flags;
}
mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64
: VALUE_TYPE_I32;
#else
mem_offset_type = VALUE_TYPE_I32;
#endif /* WASM_ENABLE_MEMORY64 */
}
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif
if (!load_init_expr(module, &p, p_end, &init_expr,
VALUE_TYPE_I32, error_buf, error_buf_size))
mem_offset_type, error_buf, error_buf_size))
return false;
read_leb_uint32(p, p_end, data_seg_len);
@ -3532,8 +3614,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
case WASM_OP_I64_STORE8:
case WASM_OP_I64_STORE16:
case WASM_OP_I64_STORE32:
skip_leb_uint32(p, p_end); /* align */
skip_leb_uint32(p, p_end); /* offset */
skip_leb_uint32(p, p_end); /* align */
skip_leb_mem_offset(p, p_end); /* offset */
break;
case WASM_OP_MEMORY_SIZE:
@ -3748,6 +3830,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
#if WASM_ENABLE_SHARED_MEMORY != 0
case WASM_OP_ATOMIC_PREFIX:
{
/* TODO: memory64 offset type changes */
uint32 opcode1;
/* atomic_op (u32_leb) + memarg (2 u32_leb) */
@ -3757,8 +3840,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache,
opcode = (uint8)opcode1;
if (opcode != WASM_OP_ATOMIC_FENCE) {
skip_leb_uint32(p, p_end); /* align */
skip_leb_uint32(p, p_end); /* offset */
skip_leb_uint32(p, p_end); /* align */
skip_leb_mem_offset(p, p_end); /* offset */
}
else {
/* atomic.fence doesn't have memarg */
@ -5075,6 +5158,11 @@ fail:
goto fail; \
} while (0)
#define PUSH_MEM_OFFSET() PUSH_OFFSET_TYPE(mem_offset_type)
#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET()
#define POP_MEM_OFFSET() POP_OFFSET_TYPE(mem_offset_type)
#define POP_AND_PUSH(type_pop, type_push) \
do { \
if (!(wasm_loader_push_pop_frame_ref_offset( \
@ -5129,6 +5217,15 @@ fail:
goto fail; \
} while (0)
#define PUSH_MEM_OFFSET() \
do { \
if (!(wasm_loader_push_frame_ref(loader_ctx, mem_offset_type, \
error_buf, error_buf_size))) \
goto fail; \
} while (0)
#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET()
#define POP_I32() \
do { \
if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_I32, error_buf, \
@ -5164,6 +5261,13 @@ fail:
goto fail; \
} while (0)
#define POP_MEM_OFFSET() \
do { \
if (!(wasm_loader_pop_frame_ref(loader_ctx, mem_offset_type, \
error_buf, error_buf_size))) \
goto fail; \
} while (0)
#define POP_AND_PUSH(type_pop, type_push) \
do { \
if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 1, type_push, \
@ -5774,10 +5878,11 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
{
uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org;
uint32 param_count, local_count, global_count;
uint8 *param_types, *local_types, local_type, global_type;
uint8 *param_types, *local_types, local_type, global_type, mem_offset_type;
BlockType func_block_type;
uint16 *local_offsets, local_offset;
uint32 count, local_idx, global_idx, u32, align, mem_offset, i;
uint32 count, local_idx, global_idx, u32, align, i;
mem_offset_t mem_offset;
int32 i32, i32_const = 0;
int64 i64_const;
uint8 opcode, u8;
@ -5799,6 +5904,19 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func,
LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n",
func->param_cell_num, func->local_cell_num, func->ret_cell_num);
#endif
#if WASM_ENABLE_MEMORY64 != 0
bool is_memory64 = false;
/* TODO: multi-memories for now assuming the memory idx type is consistent
* across multi-memories */
if (module->import_memory_count > 0)
is_memory64 = module->import_memories[0].u.memory.flags & MEMORY64_FLAG;
else if (module->memory_count > 0)
is_memory64 = module->memories[0].flags & MEMORY64_FLAG;
mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32;
#else
mem_offset_type = VALUE_TYPE_I32;
#endif
global_count = module->import_global_count + module->global_count;
@ -7107,8 +7225,8 @@ re_scan:
}
#endif
CHECK_MEMORY();
read_leb_uint32(p, p_end, align); /* align */
read_leb_uint32(p, p_end, mem_offset); /* offset */
read_leb_uint32(p, p_end, align); /* align */
read_leb_mem_offset(p, p_end, mem_offset); /* offset */
#if WASM_ENABLE_FAST_INTERP != 0
emit_uint32(loader_ctx, mem_offset);
#endif
@ -7122,7 +7240,7 @@ re_scan:
case WASM_OP_I32_LOAD8_U:
case WASM_OP_I32_LOAD16_S:
case WASM_OP_I32_LOAD16_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32);
break;
case WASM_OP_I64_LOAD:
case WASM_OP_I64_LOAD8_S:
@ -7131,35 +7249,35 @@ re_scan:
case WASM_OP_I64_LOAD16_U:
case WASM_OP_I64_LOAD32_S:
case WASM_OP_I64_LOAD32_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64);
break;
case WASM_OP_F32_LOAD:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32);
break;
case WASM_OP_F64_LOAD:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64);
break;
/* store */
case WASM_OP_I32_STORE:
case WASM_OP_I32_STORE8:
case WASM_OP_I32_STORE16:
POP_I32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_I64_STORE:
case WASM_OP_I64_STORE8:
case WASM_OP_I64_STORE16:
case WASM_OP_I64_STORE32:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_F32_STORE:
POP_F32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_F64_STORE:
POP_F64();
POP_I32();
POP_MEM_OFFSET();
break;
default:
break;
@ -7172,7 +7290,7 @@ re_scan:
/* reserved byte 0x00 */
bh_assert(*p == 0x00);
p++;
PUSH_I32();
PUSH_PAGE_COUNT();
module->possible_memory_grow = true;
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
@ -7185,7 +7303,7 @@ re_scan:
/* reserved byte 0x00 */
bh_assert(*p == 0x00);
p++;
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, mem_offset_type);
module->possible_memory_grow = true;
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
@ -7536,7 +7654,7 @@ re_scan:
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -7565,9 +7683,9 @@ re_scan:
+ module->memory_count
> 0);
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
POP_MEM_OFFSET();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -7582,9 +7700,9 @@ re_scan:
+ module->memory_count
> 0);
POP_MEM_OFFSET();
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
func->has_memory_operations = true;
#endif
@ -7748,8 +7866,8 @@ re_scan:
#endif
if (opcode1 != WASM_OP_ATOMIC_FENCE) {
CHECK_MEMORY();
read_leb_uint32(p, p_end, align); /* align */
read_leb_uint32(p, p_end, mem_offset); /* offset */
read_leb_uint32(p, p_end, align); /* align */
read_leb_mem_offset(p, p_end, mem_offset); /* offset */
#if WASM_ENABLE_FAST_INTERP != 0
emit_uint32(loader_ctx, mem_offset);
#endif
@ -7759,18 +7877,20 @@ re_scan:
#endif
switch (opcode1) {
case WASM_OP_ATOMIC_NOTIFY:
POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_WAIT32:
POP_I64();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_WAIT64:
POP_I64();
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_FENCE:
@ -7781,26 +7901,26 @@ re_scan:
case WASM_OP_ATOMIC_I32_LOAD:
case WASM_OP_ATOMIC_I32_LOAD8_U:
case WASM_OP_ATOMIC_I32_LOAD16_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32);
break;
case WASM_OP_ATOMIC_I32_STORE:
case WASM_OP_ATOMIC_I32_STORE8:
case WASM_OP_ATOMIC_I32_STORE16:
POP_I32();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_ATOMIC_I64_LOAD:
case WASM_OP_ATOMIC_I64_LOAD8_U:
case WASM_OP_ATOMIC_I64_LOAD16_U:
case WASM_OP_ATOMIC_I64_LOAD32_U:
POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64);
POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64);
break;
case WASM_OP_ATOMIC_I64_STORE:
case WASM_OP_ATOMIC_I64_STORE8:
case WASM_OP_ATOMIC_I64_STORE16:
case WASM_OP_ATOMIC_I64_STORE32:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
break;
case WASM_OP_ATOMIC_RMW_I32_ADD:
case WASM_OP_ATOMIC_RMW_I32_ADD8_U:
@ -7820,7 +7940,9 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I32_XCHG:
case WASM_OP_ATOMIC_RMW_I32_XCHG8_U:
case WASM_OP_ATOMIC_RMW_I32_XCHG16_U:
POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32);
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_RMW_I64_ADD:
case WASM_OP_ATOMIC_RMW_I64_ADD8_U:
@ -7847,7 +7969,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I64_XCHG16_U:
case WASM_OP_ATOMIC_RMW_I64_XCHG32_U:
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I64();
break;
case WASM_OP_ATOMIC_RMW_I32_CMPXCHG:
@ -7855,7 +7977,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U:
POP_I32();
POP_I32();
POP_I32();
POP_MEM_OFFSET();
PUSH_I32();
break;
case WASM_OP_ATOMIC_RMW_I64_CMPXCHG:
@ -7864,7 +7986,7 @@ re_scan:
case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U:
POP_I64();
POP_I64();
POP_I32();
POP_MEM_OFFSET();
PUSH_I64();
break;
default: