christoph/wamr
1
Fork 0
Code Releases Activity

Implement GC (Garbage Collection) feature for interpreter, AOT and LLVM-JIT (#3125)

Browse Source 
Implement the GC (Garbage Collection) feature for interpreter mode,
AOT mode and LLVM-JIT mode, and support most features of the latest
spec proposal, and also enable the stringref feature.

Use `cmake -DWAMR_BUILD_GC=1/0` to enable/disable the feature,
and `wamrc --enable-gc` to generate the AOT file with GC supported.

And update the AOT file version from 2 to 3 since there are many AOT
ABI breaks, including the changes of AOT file format, the changes of
AOT module/memory instance layouts, the AOT runtime APIs for the
AOT code to invoke and so on.
This commit is contained in:
Wenyong Huang
2024-02-06 20:47:11 +08:00
committed by GitHub
parent 5931aaacbe
commit 16a4d71b34
98 changed files with 33469 additions and 3159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/app-mgr/app-manager/module_wasm_app.c
View File

@ -1230,8 +1230,12 @@ wasm_app_module_on_install_request_byte_arrive(uint8 ch, int request_total_size,
uint8 section_type = ch;
#if WASM_ENABLE_BULK_MEMORY == 0
uint8 section_type_max = SECTION_TYPE_DATA;
#else
#if WASM_ENABLE_STRINGREF != 0
uint8 section_type_max = SECTION_TYPE_STRINGREF;
#else
uint8 section_type_max = SECTION_TYPE_DATACOUNT;
#endif /* end of WASM_ENABLE_STRINGREF != 0 */
#endif
if (section_type <= section_type_max) {
wasm_section_t *new_section;

45
core/config.h
View File

@ -305,6 +305,11 @@
#define WASM_ENABLE_SIMD 0
#endif
/* GC performance profiling */
#ifndef WASM_ENABLE_GC_PERF_PROFILING
#define WASM_ENABLE_GC_PERF_PROFILING 0
#endif
/* Memory profiling */
#ifndef WASM_ENABLE_MEMORY_PROFILING
#define WASM_ENABLE_MEMORY_PROFILING 0
@ -325,6 +330,11 @@
#define WASM_ENABLE_DUMP_CALL_STACK 0
#endif
/* AOT stack frame */
#ifndef WASM_ENABLE_AOT_STACK_FRAME
#define WASM_ENABLE_AOT_STACK_FRAME 0
#endif
/* Heap verification */
#ifndef BH_ENABLE_GC_VERIFY
#define BH_ENABLE_GC_VERIFY 0
@ -388,6 +398,13 @@
#define APP_HEAP_SIZE_MIN (256)
#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024)
/* Default min/max gc heap size of each app */
#ifndef GC_HEAP_SIZE_DEFAULT
#define GC_HEAP_SIZE_DEFAULT (128 * 1024)
#endif
#define GC_HEAP_SIZE_MIN (4 * 1024)
#define GC_HEAP_SIZE_MAX (1024 * 1024 * 1024)
/* Default wasm stack size of each app */
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
#define DEFAULT_WASM_STACK_SIZE (16 * 1024)
@ -461,6 +478,30 @@
#define WASM_ENABLE_REF_TYPES 0
#endif
#ifndef WASM_ENABLE_GC
#define WASM_ENABLE_GC 0
#endif
#ifndef WASM_CONST_EXPR_STACK_SIZE
#if WASM_ENABLE_GC != 0
#define WASM_CONST_EXPR_STACK_SIZE 8
#else
#define WASM_CONST_EXPR_STACK_SIZE 4
#endif
#endif
#ifndef WASM_ENABLE_STRINGREF
#define WASM_ENABLE_STRINGREF 0
#endif
#ifndef GC_REFTYPE_MAP_SIZE_DEFAULT
#define GC_REFTYPE_MAP_SIZE_DEFAULT 64
#endif
#ifndef GC_RTTOBJ_MAP_SIZE_DEFAULT
#define GC_RTTOBJ_MAP_SIZE_DEFAULT 64
#endif
#ifndef WASM_ENABLE_EXCE_HANDLING
#define WASM_ENABLE_EXCE_HANDLING 0
#endif
@ -525,4 +566,8 @@
#define WASM_ENABLE_QUICK_AOT_ENTRY 1
#endif
#ifndef WASM_TABLE_MAX_SIZE
#define WASM_TABLE_MAX_SIZE 1024
#endif
#endif /* end of _CONFIG_H_ */

1104
core/iwasm/aot/aot_loader.c
View File

File diff suppressed because it is too large Load Diff

53
core/iwasm/aot/aot_reloc.h
View File

@ -9,6 +9,10 @@
#include "aot_runtime.h"
#include "aot_intrinsic.h"
#if WASM_ENABLE_STRINGREF != 0
#include "string_object.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
@ -49,10 +53,11 @@ typedef struct {
#define REG_REF_TYPES_SYM()
#endif
#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0)
#if WASM_ENABLE_AOT_STACK_FRAME != 0
#define REG_AOT_TRACE_SYM() \
REG_SYM(aot_alloc_frame), \
REG_SYM(aot_free_frame),
REG_SYM(aot_free_frame), \
REG_SYM(aot_frame_update_profile_info),
#else
#define REG_AOT_TRACE_SYM()
#endif
@ -129,6 +134,48 @@ typedef struct {
#define REG_LLVM_PGO_SYM()
#endif
#if WASM_ENABLE_GC != 0
#define REG_GC_SYM() \
REG_SYM(aot_array_init_with_data), \
REG_SYM(aot_create_func_obj), \
REG_SYM(aot_obj_is_instance_of), \
REG_SYM(aot_rtt_type_new), \
REG_SYM(wasm_array_obj_copy), \
REG_SYM(wasm_array_obj_new), \
REG_SYM(wasm_externref_obj_to_internal_obj), \
REG_SYM(wasm_internal_obj_to_externref_obj), \
REG_SYM(wasm_obj_is_type_of), \
REG_SYM(wasm_struct_obj_new),
#else
#define REG_GC_SYM()
#endif
#if WASM_ENABLE_STRINGREF != 0
#define REG_STRINGREF_SYM() \
REG_SYM(wasm_stringref_obj_new), \
REG_SYM(wasm_stringview_wtf8_obj_new), \
REG_SYM(wasm_stringview_wtf16_obj_new), \
REG_SYM(wasm_stringview_iter_obj_new), \
REG_SYM(wasm_string_destroy), \
REG_SYM(wasm_string_new_const), \
REG_SYM(wasm_string_new_with_encoding), \
REG_SYM(wasm_string_measure), \
REG_SYM(wasm_string_wtf16_get_length), \
REG_SYM(wasm_string_encode), \
REG_SYM(wasm_string_concat), \
REG_SYM(wasm_string_eq), \
REG_SYM(wasm_string_is_usv_sequence), \
REG_SYM(wasm_string_create_view), \
REG_SYM(wasm_string_advance), \
REG_SYM(wasm_string_slice), \
REG_SYM(wasm_string_get_wtf16_codeunit),\
REG_SYM(wasm_string_next_codepoint), \
REG_SYM(wasm_string_rewind), \
REG_SYM(wasm_string_dump),
#else
#define REG_STRINGREF_SYM()
#endif
#define REG_COMMON_SYMBOLS \
REG_SYM(aot_set_exception_with_id), \
REG_SYM(aot_invoke_native), \
@ -160,6 +207,8 @@ typedef struct {
REG_AOT_TRACE_SYM() \
REG_INTRINSIC_SYM() \
REG_LLVM_PGO_SYM() \
REG_GC_SYM() \
REG_STRINGREF_SYM() \
#define CHECK_RELOC_OFFSET(data_size) do { \
if (!check_reloc_offset(target_section_size, \

1308
core/iwasm/aot/aot_runtime.c
View File

File diff suppressed because it is too large Load Diff

129
core/iwasm/aot/aot_runtime.h
View File

@ -10,6 +10,9 @@
#include "../common/wasm_runtime_common.h"
#include "../interpreter/wasm_runtime.h"
#include "../compilation/aot.h"
#if WASM_ENABLE_GC != 0
#include "gc_export.h"
#endif
#if WASM_ENABLE_WASI_NN != 0
#include "../libraries/wasi-nn/src/wasi_nn_private.h"
@ -19,6 +22,20 @@
extern "C" {
#endif
/* Wasm feature supported, mainly used by AOTTargetInfo now */
#define WASM_FEATURE_SIMD_128BIT (1 << 0)
#define WASM_FEATURE_BULK_MEMORY (1 << 1)
#define WASM_FEATURE_MULTI_THREAD (1 << 2)
#define WASM_FEATURE_REF_TYPES (1 << 3)
#define WASM_FEATURE_GARBAGE_COLLECTION (1 << 4)
#define WASM_FEATURE_EXCEPTION_HANDLING (1 << 5)
#define WASM_FEATURE_MEMORY64 (1 << 6)
#define WASM_FEATURE_MULTI_MEMORY (1 << 7)
#define WASM_FEATURE_DYNAMIC_LINKING (1 << 8)
#define WASM_FEATURE_COMPONENT_MODEL (1 << 9)
#define WASM_FEATURE_RELAXED_SIMD (1 << 10)
#define WASM_FEATURE_FLEXIBLE_VECTORS (1 << 11)
typedef enum AOTSectionType {
AOT_SECTION_TYPE_TARGET_INFO = 0,
AOT_SECTION_TYPE_INIT_DATA = 1,
@ -35,6 +52,7 @@ typedef enum AOTCustomSectionType {
AOT_CUSTOM_SECTION_NATIVE_SYMBOL = 1,
AOT_CUSTOM_SECTION_ACCESS_CONTROL = 2,
AOT_CUSTOM_SECTION_NAME = 3,
AOT_CUSTOM_SECTION_STRING_LITERAL = 4,
} AOTCustomSectionType;
typedef struct AOTObjectDataSection {
@ -103,6 +121,13 @@ typedef struct GOTItem {
} GOTItem, *GOTItemList;
#endif
#if WASM_ENABLE_GC != 0
typedef struct LocalRefFlag {
uint32 local_ref_flag_cell_num;
uint8 *local_ref_flags;
} LocalRefFlag;
#endif
typedef struct AOTModule {
uint32 module_type;
@ -133,9 +158,9 @@ typedef struct AOTModule {
uint32 table_init_data_count;
AOTTableInitData **table_init_data_list;
/* function type info */
uint32 func_type_count;
AOTFuncType **func_types;
/* type info */
uint32 type_count;
AOTType **types;
/* import global variable info */
uint32 import_global_count;
@ -158,6 +183,17 @@ typedef struct AOTModule {
void **func_ptrs;
/* func type indexes of AOTed (un-imported) functions */
uint32 *func_type_indexes;
#if WASM_ENABLE_AOT_STACK_FRAME != 0
/* max local cell nums of AOTed (un-imported) functions */
uint32 *max_local_cell_nums;
/* max stack cell nums of AOTed (un-imported) functions */
uint32 *max_stack_cell_nums;
#endif
#if WASM_ENABLE_GC != 0
/* params + locals ref flags of (both import and AOTed) functions */
LocalRefFlag *func_local_ref_flags;
#endif
/* export info */
uint32 export_count;
@ -239,6 +275,26 @@ typedef struct AOTModule {
bh_list import_module_list_head;
bh_list *import_module_list;
#endif
#if WASM_ENABLE_GC != 0
/* Ref types hash set */
HashMap *ref_type_set;
struct WASMRttType **rtt_types;
korp_mutex rtt_type_lock;
#if WASM_ENABLE_STRINGREF != 0
/* special rtts for stringref types
- stringref
- stringview_wtf8
- stringview_wtf16
- stringview_iter
*/
struct WASMRttType *stringref_rtts[4];
uint32 string_literal_count;
uint32 *string_literal_lengths;
const uint8 **string_literal_ptrs;
#endif
#endif
#if WASM_ENABLE_DEBUG_AOT != 0
void *elf_hdr;
uint32 elf_size;
@ -275,8 +331,10 @@ typedef struct AOTTargetInfo {
uint32 e_version;
/* Processor-specific flags */
uint32 e_flags;
/* Specify wasm features supported */
uint64 feature_flags;
/* Reserved */
uint32 reserved;
uint64 reserved;
/* Arch name */
char arch[16];
} AOTTargetInfo;
@ -292,12 +350,37 @@ typedef struct AOTFuncPerfProfInfo {
/* AOT auxiliary call stack */
typedef struct AOTFrame {
/* The frame of the caller which is calling current function */
struct AOTFrame *prev_frame;
uint32 func_index;
#if WASM_ENABLE_PERF_PROFILING != 0
uint64 time_started;
/* The non-imported function index of current function */
uintptr_t func_index;
/* Used when performance profiling is enabled */
uintptr_t time_started;
/* Used when performance profiling is enabled */
AOTFuncPerfProfInfo *func_perf_prof_info;
#endif
/* Instruction pointer: offset to the bytecode array */
uintptr_t ip_offset;
/* Operand stack top pointer of the current frame */
uint32 *sp;
/* Frame ref flags (GC only) */
uint8 *frame_ref;
/**
* Frame data, the layout is:
* local area: parameters and local variables
* stack area: wasm operand stack
* frame ref flags (GC only):
* whether each cell in local and stack area is a GC obj
* currently local's ref flags are stored in AOTModule,
* here we only reserve the padding bytes
*/
uint32 lp[1];
} AOTFrame;
#if WASM_ENABLE_STATIC_PGO != 0
@ -564,7 +647,7 @@ void
aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst,
WASMModuleInstMemConsumption *mem_conspn);
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
void
aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx);
@ -580,11 +663,11 @@ aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx,
void
aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length,
uint32 val, uint32 data_offset);
table_elem_type_t val, uint32 data_offset);
uint32
aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx,
uint32 inc_entries, uint32 init_val);
uint32 inc_entries, table_elem_type_t init_val);
#endif
bool
@ -593,6 +676,9 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index);
void
aot_free_frame(WASMExecEnv *exec_env);
void
aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame);
bool
aot_create_call_stack(struct WASMExecEnv *exec_env);
@ -655,6 +741,27 @@ void
aot_exchange_uint64(uint8 *p_data);
#endif /* end of WASM_ENABLE_STATIC_PGO != 0 */
#if WASM_ENABLE_GC != 0
void *
aot_create_func_obj(AOTModuleInstance *module_inst, uint32 func_idx,
bool throw_exce, char *error_buf, uint32 error_buf_size);
bool
aot_obj_is_instance_of(AOTModuleInstance *module_inst, WASMObjectRef gc_obj,
uint32 type_index);
WASMRttTypeRef
aot_rtt_type_new(AOTModuleInstance *module_inst, uint32 type_index);
bool
aot_array_init_with_data(AOTModuleInstance *module_inst, uint32 seg_index,
uint32 data_seg_offset, WASMArrayObjectRef array_obj,
uint32 elem_size, uint32 array_len);
bool
aot_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap);
#endif /* end of WASM_ENABLE_GC != 0 */
#ifdef __cplusplus
} /* end of extern "C" */
#endif

1001
core/iwasm/common/gc/gc_common.c Normal file
View File

File diff suppressed because it is too large Load Diff

1071
core/iwasm/common/gc/gc_object.c Normal file
View File

File diff suppressed because it is too large Load Diff

378
core/iwasm/common/gc/gc_object.h Normal file
View File

@ -0,0 +1,378 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _GC_OBJECT_H_
#define _GC_OBJECT_H_
#include "gc_type.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* Object header of a WASM object, as the adddress of allocated memory
* must be 8-byte aligned, the lowest 3 bits are zero, we use them to
* mark the object:
* bits[0] is 1: the object is an externref object
* bits[1] is 1: the object is an anyref object
* bits[2] is 1: the object has extra information
* if both bits[0] and bits[1] are 0, then this object header must
* be a pointer of a WASMRttType, denotes that the object is a
* struct object, or an array object, or a function object
*/
typedef uintptr_t WASMObjectHeader;
#define WASM_OBJ_HEADER_MASK (~((uintptr_t)7))
#define WASM_OBJ_EXTERNREF_OBJ_FLAG (((uintptr_t)1) << 0)
#define WASM_OBJ_ANYREF_OBJ_FLAG (((uintptr_t)1) << 1)
#define WASM_OBJ_EXTRA_INFO_FLAG (((uintptr_t)1) << 2)
/* Representation of WASM objects */
typedef struct WASMObject {
WASMObjectHeader header;
} WASMObject, *WASMObjectRef;
/* Representation of WASM rtt types */
typedef struct WASMRttType {
/* type_flag must be WASM_TYPE_FUNC/STRUCT/ARRAY to
denote an object of func, struct or array */
uint32 type_flag;
uint32 inherit_depth;
WASMType *defined_type;
WASMType *root_type;
} WASMRttType, *WASMRttTypeRef;
/* Representation of WASM externref objects */
typedef struct WASMExternrefObject {
/* bits[0] must be 1, denotes an externref object */
WASMObjectHeader header;
/* an object of WASMAnyrefObjectRef which encapsulates the external
object passed from host, or other internal objects passed to
opcode extern.externalize */
WASMObjectRef internal_obj;
} WASMExternrefObject, *WASMExternrefObjectRef;
/* Representation of WASM anyref objects which encapsulate the
external object passed from host */
typedef struct WASMAnyrefObject {
/* bits[1] must be 1, denotes an anyref object */
WASMObjectHeader header;
const void *host_obj;
} WASMAnyrefObject, *WASMAnyrefObjectRef;
/**
* Representation of WASM i31 objects, the lowest bit is 1:
* for a pointer of WASMObjectRef, if the lowest bit is 1, then it is an
* i31 object and bits[1..31] stores the actual i31 value, otherwise
* it is a normal object of rtt/externref/struct/array/func */
typedef uintptr_t WASMI31ObjectRef;
/* Representation of WASM struct objects */
typedef struct WASMStructObject {
/* Must be pointer of WASMRttObject of struct type */
WASMObjectHeader header;
uint8 field_data[1];
} WASMStructObject, *WASMStructObjectRef;
/* Representation of WASM array objects */
typedef struct WASMArrayObject {
/* Must be pointer of WASMRttObject of array type */
WASMObjectHeader header;
/* (<array length> << 2) | <array element size>,
* elem_count = lenght >> 2
* elem_size = 2 ^ (length & 0x3)
*/
uint32 length;
uint8 elem_data[1];
} WASMArrayObject, *WASMArrayObjectRef;
#define WASM_ARRAY_LENGTH_SHIFT 2
#define WASM_ARRAY_ELEM_SIZE_MASK 3
/* Representation of WASM function objects */
typedef struct WASMFuncObject {
/* must be pointer of WASMRttObject of func type */
WASMObjectHeader header;
uint32 func_idx_bound;
} WASMFuncObject, *WASMFuncObjectRef;
/* Representation of WASM stringref objects */
typedef struct WASMStringrefObject {
WASMObjectHeader header;
const void *str_obj;
} WASMStringrefObject, *WASMStringrefObjectRef;
typedef struct WASMStringviewWTF8Object {
WASMObjectHeader header;
const void *str_obj;
} WASMStringviewWTF8Object, *WASMStringviewWTF8ObjectRef;
typedef struct WASMStringviewWTF16Object {
WASMObjectHeader header;
const void *str_obj;
} WASMStringviewWTF16Object, *WASMStringviewWTF16ObjectRef;
typedef struct WASMStringviewIterObject {
WASMObjectHeader header;
const void *str_obj;
int32 pos;
} WASMStringviewIterObject, *WASMStringviewIterObjectRef;
struct WASMExecEnv;
inline static WASMObjectHeader
wasm_object_header(const WASMObjectRef obj)
{
return (obj->header & WASM_OBJ_HEADER_MASK);
}
WASMRttTypeRef
wasm_rtt_type_new(WASMType *defined_type, uint32 defined_type_idx,
WASMRttType **rtt_types, uint32 rtt_type_count,
korp_mutex *rtt_type_lock);
inline static WASMType *
wasm_rtt_type_get_defined_type(const WASMRttTypeRef rtt_type)
{
return rtt_type->defined_type;
}
WASMStructObjectRef
wasm_struct_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type);
WASMStructObjectRef
wasm_struct_obj_new(struct WASMExecEnv *exec_env, WASMRttTypeRef rtt_type);
void
wasm_struct_obj_set_field(WASMStructObjectRef struct_obj, uint32 field_idx,
const WASMValue *value);
void
wasm_struct_obj_get_field(const WASMStructObjectRef struct_obj,
uint32 field_idx, bool sign_extend, WASMValue *value);
WASMArrayObjectRef
wasm_array_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type,
uint32 length, WASMValue *init_value);
WASMArrayObjectRef
wasm_array_obj_new(struct WASMExecEnv *exec_env, WASMRttTypeRef rtt_type,
uint32 length, WASMValue *init_value);
void
wasm_array_obj_set_elem(WASMArrayObjectRef array_obj, uint32 elem_idx,
const WASMValue *value);
void
wasm_array_obj_get_elem(const WASMArrayObjectRef array_obj, uint32 elem_idx,
bool sign_extend, WASMValue *value);
void
wasm_array_obj_fill(const WASMArrayObjectRef array_obj, uint32 elem_idx,
uint32 len, WASMValue *value);
void
wasm_array_obj_copy(WASMArrayObjectRef dst_obj, uint32 dst_idx,
WASMArrayObjectRef src_obj, uint32 src_idx, uint32 len);
/**
* Return the logarithm of the size of array element.
*
* @param array the WASM array object
*
* @return log(size of the array element)
*/
inline static uint32
wasm_array_obj_elem_size_log(const WASMArrayObjectRef array_obj)
{
return (array_obj->length & WASM_ARRAY_ELEM_SIZE_MASK);
}
/**
* Return the length of the array.
*
* @param array_obj the WASM array object
*
* @return the length of the array
*/
uint32
wasm_array_obj_length(const WASMArrayObjectRef array_obj);
/**
* Return the address of the first element of an array object.
*
* @param array_obj the WASM array object
*
* @return the address of the first element of the array object
*/
void *
wasm_array_obj_first_elem_addr(const WASMArrayObjectRef array_obj);
/**
* Return the address of the i-th element of an array object.
*
* @param array_obj the WASM array object
* @param index the index of the element
*
* @return the address of the i-th element of the array object
*/
void *
wasm_array_obj_elem_addr(const WASMArrayObjectRef array_obj, uint32 elem_idx);
WASMFuncObjectRef
wasm_func_obj_new_internal(void *heap_handle, WASMRttTypeRef rtt_type,
uint32 func_idx_bound);
WASMFuncObjectRef
wasm_func_obj_new(struct WASMExecEnv *exec_env, WASMRttTypeRef rtt_type,
uint32 func_idx_bound);
uint32
wasm_func_obj_get_func_idx_bound(const WASMFuncObjectRef func_obj);
WASMFuncType *
wasm_func_obj_get_func_type(const WASMFuncObjectRef func_obj);
WASMExternrefObjectRef
wasm_externref_obj_new(struct WASMExecEnv *exec_env, const void *host_obj);
WASMAnyrefObjectRef
wasm_anyref_obj_new(struct WASMExecEnv *exec_env, const void *host_obj);
/* Implementation of opcode extern.internalize */
WASMObjectRef
wasm_externref_obj_to_internal_obj(const WASMExternrefObjectRef externref_obj);
/* Implementation of opcode extern.externalize */
WASMExternrefObjectRef
wasm_internal_obj_to_externref_obj(struct WASMExecEnv *exec_env,
const WASMObjectRef internal_obj);
const void *
wasm_anyref_obj_get_value(const WASMAnyrefObjectRef anyref_obj);
const void *
wasm_externref_obj_get_value(const WASMExternrefObjectRef externref_obj);
WASMI31ObjectRef
wasm_i31_obj_new(uint32 i31_value);
uint32
wasm_i31_obj_get_value(WASMI31ObjectRef i31_obj, bool sign_extend);
bool
wasm_obj_is_i31_obj(WASMObjectRef obj);
bool
wasm_obj_is_externref_obj(WASMObjectRef obj);
bool
wasm_obj_is_anyref_obj(WASMObjectRef obj);
bool
wasm_obj_is_i31_externref_or_anyref_obj(WASMObjectRef obj);
bool
wasm_obj_is_struct_obj(WASMObjectRef obj);
bool
wasm_obj_is_array_obj(WASMObjectRef obj);
bool
wasm_obj_is_func_obj(WASMObjectRef obj);
bool
wasm_obj_is_internal_obj(WASMObjectRef obj);
bool
wasm_obj_is_eq_obj(WASMObjectRef obj);
inline static bool
wasm_obj_is_null_obj(WASMObjectRef obj)
{
return obj == NULL_REF ? true : false;
}
inline static bool
wasm_obj_is_created_from_heap(WASMObjectRef obj)
{
if (obj == NULL || (((uintptr_t)obj) & 1))
/* null object or i31 object */
return false;
return true;
}
bool
wasm_obj_is_instance_of(WASMObjectRef obj, uint32 type_idx, WASMType **types,
uint32 type_count);
bool
wasm_obj_is_type_of(WASMObjectRef obj, int32 heap_type);
bool
wasm_obj_equal(WASMObjectRef obj1, WASMObjectRef obj2);
bool
wasm_object_get_ref_list(WASMObjectRef obj, bool *p_is_compact_mode,
uint32 *p_ref_num, uint16 **p_ref_list,
uint32 *p_ref_start_offset);
#if WASM_ENABLE_STRINGREF != 0
WASMStringrefObjectRef
wasm_stringref_obj_new(struct WASMExecEnv *exec_env, const void *str_obj);
WASMStringviewWTF8ObjectRef
wasm_stringview_wtf8_obj_new(struct WASMExecEnv *exec_env, const void *str_obj);
WASMStringviewWTF16ObjectRef
wasm_stringview_wtf16_obj_new(struct WASMExecEnv *exec_env,
const void *str_obj);
WASMStringviewIterObjectRef
wasm_stringview_iter_obj_new(struct WASMExecEnv *exec_env, const void *str_obj,
int32 pos);
const void *
wasm_stringref_obj_get_value(WASMStringrefObjectRef stringref_obj);
const void *
wasm_stringview_wtf8_obj_get_value(
WASMStringviewWTF8ObjectRef stringview_wtf8_obj);
const void *
wasm_stringview_wtf16_obj_get_value(
WASMStringviewWTF16ObjectRef stringview_wtf16_obj);
const void *
wasm_stringview_iter_obj_get_value(
WASMStringviewIterObjectRef stringview_iter_obj);
int32
wasm_stringview_iter_obj_get_pos(
WASMStringviewIterObjectRef stringview_iter_obj);
void
wasm_stringview_iter_obj_update_pos(
WASMStringviewIterObjectRef stringview_iter_obj, int32 pos);
bool
wasm_obj_is_stringref_obj(WASMObjectRef obj);
bool
wasm_obj_is_stringview_wtf8_obj(WASMObjectRef obj);
bool
wasm_obj_is_stringview_wtf16_obj(WASMObjectRef obj);
#endif /* end of WASM_ENABLE_STRINGREF != 0 */
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* end of _GC_OBJECT_H_ */

1253
core/iwasm/common/gc/gc_type.c Normal file
View File

File diff suppressed because it is too large Load Diff

378
core/iwasm/common/gc/gc_type.h Normal file
View File

@ -0,0 +1,378 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _GC_TYPE_H_
#define _GC_TYPE_H_
#include "../interpreter/wasm.h"
#ifdef __cplusplus
extern "C" {
#endif
void
wasm_dump_value_type(uint8 type, const WASMRefType *ref_type);
void
wasm_dump_func_type(const WASMFuncType *type);
void
wasm_dump_struct_type(const WASMStructType *type);
void
wasm_dump_array_type(const WASMArrayType *type);
/* Whether a group of value types is subtype of
another group of value types */
bool
wasm_value_types_is_subtype_of(const uint8 *types1,
const WASMRefTypeMap *ref_type_maps1,
const uint8 *types2,
const WASMRefTypeMap *ref_type_maps2,
uint32 value_type_count,
const WASMTypePtr *types, uint32 type_count);
/* Operations of function type */
/* Whether two function types are equal */
bool
wasm_func_type_equal(const WASMFuncType *type1, const WASMFuncType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Whether func type1 is subtype of func type2 */
bool
wasm_func_type_is_subtype_of(const WASMFuncType *type1,
const WASMFuncType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Whether func type1's result types are subtype of
func type2's result types */
bool
wasm_func_type_result_is_subtype_of(const WASMFuncType *type,
const WASMFuncType *type2,
const WASMTypePtr *types,
uint32 type_count);
/* Operations of struct type */
/* Whether two struct types are equal */
bool
wasm_struct_type_equal(const WASMStructType *type1, const WASMStructType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Whether struct type1 is subtype of struct type2 */
bool
wasm_struct_type_is_subtype_of(const WASMStructType *type1,
const WASMStructType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Operations of array type */
/* Whether two array types are equal */
bool
wasm_array_type_equal(const WASMArrayType *type1, const WASMArrayType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Whether array type1 is subtype of array type2 */
bool
wasm_array_type_is_subtype_of(const WASMArrayType *type1,
const WASMArrayType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Operations of wasm type */
/* Whether a wasm type is a function type */
inline static bool
wasm_type_is_func_type(const WASMType *type)
{
return type->type_flag == WASM_TYPE_FUNC ? true : false;
}
/* Whether a wasm type is a struct type */
inline static bool
wasm_type_is_struct_type(const WASMType *type)
{
return type->type_flag == WASM_TYPE_STRUCT ? true : false;
}
/* Whether a wasm type is an array type */
inline static bool
wasm_type_is_array_type(const WASMType *type)
{
return type->type_flag == WASM_TYPE_ARRAY ? true : false;
}
/* Whether two wasm types are equal */
bool
wasm_type_equal(const WASMType *type1, const WASMType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Whether wasm type1 is subtype of wasm type2 */
bool
wasm_type_is_subtype_of(const WASMType *type1, const WASMType *type2,
const WASMTypePtr *types, uint32 type_count);
/* Operations of reference type */
/* Whether a value type is a reference type */
inline static bool
wasm_is_type_reftype(uint8 type)
{
return ((type >= (uint8)REF_TYPE_ARRAYREF
&& type <= (uint8)REF_TYPE_NULLFUNCREF)
|| (type >= (uint8)REF_TYPE_HT_NULLABLE
&& type <= (uint8)REF_TYPE_HT_NON_NULLABLE)
#if WASM_ENABLE_STRINGREF != 0
|| (type >= (uint8)REF_TYPE_STRINGVIEWWTF8
&& type <= (uint8)REF_TYPE_STRINGREF)
|| (type >= (uint8)REF_TYPE_STRINGVIEWITER
&& type <= (uint8)REF_TYPE_STRINGVIEWWTF16)
#endif
)
? true
: false;
}
/* Whether a negative value is a valid heap type */
inline static bool
wasm_is_valid_heap_type(int32 heap_type)
{
return ((heap_type <= HEAP_TYPE_NOFUNC && heap_type >= HEAP_TYPE_ARRAY)
#if WASM_ENABLE_STRINGREF != 0
|| heap_type == HEAP_TYPE_STRINGREF
|| heap_type == HEAP_TYPE_STRINGVIEWWTF8
|| heap_type == HEAP_TYPE_STRINGVIEWWTF16
|| heap_type == HEAP_TYPE_STRINGVIEWITER
#endif
)
? true
: false;
}
/* Whether a value type is multi-byte type, or, requires ref type map
to retrieve extra info */
inline static bool
wasm_is_type_multi_byte_type(uint8 type)
{
return (type == (uint8)REF_TYPE_HT_NULLABLE
|| type == (uint8)REF_TYPE_HT_NON_NULLABLE)
? true
: false;
}
/* Whether a reference type is a funcref type */
inline static bool
wasm_is_reftype_funcref(uint8 type)
{
return type == (uint8)REF_TYPE_FUNCREF ? true : false;
}
/* Whether a reference type is an externref type */
inline static bool
wasm_is_reftype_externref(uint8 type)
{
return type == (uint8)REF_TYPE_EXTERNREF ? true : false;
}
/* Whether a reference type is an anyref type */
inline static bool
wasm_is_reftype_anyref(uint8 type)
{
return type == (uint8)REF_TYPE_ANYREF ? true : false;
}
/* Whether a reference type is an eqref type */
inline static bool
wasm_is_reftype_eqref(uint8 type)
{
return type == (uint8)REF_TYPE_EQREF ? true : false;
}
/* Whether a reference type is a (ref null ht) type */
inline static bool
wasm_is_reftype_htref_nullable(uint8 type)
{
return type == (uint8)REF_TYPE_HT_NULLABLE ? true : false;
}
/* Whether a reference type is a (ref ht) type */
inline static bool
wasm_is_reftype_htref_non_nullable(uint8 type)
{
return type == (uint8)REF_TYPE_HT_NON_NULLABLE ? true : false;
}
/* Whether a reference type is an i31ref type */
inline static bool
wasm_is_reftype_i31ref(uint8 type)
{
return type == (uint8)REF_TYPE_I31REF ? true : false;
}
/* Whether a reference type is a structref type */
inline static bool
wasm_is_reftype_structref(uint8 type)
{
return type == (uint8)REF_TYPE_STRUCTREF ? true : false;
}
/* Whether a reference type is an arrayref type */
inline static bool
wasm_is_reftype_arrayref(uint8 type)
{
return type == (uint8)REF_TYPE_ARRAYREF ? true : false;
}
/* Whether a reference type is a nullref type */
inline static bool
wasm_is_reftype_nullref(uint8 type)
{
return type == (uint8)REF_TYPE_NULLREF ? true : false;
}
/* Whether a reference type is a nullfuncref type */
inline static bool
wasm_is_reftype_nullfuncref(uint8 type)
{
return type == (uint8)REF_TYPE_NULLFUNCREF ? true : false;
}
/* Whether a reference type is a nullexternref type */
inline static bool
wasm_is_reftype_nullexternref(uint8 type)
{
return type == (uint8)REF_TYPE_NULLEXTERNREF ? true : false;
}
/* Return the size of a reference type */
uint32
wasm_reftype_size(uint8 type);
/* Return the actual WASMRefType struct size required of a reference type */
uint32
wasm_reftype_struct_size(const WASMRefType *ref_type);
/* Operations of ref heap type */
/* Whether a ref heap type is (type i), i : typeidx, >= 0 */
inline static bool
wasm_is_refheaptype_typeidx(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type >= 0 ? true : false;
}
/* Whether a ref heap type is a common type: func/any/eq/i31/data,
not (type i) or (rtt n i) or (rtt i) */
inline static bool
wasm_is_refheaptype_common(const RefHeapType_Common *ref_heap_type)
{
return ((ref_heap_type->heap_type >= (int32)HEAP_TYPE_ARRAY
&& ref_heap_type->heap_type <= (int32)HEAP_TYPE_NONE)
#if WASM_ENABLE_STRINGREF != 0
|| (ref_heap_type->heap_type >= (int32)HEAP_TYPE_STRINGVIEWITER
&& ref_heap_type->heap_type <= (int32)HEAP_TYPE_I31)
#endif
)
? true
: false;
}
/* Whether a ref heap type is a func type */
inline static bool
wasm_is_refheaptype_func(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type == (int32)HEAP_TYPE_FUNC ? true : false;
}
/* Whether a ref heap type is an any type */
inline static bool
wasm_is_refheaptype_any(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type == (int32)HEAP_TYPE_ANY ? true : false;
}
/* Whether a ref heap type is an eq type */
inline static bool
wasm_is_refheaptype_eq(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type == (int32)HEAP_TYPE_EQ ? true : false;
}
/* Whether a ref heap type is an i31 type */
inline static bool
wasm_is_refheaptype_i31(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type == (int32)HEAP_TYPE_I31 ? true : false;
}
/* Whether a ref heap type is an array type */
inline static bool
wasm_is_refheaptype_array(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type == (int32)HEAP_TYPE_ARRAY ? true : false;
}
#if WASM_ENABLE_STRINGREF != 0
inline static bool
wasm_is_refheaptype_stringrefs(const RefHeapType_Common *ref_heap_type)
{
return ref_heap_type->heap_type <= (int32)HEAP_TYPE_STRINGREF
&& ref_heap_type->heap_type >= HEAP_TYPE_STRINGVIEWITER
? true
: false;
}
#endif
/* Whether two ref heap types are equal */
bool
wasm_refheaptype_equal(const RefHeapType_Common *ref_heap_type1,
const RefHeapType_Common *ref_heap_type2,
const WASMTypePtr *types, uint32 type_count);
/* Whether two ref types are equal */
bool
wasm_reftype_equal(uint8 type1, const WASMRefType *reftype1, uint8 type2,
const WASMRefType *reftype2, const WASMTypePtr *types,
uint32 type_count);
/* Whether ref type1 is subtype of ref type2 */
bool
wasm_reftype_is_subtype_of(uint8 type1, const WASMRefType *reftype1,
uint8 type2, const WASMRefType *reftype2,
const WASMTypePtr *types, uint32 type_count);
/* Returns a new reference type which is a duplication of ref_type,
the caller should use wasm_runtime_free() to free the new ref type */
WASMRefType *
wasm_reftype_dup(const WASMRefType *ref_type);
/* Set fields of RefHeapType_TypeIdx */
void
wasm_set_refheaptype_typeidx(RefHeapType_TypeIdx *ref_ht_typeidx, bool nullable,
int32 type_idx);
/* Set fields of RefHeapType_Common */
void
wasm_set_refheaptype_common(RefHeapType_Common *ref_ht_common, bool nullable,
int32 heap_type);
/* Find the related reftype in reftype map array with index */
WASMRefType *
wasm_reftype_map_find(WASMRefTypeMap *ref_type_maps, uint32 ref_type_map_count,
uint32 index_to_find);
/* Create a new hash set of reference type */
HashMap *
wasm_reftype_set_create(uint32 size);
/* Insert a reference type into the hash set */
WASMRefType *
wasm_reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type);
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* end of _GC_TYPE_H_ */

36
core/iwasm/common/gc/iwasm_gc.cmake Normal file
View File

@ -0,0 +1,36 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
set (IWASM_GC_DIR ${CMAKE_CURRENT_LIST_DIR})
add_definitions (-DWASM_ENABLE_GC=1)
if (WAMR_TEST_GC EQUAL 1)
add_definitions (-DGC_MANUALLY=1 -DGC_IN_EVERY_ALLOCATION=1)
endif ()
include_directories (${IWASM_GC_DIR})
file (GLOB source_all ${IWASM_GC_DIR}/*.c)
set (IWASM_GC_SOURCE ${source_all})
if (WAMR_BUILD_STRINGREF EQUAL 1)
set (IWASM_STRINGREF_DIR ${CMAKE_CURRENT_LIST_DIR}/stringref)
add_definitions (-DWASM_ENABLE_STRINGREF=1)
include_directories (${IWASM_STRINGREF_DIR})
if (NOT DEFINED WAMR_STRINGREF_IMPL_SOURCE)
message(FATAL_ERROR "stringref feature enabled, but WAMR_STRINGREF_IMPL_SOURCE not set" )
else ()
if (${WAMR_STRINGREF_IMPL_SOURCE} STREQUAL "STUB")
set (IWASM_STRINGREF_SOURCE ${IWASM_STRINGREF_DIR}/stringref_stub.c)
else()
set (IWASM_STRINGREF_SOURCE ${WAMR_STRINGREF_IMPL_SOURCE})
endif()
endif ()
set (IWASM_GC_SOURCE ${IWASM_GC_SOURCE} ${IWASM_STRINGREF_SOURCE})
endif ()

121
core/iwasm/common/gc/stringref/string_object.h Normal file
View File

@ -0,0 +1,121 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _STRING_OBJECT_H_
#define _STRING_OBJECT_H_
#include "wasm.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef enum EncodingFlag {
UTF8,
WTF8,
WTF16,
LOSSY_UTF8,
} EncodingFlag;
typedef enum StringViewType {
STRING_VIEW_WTF8,
STRING_VIEW_WTF16,
STRING_VIEW_ITER,
} StringViewType;
typedef enum ErrorCode {
Insufficient_Space = -3,
Encode_Fail = -2,
Isolated_Surrogate = -1,
} ErrorCode;
/******************* gc finalizer *****************/
void
wasm_string_destroy(WASMString str_obj);
/******************* opcode functions *****************/
/* string.const */
WASMString
wasm_string_new_const(const char *content, uint32 length);
/* string.new_xx8/new_wtf16 */
/* string.new_xx8_array */
/* string.new_wtf16_array */
WASMString
wasm_string_new_with_encoding(void *addr, uint32 count, EncodingFlag flag);
/* string.measure */
int32
wasm_string_measure(WASMString str_obj, EncodingFlag flag);
/* stringview_wtf16.length */
int32
wasm_string_wtf16_get_length(WASMString str_obj);
/* string.encode_xx8 */
/* string.encode_wtf16 */
/* stringview_wtf8.encode_xx */
/* stringview_wtf16.encode */
/* string.encode_xx8_array */
/* string.encode_wtf16_array */
int32
wasm_string_encode(WASMString str_obj, uint32 pos, uint32 count, void *addr,
uint32 *next_pos, EncodingFlag flag);
/* string.concat */
WASMString
wasm_string_concat(WASMString str_obj1, WASMString str_obj2);
/* string.eq */
int32
wasm_string_eq(WASMString str_obj1, WASMString str_obj2);
/* string.is_usv_sequence */
int32
wasm_string_is_usv_sequence(WASMString str_obj);
/* string.as_wtf8 */
/* string.as_wtf16 */
/* string.as_iter */
WASMString
wasm_string_create_view(WASMString str_obj, StringViewType type);
/* stringview_wtf8.advance */
/* stringview_iter.advance */
int32
wasm_string_advance(WASMString str_obj, uint32 pos, uint32 count,
uint32 *target_pos);
/* stringview_wtf8.slice */
/* stringview_wtf16.slice */
/* stringview_iter.slice */
WASMString
wasm_string_slice(WASMString str_obj, uint32 start, uint32 end,
StringViewType type);
/* stringview_wtf16.get_codeunit */
int16
wasm_string_get_wtf16_codeunit(WASMString str_obj, int32 pos);
/* stringview_iter.next */
uint32
wasm_string_next_codepoint(WASMString str_obj, uint32 pos);
/* stringview_iter.rewind */
uint32
wasm_string_rewind(WASMString str_obj, uint32 pos, uint32 count,
uint32 *target_pos);
/******************* application functions *****************/
void
wasm_string_dump(WASMString str_obj);
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* end of _STRING_OBJECT_H_ */

136
core/iwasm/common/gc/stringref/stringref_stub.c Normal file
View File

@ -0,0 +1,136 @@
/*
* Copyright (C) 2023 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
/* This file is the stub for stringref implementation, only used for wamrc
* compiler. The runtime embedder SHOULD NOT use this file */
#include "string_object.h"
/******************* gc finalizer *****************/
void
wasm_string_destroy(WASMString str_obj)
{}
/******************* opcode functions *****************/
/* string.const */
WASMString
wasm_string_new_const(const char *str, uint32 length)
{
return NULL;
}
/* string.new_xx8 */
/* string.new_wtf16 */
/* string.new_xx8_array */
/* string.new_wtf16_array */
WASMString
wasm_string_new_with_encoding(void *addr, uint32 count, EncodingFlag flag)
{
return NULL;
}
/* string.measure */
/* stringview_wtf16.length */
int32
wasm_string_measure(WASMString str_obj, EncodingFlag flag)
{
return 0;
}
/* stringview_wtf16.length */
int32
wasm_string_wtf16_get_length(WASMString str_obj)
{
return 0;
}
/* string.encode_xx8 */
/* string.encode_wtf16 */
/* stringview_wtf8.encode_xx */
/* stringview_wtf16.encode */
/* string.encode_xx8_array */
/* string.encode_wtf16_array */
int32
wasm_string_encode(WASMString str_obj, uint32 pos, uint32 count, void *addr,
uint32 *next_pos, EncodingFlag flag)
{
return 0;
}
/* string.concat */
WASMString
wasm_string_concat(WASMString str_obj1, WASMString str_obj2)
{
return NULL;
}
/* string.eq */
int32
wasm_string_eq(WASMString str_obj1, WASMString str_obj2)
{
return 0;
}
/* string.is_usv_sequence */
int32
wasm_string_is_usv_sequence(WASMString str_obj)
{
return 0;
}
/* string.as_wtf8 */
/* string.as_wtf16 */
/* string.as_iter */
WASMString
wasm_string_create_view(WASMString str_obj, StringViewType type)
{
return NULL;
}
/* stringview_wtf8.advance */
/* stringview_iter.advance */
int32
wasm_string_advance(WASMString str_obj, uint32 pos, uint32 count,
uint32 *consumed)
{
return 0;
}
/* stringview_wtf8.slice */
/* stringview_wtf16.slice */
/* stringview_iter.slice */
WASMString
wasm_string_slice(WASMString str_obj, uint32 start, uint32 end,
StringViewType type)
{
return NULL;
}
/* stringview_wtf16.get_codeunit */
int16
wasm_string_get_wtf16_codeunit(WASMString str_obj, int32 pos)
{
return 0;
}
/* stringview_iter.next */
uint32
wasm_string_next_codepoint(WASMString str_obj, uint32 pos)
{
return 0;
}
/* stringview_iter.rewind */
uint32
wasm_string_rewind(WASMString str_obj, uint32 pos, uint32 count,
uint32 *consumed)
{
return 0;
}
void
wasm_string_dump(WASMString str_obj)
{}

272
core/iwasm/common/wasm_application.c
View File

@ -13,6 +13,15 @@
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif
#if WASM_ENABLE_GC != 0
#include "gc/gc_object.h"
#if WASM_ENABLE_STRINGREF != 0
#include "string_object.h"
#endif
#if WASM_ENABLE_GC_PERF_PROFILING != 0
#include "../../shared/mem-alloc/mem_alloc.h"
#endif
#endif
static void
set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
@ -52,7 +61,7 @@ static union {
* Implementation of wasm_application_execute_main()
*/
static bool
check_main_func_type(const WASMType *type)
check_main_func_type(const WASMFuncType *type)
{
if (!(type->param_count == 0 || type->param_count == 2)
|| type->result_count > 1) {
@ -83,7 +92,7 @@ static bool
execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[])
{
WASMFunctionInstanceCommon *func;
WASMType *func_type = NULL;
WASMFuncType *func_type = NULL;
WASMExecEnv *exec_env = NULL;
uint32 argc1 = 0, argv1[2] = { 0 };
uint32 total_argv_size = 0;
@ -244,6 +253,11 @@ wasm_application_execute_main(WASMModuleInstanceCommon *module_inst, int32 argc,
}
#endif
#if WASM_ENABLE_GC_PERF_PROFILING != 0
void *handle = wasm_runtime_get_gc_heap_handle(module_inst);
mem_allocator_dump_perf_profiling(handle);
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
wasm_runtime_dump_perf_profiling(module_inst);
#endif
@ -304,10 +318,15 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
int32 argc, char *argv[])
{
WASMFunctionInstanceCommon *target_func;
WASMType *type = NULL;
WASMFuncType *type = NULL;
WASMExecEnv *exec_env = NULL;
#if WASM_ENABLE_GC != 0
WASMRefTypeMap *ref_type_map;
WASMLocalObjectRef *local_ref;
uint32 num_local_ref_pushed = 0;
#endif
uint32 argc1, *argv1 = NULL, cell_num = 0, j, k = 0;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
uint32 param_size_in_double_world = 0, result_size_in_double_world = 0;
#endif
int32 i, p, module_type;
@ -337,7 +356,14 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
goto fail;
}
#if WASM_ENABLE_REF_TYPES != 0
exec_env = wasm_runtime_get_exec_env_singleton(module_inst);
if (!exec_env) {
wasm_runtime_set_exception(module_inst,
"create singleton exec_env failed");
goto fail;
}
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
for (i = 0; i < type->param_count; i++) {
param_size_in_double_world +=
wasm_value_type_cell_num_outside(type->types[i]);
@ -360,6 +386,9 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
goto fail;
}
#if WASM_ENABLE_GC != 0
ref_type_map = type->ref_type_maps;
#endif
/* Parse arguments */
for (i = 0, p = 0; i < argc; i++) {
char *endptr = NULL;
@ -482,8 +511,11 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
break;
}
#endif /* WASM_ENABLE_SIMD != 0 */
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#if UINTPTR_MAX == UINT32_MAX
case VALUE_TYPE_EXTERNREF:
#endif
{
if (strncasecmp(argv[i], "null", 4) == 0) {
argv1[p++] = (uint32)-1;
@ -493,16 +525,9 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
}
break;
}
#if UINTPTR_MAX == UINT64_MAX
case VALUE_TYPE_EXTERNREF:
{
#if UINTPTR_MAX == UINT32_MAX
if (strncasecmp(argv[i], "null", 4) == 0) {
argv1[p++] = (uint32)-1;
}
else {
argv1[p++] = strtoul(argv[i], &endptr, 0);
}
#else
union {
uintptr_t val;
uint32 parts[2];
@ -515,13 +540,97 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
}
argv1[p++] = u.parts[0];
argv1[p++] = u.parts[1];
#endif
break;
}
#endif /* WASM_ENABLE_REF_TYPES */
#endif
#endif /* WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */
default:
{
#if WASM_ENABLE_GC != 0
bool is_extern_ref = false;
bool is_anyref = false;
if (wasm_is_type_reftype(type->types[i])) {
if (strncasecmp(argv[i], "null", 4) == 0) {
PUT_REF_TO_ADDR(argv1 + p, NULL_REF);
p += REF_CELL_NUM;
break;
}
else if (type->types[i] == VALUE_TYPE_EXTERNREF) {
is_extern_ref = true;
}
else if (type->types[i] == VALUE_TYPE_ANYREF) {
is_anyref = true;
}
if (wasm_is_type_multi_byte_type(
type->types[type->param_count + i])) {
WASMRefType *ref_type = ref_type_map->ref_type;
if (wasm_is_refheaptype_common(
&ref_type->ref_ht_common)) {
int32 heap_type = ref_type->ref_ht_common.heap_type;
if (heap_type == HEAP_TYPE_EXTERN) {
is_extern_ref = true;
}
else if (heap_type == HEAP_TYPE_ANY) {
is_anyref = true;
}
}
ref_type_map++;
}
if (is_extern_ref) {
WASMExternrefObjectRef gc_obj;
void *extern_obj =
(void *)(uintptr_t)strtoull(argv[i], &endptr, 0);
gc_obj = wasm_externref_obj_new(exec_env, extern_obj);
if (!gc_obj) {
wasm_runtime_set_exception(
module_inst, "create extern object failed");
goto fail;
}
if (!(local_ref =
runtime_malloc(sizeof(WASMLocalObjectRef),
module_inst, NULL, 0))) {
goto fail;
}
wasm_runtime_push_local_obj_ref(exec_env, local_ref);
local_ref->val = (WASMObjectRef)gc_obj;
num_local_ref_pushed++;
PUT_REF_TO_ADDR(argv1 + p, gc_obj);
p += REF_CELL_NUM;
}
else if (is_anyref) {
/* If a parameter type is (ref null? any) and its value
* is not null, then we treat the value as host ptr */
WASMAnyrefObjectRef gc_obj;
void *host_obj =
(void *)(uintptr_t)strtoull(argv[i], &endptr, 0);
gc_obj = wasm_anyref_obj_new(exec_env, host_obj);
if (!gc_obj) {
wasm_runtime_set_exception(
module_inst, "create anyref object failed");
goto fail;
}
if (!(local_ref =
runtime_malloc(sizeof(WASMLocalObjectRef),
module_inst, NULL, 0))) {
goto fail;
}
wasm_runtime_push_local_obj_ref(exec_env, local_ref);
local_ref->val = (WASMObjectRef)gc_obj;
num_local_ref_pushed++;
PUT_REF_TO_ADDR(argv1 + p, gc_obj);
p += REF_CELL_NUM;
}
break;
}
#endif /* end of WASM_ENABLE_GC != 0 */
bh_assert(0);
break;
}
}
if (endptr && *endptr != '\0' && *endptr != '_') {
snprintf(buf, sizeof(buf), "invalid input argument %" PRId32 ": %s",
@ -532,21 +641,17 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
}
wasm_runtime_set_exception(module_inst, NULL);
#if WASM_ENABLE_REF_TYPES == 0
#if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0
bh_assert(p == (int32)argc1);
#endif
exec_env = wasm_runtime_get_exec_env_singleton(module_inst);
if (!exec_env) {
wasm_runtime_set_exception(module_inst,
"create singleton exec_env failed");
goto fail;
}
if (!wasm_runtime_call_wasm(exec_env, target_func, argc1, argv1)) {
goto fail;
}
#if WASM_ENABLE_GC != 0
ref_type_map = type->result_ref_type_maps;
#endif
/* print return value */
for (j = 0; j < type->result_count; j++) {
switch (type->types[type->param_count + j]) {
@ -586,7 +691,7 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
os_printf("%.7g:f64", u.val);
break;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
{
if (argv1[k] != NULL_REF)
@ -619,7 +724,7 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
#endif
break;
}
#endif
#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */
#if WASM_ENABLE_SIMD != 0
case VALUE_TYPE_V128:
{
@ -631,14 +736,117 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name,
}
#endif /* WASM_ENABLE_SIMD != 0 */
default:
{
#if WASM_ENABLE_GC != 0
if (wasm_is_type_reftype(type->types[type->param_count + j])) {
void *gc_obj = GET_REF_FROM_ADDR(argv1 + k);
k += REF_CELL_NUM;
if (!gc_obj) {
uint8 type1 = type->types[type->param_count + j];
WASMRefType *ref_type1 = NULL;
WASMType **types = NULL;
uint32 type_count = 0;
if (wasm_is_type_multi_byte_type(
type->types[type->param_count + j]))
ref_type1 = ref_type_map->ref_type;
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
WASMModule *module =
((WASMModuleInstance *)module_inst)->module;
types = module->types;
type_count = module->type_count;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModule *module =
(AOTModule *)((AOTModuleInstance *)module_inst)
->module;
types = module->types;
type_count = module->type_count;
}
#endif
bh_assert(type);
if (wasm_reftype_is_subtype_of(type1, ref_type1,
REF_TYPE_ANYREF, NULL,
types, type_count))
os_printf("any:");
else if (wasm_reftype_is_subtype_of(
type1, ref_type1, REF_TYPE_FUNCREF, NULL,
types, type_count))
os_printf("func:");
if (wasm_reftype_is_subtype_of(type1, ref_type1,
REF_TYPE_EXTERNREF, NULL,
types, type_count))
os_printf("extern:");
os_printf("ref.null");
}
else if (wasm_obj_is_func_obj(gc_obj))
os_printf("ref.func");
#if WASM_ENABLE_STRINGREF != 0
else if (wasm_obj_is_stringref_obj(gc_obj)
|| wasm_obj_is_stringview_wtf8_obj(gc_obj)) {
wasm_string_dump(
(WASMString)wasm_stringref_obj_get_value(gc_obj));
}
else if (wasm_obj_is_stringview_wtf16_obj(gc_obj)) {
wasm_string_dump(
(WASMString)wasm_stringview_wtf16_obj_get_value(
gc_obj));
}
#endif
else if (wasm_obj_is_externref_obj(gc_obj)) {
#if WASM_ENABLE_SPEC_TEST != 0
WASMObjectRef obj = wasm_externref_obj_to_internal_obj(
(WASMExternrefObjectRef)gc_obj);
if (wasm_obj_is_anyref_obj(obj))
os_printf("0x%" PRIxPTR ":ref.extern",
(uintptr_t)wasm_anyref_obj_get_value(
(WASMAnyrefObjectRef)obj));
else
#endif
os_printf("ref.extern");
}
else if (wasm_obj_is_i31_obj(gc_obj))
os_printf("ref.i31");
else if (wasm_obj_is_array_obj(gc_obj))
os_printf("ref.array");
else if (wasm_obj_is_struct_obj(gc_obj))
os_printf("ref.struct");
else if (wasm_obj_is_eq_obj(gc_obj))
os_printf("ref.eq");
else if (wasm_obj_is_anyref_obj(gc_obj))
os_printf("0x%" PRIxPTR ":ref.host",
(uintptr_t)wasm_anyref_obj_get_value(
(WASMAnyrefObjectRef)gc_obj));
else if (wasm_obj_is_internal_obj(gc_obj))
os_printf("ref.any");
if (wasm_is_type_multi_byte_type(
type->types[type->param_count + j]))
ref_type_map++;
break;
}
#endif /* endof WASM_ENABLE_GC != 0 */
bh_assert(0);
break;
}
}
if (j < (uint32)(type->result_count - 1))
os_printf(",");
}
os_printf("\n");
#if WASM_ENABLE_GC != 0
for (j = 0; j < num_local_ref_pushed; j++) {
local_ref = wasm_runtime_pop_local_obj_ref(exec_env);
wasm_runtime_free(local_ref);
}
#endif
wasm_runtime_free(argv1);
return true;
@ -646,6 +854,13 @@ fail:
if (argv1)
wasm_runtime_free(argv1);
#if WASM_ENABLE_GC != 0
for (j = 0; j < num_local_ref_pushed; j++) {
local_ref = wasm_runtime_pop_local_obj_ref(exec_env);
wasm_runtime_free(local_ref);
}
#endif
bh_assert(wasm_runtime_get_exception(module_inst));
return false;
}
@ -668,6 +883,11 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst,
}
#endif
#if WASM_ENABLE_GC_PERF_PROFILING != 0
void *handle = wasm_runtime_get_gc_heap_handle(module_inst);
mem_allocator_dump_perf_profiling(handle);
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
wasm_runtime_dump_perf_profiling(module_inst);
#endif

84
core/iwasm/common/wasm_c_api.c
View File

@ -774,7 +774,7 @@ wasm_valtype_new(wasm_valkind_t kind)
wasm_valtype_t *val_type;
if (kind > WASM_F64 && WASM_FUNCREF != kind
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
&& WASM_ANYREF != kind
#endif
) {
@ -811,7 +811,7 @@ wasm_valtype_kind(const wasm_valtype_t *val_type)
}
static wasm_functype_t *
wasm_functype_new_internal(WASMType *type_rt)
wasm_functype_new_internal(WASMFuncType *type_rt)
{
wasm_functype_t *type = NULL;
wasm_valtype_t *param_type = NULL, *result_type = NULL;
@ -827,7 +827,7 @@ wasm_functype_new_internal(WASMType *type_rt)
type->extern_kind = WASM_EXTERN_FUNC;
/* WASMType->types[0 : type_rt->param_count) -> type->params */
/* WASMFuncType->types[0 : type_rt->param_count) -> type->params */
INIT_VEC(type->params, wasm_valtype_vec_new_uninitialized,
type_rt->param_count);
for (i = 0; i < type_rt->param_count; ++i) {
@ -841,7 +841,7 @@ wasm_functype_new_internal(WASMType *type_rt)
}
}
/* WASMType->types[type_rt->param_count : type_rt->result_count) ->
/* WASMFuncType->types[type_rt->param_count : type_rt->result_count) ->
* type->results */
INIT_VEC(type->results, wasm_valtype_vec_new_uninitialized,
type_rt->result_count);
@ -983,7 +983,7 @@ cmp_val_kind_with_val_type(wasm_valkind_t v_k, uint8 v_t)
*/
static bool
wasm_functype_same_internal(const wasm_functype_t *type,
const WASMType *type_intl)
const WASMFuncType *type_intl)
{
uint32 i = 0;
@ -1132,7 +1132,7 @@ wasm_tabletype_new(own wasm_valtype_t *val_type, const wasm_limits_t *limits)
}
if (wasm_valtype_kind(val_type) != WASM_FUNCREF
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
&& wasm_valtype_kind(val_type) != WASM_ANYREF
#endif
) {
@ -1646,7 +1646,7 @@ rt_val_to_wasm_val(const uint8 *data, uint8 val_type_rt, wasm_val_t *out)
out->kind = WASM_F64;
out->of.f64 = *((float64 *)data);
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
out->kind = WASM_ANYREF;
if (NULL_REF == *(uint32 *)data) {
@ -1687,7 +1687,7 @@ wasm_val_to_rt_val(WASMModuleInstanceCommon *inst_comm_rt, uint8 val_type_rt,
bh_assert(WASM_F64 == v->kind);
*((float64 *)data) = v->of.f64;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
bh_assert(WASM_ANYREF == v->kind);
ret =
@ -2470,7 +2470,7 @@ wasm_module_imports(const wasm_module_t *module, own wasm_importtype_vec_t *out)
if (i < import_func_count) {
wasm_functype_t *type = NULL;
WASMType *type_rt = NULL;
WASMFuncType *type_rt = NULL;
#if WASM_ENABLE_INTERP != 0
if ((*module)->module_type == Wasm_Module_Bytecode) {
@ -2715,13 +2715,13 @@ wasm_module_exports(const wasm_module_t *module, wasm_exporttype_vec_t *out)
goto failed;
}
/* WASMExport -> (WASMType, (uint8, bool)) -> (wasm_functype_t,
/* WASMExport -> (WASMFuncType, (uint8, bool)) -> (wasm_functype_t,
* wasm_globaltype_t) -> wasm_externtype_t*/
switch (export->kind) {
case EXPORT_KIND_FUNC:
{
wasm_functype_t *type = NULL;
WASMType *type_rt;
WASMFuncType *type_rt;
if (!wasm_runtime_get_export_func_type(*module, export,
&type_rt)) {
@ -2776,12 +2776,22 @@ wasm_module_exports(const wasm_module_t *module, wasm_exporttype_vec_t *out)
{
wasm_tabletype_t *type = NULL;
uint8 elem_type_rt = 0;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type_rt;
#endif
uint32 min_size = 0, max_size = 0;
if (!wasm_runtime_get_export_table_type(
*module, export, &elem_type_rt, &min_size, &max_size)) {
if (!wasm_runtime_get_export_table_type(*module, export,
&elem_type_rt,
#if WASM_ENABLE_GC != 0
&elem_ref_type_rt,
#endif
&min_size, &max_size)) {
goto failed;
}
#if WASM_ENABLE_GC != 0
(void)elem_ref_type_rt; /* TODO */
#endif
if (!(type = wasm_tabletype_new_internal(elem_type_rt, min_size,
max_size))) {
@ -3032,7 +3042,7 @@ wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt,
WASMModuleInstanceCommon *inst_comm_rt)
{
wasm_func_t *func = NULL;
WASMType *type_rt = NULL;
WASMFuncType *type_rt = NULL;
bh_assert(singleton_engine);
@ -3070,9 +3080,9 @@ wasm_func_new_internal(wasm_store_t *store, uint16 func_idx_rt,
}
else {
type_rt =
module_aot->func_types[module_aot->func_type_indexes
[func_idx_rt
- module_aot->import_func_count]];
(AOTFuncType *)module_aot
->types[module_aot->func_type_indexes
[func_idx_rt - module_aot->import_func_count]];
}
}
#endif
@ -3204,8 +3214,7 @@ params_to_argv(const wasm_val_vec_t *params,
*(int64 *)argv = param->of.i64;
argv += 2;
break;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case WASM_ANYREF:
*(uintptr_t *)argv = (uintptr_t)param->of.ref;
argv += sizeof(uintptr_t) / sizeof(uint32);
@ -3247,7 +3256,7 @@ argv_to_results(const uint32 *argv, const wasm_valtype_vec_t *result_defs,
result->of.i64 = *(int64 *)argv;
argv += 2;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case WASM_ANYREF:
result->of.ref = (struct wasm_ref_t *)(*(uintptr_t *)argv);
argv += sizeof(uintptr_t) / sizeof(uint32);
@ -3830,6 +3839,9 @@ wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt,
{
wasm_table_t *table = NULL;
uint8 val_type_rt = 0;
#if WASM_ENABLE_GC != 0
WASMRefType *val_ref_type_rt;
#endif
uint32 init_size = 0, max_size = 0;
bh_assert(singleton_engine);
@ -3845,14 +3857,21 @@ wasm_table_new_internal(wasm_store_t *store, uint16 table_idx_rt,
table->store = store;
table->kind = WASM_EXTERN_TABLE;
if (!wasm_runtime_get_table_inst_elem_type(
inst_comm_rt, table_idx_rt, &val_type_rt, &init_size, &max_size)) {
if (!wasm_runtime_get_table_inst_elem_type(inst_comm_rt, table_idx_rt,
&val_type_rt,
#if WASM_ENABLE_GC != 0
&val_ref_type_rt,
#endif
&init_size, &max_size)) {
/*
* a wrong combination of module filetype and compilation flags
* leads to below branch
*/
goto failed;
}
#if WASM_ENABLE_GC != 0
(void)val_ref_type_rt; /* TODO */
#endif
if (!(table->type =
wasm_tabletype_new_internal(val_type_rt, init_size, max_size))) {
@ -3922,6 +3941,7 @@ wasm_table_type(const wasm_table_t *table)
return wasm_tabletype_copy(table->type);
}
#if WASM_ENABLE_GC == 0
own wasm_ref_t *
wasm_table_get(const wasm_table_t *table, wasm_table_size_t index)
{
@ -4010,7 +4030,7 @@ wasm_table_set(wasm_table_t *table, wasm_table_size_t index,
return false;
}
p_ref_idx = table_interp->elems + index;
p_ref_idx = (uint32 *)(table_interp->elems + index);
function_count =
((WASMModuleInstance *)table->inst_comm_rt)->e->function_count;
}
@ -4026,7 +4046,7 @@ wasm_table_set(wasm_table_t *table, wasm_table_size_t index,
return false;
}
p_ref_idx = table_aot->elems + index;
p_ref_idx = (uint32 *)(table_aot->elems + index);
function_count = module_aot->func_count;
}
#endif
@ -4062,6 +4082,22 @@ wasm_table_set(wasm_table_t *table, wasm_table_size_t index,
return true;
}
#else /* else of WASM_ENABLE_GC == 0 */
own wasm_ref_t *
wasm_table_get(const wasm_table_t *table, wasm_table_size_t index)
{
/* TODO */
return NULL;
}
bool
wasm_table_set(wasm_table_t *table, wasm_table_size_t index,
own wasm_ref_t *ref)
{
/* TODO */
return false;
}
#endif /* end of WASM_ENABLE_GC == 0 */
wasm_table_size_t
wasm_table_size(const wasm_table_t *table)

31
core/iwasm/common/wasm_exec_env.c
View File

@ -5,6 +5,9 @@
#include "wasm_exec_env.h"
#include "wasm_runtime_common.h"
#if WASM_ENABLE_GC != 0
#include "mem_alloc.h"
#endif
#if WASM_ENABLE_INTERP != 0
#include "../interpreter/wasm_runtime.h"
#endif
@ -28,7 +31,7 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
uint32 stack_size)
{
uint64 total_size =
offsetof(WASMExecEnv, wasm_stack.s.bottom) + (uint64)stack_size;
offsetof(WASMExecEnv, wasm_stack_u.bottom) + (uint64)stack_size;
WASMExecEnv *exec_env;
if (total_size >= UINT32_MAX
@ -65,9 +68,10 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
exec_env->module_inst = module_inst;
exec_env->wasm_stack_size = stack_size;
exec_env->wasm_stack.s.top_boundary =
exec_env->wasm_stack.s.bottom + stack_size;
exec_env->wasm_stack.s.top = exec_env->wasm_stack.s.bottom;
exec_env->wasm_stack.bottom = exec_env->wasm_stack_u.bottom;
exec_env->wasm_stack.top_boundary =
exec_env->wasm_stack.bottom + stack_size;
exec_env->wasm_stack.top = exec_env->wasm_stack.bottom;
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
@ -134,6 +138,9 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst,
#endif
WASMExecEnv *exec_env =
wasm_exec_env_create_internal(module_inst, stack_size);
#if WASM_ENABLE_GC != 0
void *gc_heap_handle = NULL;
#endif
if (!exec_env)
return NULL;
@ -145,6 +152,10 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst,
exec_env->aux_stack_bottom.bottom = module->aux_stack_bottom;
exec_env->aux_stack_boundary.boundary =
module->aux_stack_bottom - module->aux_stack_size;
#if WASM_ENABLE_GC != 0
gc_heap_handle =
((WASMModuleInstance *)module_inst)->e->common.gc_heap_pool;
#endif
}
#endif
#if WASM_ENABLE_AOT != 0
@ -155,6 +166,11 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst,
exec_env->aux_stack_bottom.bottom = module->aux_stack_bottom;
exec_env->aux_stack_boundary.boundary =
module->aux_stack_bottom - module->aux_stack_size;
#if WASM_ENABLE_GC != 0
gc_heap_handle =
((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e)
->common.gc_heap_handle;
#endif
}
#endif
@ -164,6 +180,13 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst,
wasm_exec_env_destroy_internal(exec_env);
return NULL;
}
#if WASM_ENABLE_GC != 0
mem_allocator_enable_gc_reclaim(gc_heap_handle, cluster);
#endif
#else
#if WASM_ENABLE_GC != 0
mem_allocator_enable_gc_reclaim(gc_heap_handle, exec_env);
#endif
#endif /* end of WASM_ENABLE_THREAD_MGR */
return exec_env;

50
core/iwasm/common/wasm_exec_env.h
View File

@ -38,8 +38,8 @@ typedef struct WASMExecEnv {
/* Next thread's exec env of a WASM module instance. */
struct WASMExecEnv *next;
/* Previous thread's exec env of a WASM module instance. */
struct WASMExecEnv *prev;
/* Current interpreter/AOT frame of current thread */
struct WASMInterpFrame *cur_frame;
/* Note: field module_inst, argv_buf, native_stack_boundary,
suspend_flags, aux_stack_boundary, aux_stack_bottom, and
@ -84,6 +84,15 @@ typedef struct WASMExecEnv {
*/
uint8 *native_stack_top_min;
struct {
/* The top boundary of the stack. */
uint8 *top_boundary;
/* The top to of the wasm stack which is free. */
uint8 *top;
/* The bottom of the wasm stack. */
uint8 *bottom;
} wasm_stack;
#if WASM_ENABLE_FAST_JIT != 0
/**
* Cache for
@ -116,6 +125,11 @@ typedef struct WASMExecEnv {
bool thread_is_detached;
#endif
#if WASM_ENABLE_GC != 0
/* Current local object reference variable */
struct WASMLocalObjectRef *cur_local_object_ref;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
WASMCurrentEnvStatus *current_status;
#endif
@ -125,9 +139,6 @@ typedef struct WASMExecEnv {
void *user_data;
/* Current interpreter frame of current thread */
struct WASMInterpFrame *cur_frame;
/* The native thread handle of current thread */
korp_tid handle;
@ -151,18 +162,9 @@ typedef struct WASMExecEnv {
/* The WASM stack of current thread */
union {
uint64 __make_it_8_byte_aligned_;
struct {
/* The top boundary of the stack. */
uint8 *top_boundary;
/* Top cell index which is free. */
uint8 *top;
/* The WASM stack. */
uint8 bottom[1];
} s;
} wasm_stack;
/* The WASM stack. */
uint8 bottom[1];
} wasm_stack_u;
} WASMExecEnv;
#if WASM_ENABLE_MEMORY_PROFILING != 0
@ -209,7 +211,7 @@ wasm_exec_env_is_aux_stack_managed_by_runtime(WASMExecEnv *exec_env)
static inline void *
wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size)
{
uint8 *addr = exec_env->wasm_stack.s.top;
uint8 *addr = exec_env->wasm_stack.top;
bh_assert(!(size & 3));
@ -220,17 +222,17 @@ wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size)
frame size, we should check again before putting the function arguments
into the outs area. */
if (size * 2
> (uint32)(uintptr_t)(exec_env->wasm_stack.s.top_boundary - addr)) {
> (uint32)(uintptr_t)(exec_env->wasm_stack.top_boundary - addr)) {
/* WASM stack overflow. */
return NULL;
}
exec_env->wasm_stack.s.top += size;
exec_env->wasm_stack.top += size;
#if WASM_ENABLE_MEMORY_PROFILING != 0
{
uint32 wasm_stack_used =
exec_env->wasm_stack.s.top - exec_env->wasm_stack.s.bottom;
exec_env->wasm_stack.top - exec_env->wasm_stack.bottom;
if (wasm_stack_used > exec_env->max_wasm_stack_used)
exec_env->max_wasm_stack_used = wasm_stack_used;
}
@ -241,8 +243,8 @@ wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size)
static inline void
wasm_exec_env_free_wasm_frame(WASMExecEnv *exec_env, void *prev_top)
{
bh_assert((uint8 *)prev_top >= exec_env->wasm_stack.s.bottom);
exec_env->wasm_stack.s.top = (uint8 *)prev_top;
bh_assert((uint8 *)prev_top >= exec_env->wasm_stack.bottom);
exec_env->wasm_stack.top = (uint8 *)prev_top;
}
/**
@ -255,7 +257,7 @@ wasm_exec_env_free_wasm_frame(WASMExecEnv *exec_env, void *prev_top)
static inline void *
wasm_exec_env_wasm_stack_top(WASMExecEnv *exec_env)
{
return exec_env->wasm_stack.s.top;
return exec_env->wasm_stack.top;
}
/**

21
core/iwasm/common/wasm_native.c
View File

@ -81,7 +81,17 @@ compare_type_with_signautre(uint8 type, const char signature)
}
#if WASM_ENABLE_REF_TYPES != 0
if ('r' == signature && type == VALUE_TYPE_EXTERNREF)
if ('r' == signature
#if WASM_ENABLE_GC != 0
#if WASM_ENABLE_STRINGREF != 0
&& (type >= REF_TYPE_STRINGVIEWITER && type <= REF_TYPE_FUNCREF)
#else
&& (type >= REF_TYPE_NULLREF && type <= REF_TYPE_FUNCREF)
#endif
#else
&& type == VALUE_TYPE_EXTERNREF
#endif
)
return true;
#endif
@ -90,7 +100,7 @@ compare_type_with_signautre(uint8 type, const char signature)
}
static bool
check_symbol_signature(const WASMType *type, const char *signature)
check_symbol_signature(const WASMFuncType *type, const char *signature)
{
const char *p = signature, *p_end;
char sig;
@ -189,8 +199,9 @@ lookup_symbol(NativeSymbol *native_symbols, uint32 n_native_symbols,
*/
void *
wasm_native_resolve_symbol(const char *module_name, const char *field_name,
const WASMType *func_type, const char **p_signature,
void **p_attachment, bool *p_call_conv_raw)
const WASMFuncType *func_type,
const char **p_signature, void **p_attachment,
bool *p_call_conv_raw)
{
NativeSymbolsNode *node, *node_next;
const char *signature = NULL;
@ -1442,7 +1453,7 @@ quick_aot_entry_init()
}
void *
wasm_native_lookup_quick_aot_entry(const WASMType *func_type)
wasm_native_lookup_quick_aot_entry(const WASMFuncType *func_type)
{
char signature[16] = { 0 };
uint32 param_count = func_type->param_count;

7
core/iwasm/common/wasm_native.h
View File

@ -51,8 +51,9 @@ wasm_native_lookup_libc_builtin_global(const char *module_name,
*/
void *
wasm_native_resolve_symbol(const char *module_name, const char *field_name,
const WASMType *func_type, const char **p_signature,
void **p_attachment, bool *p_call_conv_raw);
const WASMFuncType *func_type,
const char **p_signature, void **p_attachment,
bool *p_call_conv_raw);
bool
wasm_native_register_natives(const char *module_name,
@ -106,7 +107,7 @@ wasm_native_destroy();
#if WASM_ENABLE_QUICK_AOT_ENTRY != 0
void *
wasm_native_lookup_quick_aot_entry(const WASMType *func_type);
wasm_native_lookup_quick_aot_entry(const WASMFuncType *func_type);
#endif
#ifdef __cplusplus

462
core/iwasm/common/wasm_runtime_common.c
View File

@ -19,6 +19,9 @@
#include "../aot/debug/jit_debug.h"
#endif
#endif
#if WASM_ENABLE_GC != 0
#include "gc/gc_object.h"
#endif
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#if WASM_ENABLE_DEBUG_INTERP != 0
@ -88,7 +91,7 @@ wasm_runtime_destroy_registered_module_list();
#define E_TYPE_XIP 4
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
/* Initialize externref hashmap */
static bool
wasm_externref_map_init();
@ -96,7 +99,7 @@ wasm_externref_map_init();
/* Destroy externref hashmap */
static void
wasm_externref_map_destroy();
#endif /* WASM_ENABLE_REF_TYPES */
#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */
static void
set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
@ -163,6 +166,10 @@ static JitCompOptions jit_options = { 0 };
static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false };
#endif
#if WASM_ENABLE_GC != 0
static uint32 gc_heap_size_default = GC_HEAP_SIZE_DEFAULT;
#endif
static RunningMode runtime_running_mode = Mode_Default;
#ifdef OS_ENABLE_HW_BOUND_CHECK
@ -469,7 +476,7 @@ wasm_runtime_env_init()
#endif
#endif
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
if (!wasm_externref_map_init()) {
goto fail8;
}
@ -510,11 +517,11 @@ fail10:
#endif
#if WASM_ENABLE_FAST_JIT != 0
fail9:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
wasm_externref_map_destroy();
#endif
#endif
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
fail8:
#endif
#if WASM_ENABLE_AOT != 0
@ -552,9 +559,9 @@ static bool
wasm_runtime_exec_env_check(WASMExecEnv *exec_env)
{
return exec_env && exec_env->module_inst && exec_env->wasm_stack_size > 0
&& exec_env->wasm_stack.s.top_boundary
== exec_env->wasm_stack.s.bottom + exec_env->wasm_stack_size
&& exec_env->wasm_stack.s.top <= exec_env->wasm_stack.s.top_boundary;
&& exec_env->wasm_stack.top_boundary
== exec_env->wasm_stack.bottom + exec_env->wasm_stack_size
&& exec_env->wasm_stack.top <= exec_env->wasm_stack.top_boundary;
}
bool
@ -574,7 +581,7 @@ wasm_runtime_init()
void
wasm_runtime_destroy()
{
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
wasm_externref_map_destroy();
#endif
@ -647,6 +654,14 @@ wasm_runtime_get_llvm_jit_options(void)
}
#endif
#if WASM_ENABLE_GC != 0
uint32
wasm_runtime_get_gc_heap_size_default(void)
{
return gc_heap_size_default;
}
#endif
bool
wasm_runtime_full_init(RuntimeInitArgs *init_args)
{
@ -663,6 +678,10 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args)
jit_options.code_cache_size = init_args->fast_jit_code_cache_size;
#endif
#if WASM_ENABLE_GC != 0
gc_heap_size_default = init_args->gc_heap_size;
#endif
#if WASM_ENABLE_JIT != 0
llvm_jit_options.size_level = init_args->llvm_jit_size_level;
llvm_jit_options.opt_level = init_args->llvm_jit_opt_level;
@ -1572,11 +1591,11 @@ void
wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env)
{
uint32 total_size =
offsetof(WASMExecEnv, wasm_stack.s.bottom) + exec_env->wasm_stack_size;
offsetof(WASMExecEnv, wasm_stack_u.bottom) + exec_env->wasm_stack_size;
os_printf("Exec env memory consumption, total size: %u\n", total_size);
os_printf(" exec env struct size: %u\n",
offsetof(WASMExecEnv, wasm_stack.s.bottom));
offsetof(WASMExecEnv, wasm_stack_u.bottom));
#if WASM_ENABLE_INTERP != 0 && WASM_ENABLE_FAST_INTERP == 0
os_printf(" block addr cache size: %u\n",
sizeof(exec_env->block_addr_cache));
@ -1637,7 +1656,7 @@ wasm_runtime_dump_mem_consumption(WASMExecEnv *exec_env)
app_heap_peak_size = gc_get_heap_highmark_size(heap_handle);
}
total_size = offsetof(WASMExecEnv, wasm_stack.s.bottom)
total_size = offsetof(WASMExecEnv, wasm_stack_u.bottom)
+ exec_env->wasm_stack_size + module_mem_consps.total_size
+ module_inst_mem_consps.total_size;
@ -1771,11 +1790,11 @@ wasm_runtime_access_exce_check_guard_page()
}
#endif
WASMType *
WASMFuncType *
wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function,
uint32 module_type)
{
WASMType *type = NULL;
WASMFuncType *type = NULL;
#if WASM_ENABLE_INTERP != 0
if (module_type == Wasm_Module_Bytecode) {
@ -1816,7 +1835,7 @@ uint32
wasm_func_get_param_count(WASMFunctionInstanceCommon *const func_inst,
WASMModuleInstanceCommon *const module_inst)
{
WASMType *type =
WASMFuncType *type =
wasm_runtime_get_function_type(func_inst, module_inst->module_type);
bh_assert(type);
@ -1827,7 +1846,7 @@ uint32
wasm_func_get_result_count(WASMFunctionInstanceCommon *const func_inst,
WASMModuleInstanceCommon *const module_inst)
{
WASMType *type =
WASMFuncType *type =
wasm_runtime_get_function_type(func_inst, module_inst->module_type);
bh_assert(type);
@ -1861,7 +1880,7 @@ wasm_func_get_param_types(WASMFunctionInstanceCommon *const func_inst,
WASMModuleInstanceCommon *const module_inst,
wasm_valkind_t *param_types)
{
WASMType *type =
WASMFuncType *type =
wasm_runtime_get_function_type(func_inst, module_inst->module_type);
uint32 i;
@ -1877,7 +1896,7 @@ wasm_func_get_result_types(WASMFunctionInstanceCommon *const func_inst,
WASMModuleInstanceCommon *const module_inst,
wasm_valkind_t *result_types)
{
WASMType *type =
WASMFuncType *type =
wasm_runtime_get_function_type(func_inst, module_inst->module_type);
uint32 i;
@ -1889,7 +1908,7 @@ wasm_func_get_result_types(WASMFunctionInstanceCommon *const func_inst,
}
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
/* (uintptr_t)externref -> (uint32)index */
/* argv -> *ret_argv */
static bool
@ -1903,7 +1922,7 @@ wasm_runtime_prepare_call_function(WASMExecEnv *exec_env,
result_i = 0;
bool need_param_transform = false, need_result_transform = false;
uint64 size = 0;
WASMType *func_type = wasm_runtime_get_function_type(
WASMFuncType *func_type = wasm_runtime_get_function_type(
function, exec_env->module_inst->module_type);
bh_assert(func_type);
@ -1996,7 +2015,7 @@ wasm_runtime_finalize_call_function(WASMExecEnv *exec_env,
uint32 *argv, uint32 argc, uint32 *ret_argv)
{
uint32 argv_i = 0, result_i = 0, ret_argv_i = 0;
WASMType *func_type;
WASMFuncType *func_type;
bh_assert((argv && ret_argv) || (argc == 0));
@ -2058,7 +2077,7 @@ wasm_runtime_call_wasm(WASMExecEnv *exec_env,
{
bool ret = false;
uint32 *new_argv = NULL, param_argc;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
uint32 result_argc = 0;
#endif
@ -2067,7 +2086,7 @@ wasm_runtime_call_wasm(WASMExecEnv *exec_env,
return false;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
if (!wasm_runtime_prepare_call_function(exec_env, function, argv, argc,
&new_argv, &param_argc,
&result_argc)) {
@ -2097,7 +2116,7 @@ wasm_runtime_call_wasm(WASMExecEnv *exec_env,
return false;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
if (!wasm_runtime_finalize_call_function(exec_env, function, new_argv,
result_argc, argv)) {
wasm_runtime_set_exception(exec_env->module_inst,
@ -2110,7 +2129,8 @@ wasm_runtime_call_wasm(WASMExecEnv *exec_env,
}
static void
parse_args_to_uint32_array(WASMType *type, wasm_val_t *args, uint32 *out_argv)
parse_args_to_uint32_array(WASMFuncType *type, wasm_val_t *args,
uint32 *out_argv)
{
uint32 i, p;
@ -2152,11 +2172,15 @@ parse_args_to_uint32_array(WASMType *type, wasm_val_t *args, uint32 *out_argv)
break;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0
case WASM_FUNCREF:
{
out_argv[p++] = args[i].of.i32;
break;
}
#else
case WASM_FUNCREF:
#endif
case WASM_ANYREF:
{
#if UINTPTR_MAX == UINT32_MAX
@ -2182,7 +2206,7 @@ parse_args_to_uint32_array(WASMType *type, wasm_val_t *args, uint32 *out_argv)
}
static void
parse_uint32_array_to_results(WASMType *type, uint32 *argv,
parse_uint32_array_to_results(WASMFuncType *type, uint32 *argv,
wasm_val_t *out_results)
{
uint32 i, p;
@ -2229,6 +2253,7 @@ parse_uint32_array_to_results(WASMType *type, uint32 *argv,
break;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0
case VALUE_TYPE_FUNCREF:
{
out_results[i].kind = WASM_I32;
@ -2236,6 +2261,20 @@ parse_uint32_array_to_results(WASMType *type, uint32 *argv,
break;
}
case VALUE_TYPE_EXTERNREF:
#else
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#endif /* end of WASM_ENABLE_GC == 0 */
{
#if UINTPTR_MAX == UINT32_MAX
out_results[i].kind = WASM_ANYREF;
@ -2252,7 +2291,7 @@ parse_uint32_array_to_results(WASMType *type, uint32 *argv,
#endif
break;
}
#endif
#endif /* end of WASM_ENABLE_REF_TYPES != 0 */
default:
bh_assert(0);
break;
@ -2267,11 +2306,11 @@ wasm_runtime_call_wasm_a(WASMExecEnv *exec_env,
uint32 num_args, wasm_val_t args[])
{
uint32 argc, argv_buf[16] = { 0 }, *argv = argv_buf, cell_num, module_type;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
uint32 i, param_size_in_double_world = 0, result_size_in_double_world = 0;
#endif
uint64 total_size;
WASMType *type;
WASMFuncType *type;
bool ret = false;
module_type = exec_env->module_inst->module_type;
@ -2283,7 +2322,7 @@ wasm_runtime_call_wasm_a(WASMExecEnv *exec_env,
goto fail1;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
for (i = 0; i < type->param_count; i++) {
param_size_in_double_world +=
wasm_value_type_cell_num_outside(type->types[i]);
@ -2341,7 +2380,7 @@ wasm_runtime_call_wasm_v(WASMExecEnv *exec_env,
uint32 num_args, ...)
{
wasm_val_t args_buf[8] = { 0 }, *args = args_buf;
WASMType *type = NULL;
WASMFuncType *type = NULL;
bool ret = false;
uint64 total_size;
uint32 i = 0, module_type;
@ -2389,7 +2428,7 @@ wasm_runtime_call_wasm_v(WASMExecEnv *exec_env,
args[i].kind = WASM_F64;
args[i].of.f64 = va_arg(vargs, float64);
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
{
args[i].kind = WASM_FUNCREF;
@ -2506,6 +2545,23 @@ static const char *exception_msgs[] = {
"out of bounds table access", /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */
"wasm operand stack overflow", /* EXCE_OPERAND_STACK_OVERFLOW */
"failed to compile fast jit function", /* EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC */
/* GC related exceptions */
"null function object", /* EXCE_NULL_FUNC_OBJ */
"null structure object", /* EXCE_NULL_STRUCT_OBJ */
"null array reference", /* EXCE_NULL_ARRAY_OBJ */
"null i31 reference", /* EXCE_NULL_I31_OBJ */
"null reference", /* EXCE_NULL_REFERENCE */
"create rtt type failed", /* EXCE_FAILED_TO_CREATE_RTT_TYPE */
"create struct object failed", /* EXCE_FAILED_TO_CREATE_STRUCT_OBJ */
"create array object failed", /* EXCE_FAILED_TO_CREATE_ARRAY_OBJ */
"create externref object failed", /* EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ */
"cast failure", /* EXCE_CAST_FAILURE */
"out of bounds array access", /* EXCE_ARRAY_IDX_OOB */
/* stringref related exceptions */
"create string object failed", /* EXCE_FAILED_TO_CREATE_STRING */
"create stringref failed", /* EXCE_FAILED_TO_CREATE_STRINGREF */
"create stringview failed", /* EXCE_FAILED_TO_CREATE_STRINGVIEW */
"encode failed", /* EXCE_FAILED_TO_ENCODE_STRING */
"", /* EXCE_ALREADY_THROWN */
};
/* clang-format on */
@ -3568,9 +3624,9 @@ wasm_runtime_unregister_natives(const char *module_name,
bool
wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type, const char *signature,
void *attachment, uint32 *argv, uint32 argc,
uint32 *argv_ret)
const WASMFuncType *func_type,
const char *signature, void *attachment,
uint32 *argv, uint32 argc, uint32 *argv_ret)
{
WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env);
typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *);
@ -3595,7 +3651,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
for (i = 0; i < func_type->param_count; i++, argv_dst++) {
switch (func_type->types[i]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
{
@ -3640,7 +3696,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
case VALUE_TYPE_F32:
*(float32 *)argv_dst = *(float32 *)argv_src++;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
uint32 externref_idx = *argv_src++;
@ -3654,6 +3710,32 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
sizeof(uintptr_t));
break;
}
#endif
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
{
bh_memcpy_s(argv_dst, sizeof(uintptr_t), argv_src,
sizeof(uintptr_t));
argv_src += sizeof(uintptr_t) / sizeof(uint32);
break;
}
#endif
default:
bh_assert(0);
@ -3668,7 +3750,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
if (func_type->result_count > 0) {
switch (func_type->types[func_type->param_count]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
argv_ret[0] = *(uint32 *)argv1;
@ -3681,7 +3763,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
bh_memcpy_s(argv_ret, sizeof(uint32) * 2, argv1,
sizeof(uint64));
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
uint32 externref_idx;
@ -3697,6 +3779,31 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
argv_ret[0] = externref_idx;
break;
}
#endif
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
{
bh_memcpy_s(argv_ret, sizeof(uintptr_t), argv1,
sizeof(uintptr_t));
break;
}
#endif
default:
bh_assert(0);
@ -3751,7 +3858,7 @@ static volatile VoidFuncPtr invokeNative_Void =
bool
wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type, const char *signature,
const WASMFuncType *func_type, const char *signature,
void *attachment, uint32 *argv, uint32 argc,
uint32 *argv_ret)
{
@ -3764,7 +3871,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
uint32 result_count = func_type->result_count;
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
bool ret = false;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
bool is_aot_func = (NULL == signature);
#endif
#if !defined(BUILD_TARGET_RISCV32_ILP32) && !defined(BUILD_TARGET_ARC)
@ -3781,7 +3888,27 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
for (i = 0; i < func_type->param_count; i++) {
switch (func_type->types[i]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
#endif
@ -3925,7 +4052,27 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
for (i = 0; i < func_type->param_count; i++) {
switch (func_type->types[i]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
{
@ -4082,7 +4229,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
break;
}
#endif /* BUILD_TARGET_RISCV32_ILP32D */
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
uint32 externref_idx = *argv_src++;
@ -4128,7 +4275,27 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
else {
switch (func_type->types[func_type->param_count]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
argv_ret[0] =
@ -4146,7 +4313,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
PUT_F64_TO_ADDR(
argv_ret, invokeNative_Float64(func_ptr, argv1, n_stacks));
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
if (is_aot_func) {
@ -4223,7 +4390,7 @@ word_copy(uint32 *dest, uint32 *src, unsigned num)
bool
wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type, const char *signature,
const WASMFuncType *func_type, const char *signature,
void *attachment, uint32 *argv, uint32 argc,
uint32 *argv_ret)
{
@ -4234,7 +4401,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
uint64 size;
bool ret = false;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
bool is_aot_func = (NULL == signature);
#endif
@ -4260,7 +4427,27 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
for (i = 0; i < func_type->param_count; i++) {
switch (func_type->types[i]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
{
@ -4311,7 +4498,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
case VALUE_TYPE_F32:
argv1[j++] = *argv++;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
uint32 externref_idx = *argv++;
@ -4346,7 +4533,27 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
else {
switch (func_type->types[func_type->param_count]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
argv_ret[0] =
@ -4364,7 +4571,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
PUT_F64_TO_ADDR(argv_ret,
invokeNative_Float64(func_ptr, argv1, argc1));
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
if (is_aot_func) {
@ -4491,7 +4698,7 @@ __attribute__((no_sanitize_address))
#endif
bool
wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type, const char *signature,
const WASMFuncType *func_type, const char *signature,
void *attachment, uint32 *argv, uint32 argc,
uint32 *argv_ret)
{
@ -4503,7 +4710,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
uint32 result_count = func_type->result_count;
uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
bool ret = false;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
bool is_aot_func = (NULL == signature);
#endif
#ifndef BUILD_TARGET_RISCV64_LP64
@ -4555,7 +4762,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
for (i = 0; i < func_type->param_count; i++) {
switch (func_type->types[i]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
{
@ -4595,6 +4802,26 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
break;
}
case VALUE_TYPE_I64:
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
if (n_ints < MAX_REG_INTS)
ints[n_ints++] = *(uint64 *)argv_src;
else
@ -4618,7 +4845,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
}
argv_src += 2;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
uint32 externref_idx = *argv_src++;
@ -4677,13 +4904,33 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
/* Invoke the native function and get the first result value */
switch (func_type->types[func_type->param_count]) {
case VALUE_TYPE_I32:
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
#endif
argv_ret[0] =
(uint32)invokeNative_Int32(func_ptr, argv1, n_stacks);
break;
case VALUE_TYPE_I64:
#if WASM_ENABLE_GC != 0
case REF_TYPE_FUNCREF:
case REF_TYPE_EXTERNREF:
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
case REF_TYPE_NULLREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
#endif
PUT_I64_TO_ADDR(argv_ret,
invokeNative_Int64(func_ptr, argv1, n_stacks));
break;
@ -4695,7 +4942,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
PUT_F64_TO_ADDR(
argv_ret, invokeNative_Float64(func_ptr, argv1, n_stacks));
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
{
if (is_aot_func) {
@ -4887,7 +5134,7 @@ wasm_runtime_join_thread(wasm_thread_t tid, void **retval)
#endif /* end of WASM_ENABLE_THREAD_MGR */
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
static korp_mutex externref_lock;
static uint32 externref_global_id = 1;
@ -5167,7 +5414,8 @@ mark_externref(uint32 externref_idx)
static void
interp_mark_all_externrefs(WASMModuleInstance *module_inst)
{
uint32 i, j, externref_idx, *table_data;
uint32 i, j, externref_idx;
table_elem_type_t *table_data;
uint8 *global_data = module_inst->global_data;
WASMGlobalInstance *global;
WASMTableInstance *table;
@ -5289,7 +5537,7 @@ wasm_externref_retain(uint32 externref_idx)
os_mutex_unlock(&externref_lock);
return false;
}
#endif /* end of WASM_ENABLE_REF_TYPES */
#endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */
#if WASM_ENABLE_DUMP_CALL_STACK != 0
uint32
@ -5405,6 +5653,9 @@ wasm_runtime_dump_pgo_prof_data_to_buf(WASMModuleInstanceCommon *module_inst,
bool
wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
uint32 table_idx, uint8 *out_elem_type,
#if WASM_ENABLE_GC != 0
WASMRefType **out_ref_type,
#endif
uint32 *out_min_size, uint32 *out_max_size)
{
#if WASM_ENABLE_INTERP != 0
@ -5415,6 +5666,9 @@ wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
WASMTableImport *import_table =
&((module->import_tables + table_idx)->u.table);
*out_elem_type = import_table->elem_type;
#if WASM_ENABLE_GC != 0
*out_ref_type = import_table->elem_ref_type;
#endif
*out_min_size = import_table->init_size;
*out_max_size = import_table->max_size;
}
@ -5422,6 +5676,9 @@ wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
WASMTable *table =
module->tables + (table_idx - module->import_table_count);
*out_elem_type = table->elem_type;
#if WASM_ENABLE_GC != 0
*out_ref_type = table->elem_ref_type;
#endif
*out_min_size = table->init_size;
*out_max_size = table->max_size;
}
@ -5435,7 +5692,10 @@ wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
if (table_idx < module->import_table_count) {
AOTImportTable *import_table = module->import_tables + table_idx;
*out_elem_type = VALUE_TYPE_FUNCREF;
*out_elem_type = import_table->elem_type;
#if WASM_ENABLE_GC != 0
*out_ref_type = NULL; /* TODO */
#endif
*out_min_size = import_table->table_init_size;
*out_max_size = import_table->table_max_size;
}
@ -5443,6 +5703,9 @@ wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
AOTTable *table =
module->tables + (table_idx - module->import_table_count);
*out_elem_type = table->elem_type;
#if WASM_ENABLE_GC != 0
*out_ref_type = NULL; /* TODO */
#endif
*out_min_size = table->table_init_size;
*out_max_size = table->table_max_size;
}
@ -5456,31 +5719,28 @@ wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
bool
wasm_runtime_get_table_inst_elem_type(
const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx,
uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size)
uint8 *out_elem_type,
#if WASM_ENABLE_GC != 0
WASMRefType **out_ref_type,
#endif
uint32 *out_min_size, uint32 *out_max_size)
{
#if WASM_ENABLE_INTERP != 0
if (module_inst_comm->module_type == Wasm_Module_Bytecode) {
WASMModuleInstance *module_inst =
(WASMModuleInstance *)module_inst_comm;
return wasm_runtime_get_table_elem_type(
(WASMModuleCommon *)module_inst->module, table_idx, out_elem_type,
out_min_size, out_max_size);
}
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode
|| module_inst_comm->module_type == Wasm_Module_AoT);
return wasm_runtime_get_table_elem_type(
(WASMModuleCommon *)module_inst->module, table_idx, out_elem_type,
#if WASM_ENABLE_GC != 0
out_ref_type,
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst_comm->module_type == Wasm_Module_AoT) {
AOTModuleInstance *module_inst = (AOTModuleInstance *)module_inst_comm;
return wasm_runtime_get_table_elem_type(
(WASMModuleCommon *)module_inst->module, table_idx, out_elem_type,
out_min_size, out_max_size);
}
#endif
return false;
out_min_size, out_max_size);
}
bool
wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm,
const WASMExport *export, WASMType **out)
const WASMExport *export, WASMFuncType **out)
{
#if WASM_ENABLE_INTERP != 0
if (module_comm->module_type == Wasm_Module_Bytecode) {
@ -5503,13 +5763,14 @@ wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm,
AOTModule *module = (AOTModule *)module_comm;
if (export->index < module->import_func_count) {
*out = module->func_types[module->import_funcs[export->index]
.func_type_index];
*out = (WASMFuncType *)
module->types[module->import_funcs[export->index]
.func_type_index];
}
else {
*out = module->func_types
[module->func_type_indexes[export->index
- module->import_func_count]];
*out = (WASMFuncType *)module
->types[module->func_type_indexes
[export->index - module->import_func_count]];
}
return true;
}
@ -5615,15 +5876,23 @@ wasm_runtime_get_export_memory_type(const WASMModuleCommon *module_comm,
bool
wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm,
const WASMExport *export,
uint8 *out_elem_type, uint32 *out_min_size,
uint32 *out_max_size)
uint8 *out_elem_type,
#if WASM_ENABLE_GC != 0
WASMRefType **out_ref_type,
#endif
uint32 *out_min_size, uint32 *out_max_size)
{
return wasm_runtime_get_table_elem_type(
module_comm, export->index, out_elem_type, out_min_size, out_max_size);
return wasm_runtime_get_table_elem_type(module_comm, export->index,
out_elem_type,
#if WASM_ENABLE_GC != 0
out_ref_type,
#endif
out_min_size, out_max_size);
}
static inline bool
argv_to_params(wasm_val_t *out_params, const uint32 *argv, WASMType *func_type)
argv_to_params(wasm_val_t *out_params, const uint32 *argv,
WASMFuncType *func_type)
{
wasm_val_t *param = out_params;
uint32 i = 0, *u32;
@ -5650,7 +5919,7 @@ argv_to_params(wasm_val_t *out_params, const uint32 *argv, WASMType *func_type)
u32[0] = *argv++;
u32[1] = *argv++;
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
param->kind = WASM_ANYREF;
@ -5672,7 +5941,7 @@ argv_to_params(wasm_val_t *out_params, const uint32 *argv, WASMType *func_type)
static inline bool
results_to_argv(WASMModuleInstanceCommon *module_inst, uint32 *out_argv,
const wasm_val_t *results, WASMType *func_type)
const wasm_val_t *results, WASMFuncType *func_type)
{
const wasm_val_t *result = results;
uint32 *argv = out_argv, *u32, i;
@ -5690,10 +5959,11 @@ results_to_argv(WASMModuleInstanceCommon *module_inst, uint32 *out_argv,
*argv++ = u32[0];
*argv++ = u32[1];
break;
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_EXTERNREF:
if (!wasm_externref_obj2ref(module_inst,
(void *)result->of.foreign, argv)) {
(void *)result->of.foreign,
(uint32 *)argv)) {
return false;
}
argv++;
@ -5709,7 +5979,7 @@ results_to_argv(WASMModuleInstanceCommon *module_inst, uint32 *out_argv,
bool
wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst,
void *func_ptr, WASMType *func_type,
void *func_ptr, WASMFuncType *func_type,
uint32 argc, uint32 *argv, bool with_env,
void *wasm_c_api_env)
{

92
core/iwasm/common/wasm_runtime_common.h
View File

@ -12,6 +12,9 @@
#include "wasm_native.h"
#include "../include/wasm_export.h"
#include "../interpreter/wasm.h"
#if WASM_ENABLE_GC != 0
#include "gc/gc_object.h"
#endif
#if WASM_ENABLE_LIBC_WASI != 0
#if WASM_ENABLE_UVWASI == 0
@ -38,9 +41,14 @@ extern "C" {
do { \
*(float64 *)(addr) = (float64)(value); \
} while (0)
#define PUT_REF_TO_ADDR(addr, value) \
do { \
*(void **)(addr) = (void *)(value); \
} while (0)
#define GET_I64_FROM_ADDR(addr) (*(int64 *)(addr))
#define GET_F64_FROM_ADDR(addr) (*(float64 *)(addr))
#define GET_REF_FROM_ADDR(addr) (*(void **)(addr))
/* For STORE opcodes */
#define STORE_I64 PUT_I64_TO_ADDR
@ -97,6 +105,24 @@ STORE_U8(void *addr, uint8_t value)
addr_u32[0] = u.parts[0]; \
addr_u32[1] = u.parts[1]; \
} while (0)
#if UINTPTR_MAX == UINT64_MAX
#define PUT_REF_TO_ADDR(addr, value) \
do { \
uint32 *addr_u32 = (uint32 *)(addr); \
union { \
void *val; \
uint32 parts[2]; \
} u; \
u.val = (void *)(value); \
addr_u32[0] = u.parts[0]; \
addr_u32[1] = u.parts[1]; \
} while (0)
#else
#define PUT_REF_TO_ADDR(addr, value) \
do { \
*(void **)(addr) = (void *)(value); \
} while (0)
#endif
static inline int64
GET_I64_FROM_ADDR(uint32 *addr)
@ -122,6 +148,22 @@ GET_F64_FROM_ADDR(uint32 *addr)
return u.val;
}
#if UINTPTR_MAX == UINT64_MAX
static inline void *
GET_REF_FROM_ADDR(uint32 *addr)
{
union {
void *val;
uint32 parts[2];
} u;
u.parts[0] = addr[0];
u.parts[1] = addr[1];
return u.val;
}
#else
#define GET_REF_FROM_ADDR(addr) (*(void **)(addr))
#endif
/* For STORE opcodes */
#define STORE_I64(addr, value) \
do { \
@ -426,6 +468,10 @@ typedef struct wasm_frame_t {
uint32 func_index;
uint32 func_offset;
const char *func_name_wp;
uint32 *sp;
uint8 *frame_ref;
uint32 *lp;
} WASMCApiFrame;
#if WASM_ENABLE_JIT != 0
@ -471,6 +517,12 @@ LLVMJITOptions *
wasm_runtime_get_llvm_jit_options(void);
#endif
#if WASM_ENABLE_GC != 0
/* Internal API */
uint32
wasm_runtime_get_gc_heap_size_default(void);
#endif
/* See wasm_export.h for description */
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_full_init(RuntimeInitArgs *init_args);
@ -551,7 +603,7 @@ wasm_runtime_lookup_function(WASMModuleInstanceCommon *const module_inst,
const char *name, const char *signature);
/* Internal API */
WASMType *
WASMFuncType *
wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function,
uint32 module_type);
@ -918,6 +970,15 @@ wasm_runtime_set_wasi_ns_lookup_pool(wasm_module_t module,
uint32 ns_lookup_pool_size);
#endif /* end of WASM_ENABLE_LIBC_WASI */
#if WASM_ENABLE_GC != 0
void
wasm_runtime_set_gc_heap_handle(WASMModuleInstanceCommon *module_inst,
void *gc_heap_handle);
void *
wasm_runtime_get_gc_heap_handle(WASMModuleInstanceCommon *module_inst);
#endif
#if WASM_ENABLE_REF_TYPES != 0
/* See wasm_export.h for description */
WASM_RUNTIME_API_EXTERN bool
@ -1011,15 +1072,15 @@ wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key);
bool
wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type, const char *signature,
const WASMFuncType *func_type, const char *signature,
void *attachment, uint32 *argv, uint32 argc,
uint32 *ret);
bool
wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type, const char *signature,
void *attachment, uint32 *argv, uint32 argc,
uint32 *ret);
const WASMFuncType *func_type,
const char *signature, void *attachment,
uint32 *argv, uint32 argc, uint32 *ret);
void
wasm_runtime_read_v128(const uint8 *bytes, uint64 *ret1, uint64 *ret2);
@ -1037,16 +1098,24 @@ wasm_runtime_dump_exec_env_mem_consumption(const WASMExecEnv *exec_env);
bool
wasm_runtime_get_table_elem_type(const WASMModuleCommon *module_comm,
uint32 table_idx, uint8 *out_elem_type,
#if WASM_ENABLE_GC != 0
WASMRefType **out_ref_type,
#endif
uint32 *out_min_size, uint32 *out_max_size);
bool
wasm_runtime_get_table_inst_elem_type(
const WASMModuleInstanceCommon *module_inst_comm, uint32 table_idx,
uint8 *out_elem_type, uint32 *out_min_size, uint32 *out_max_size);
uint8 *out_elem_type,
#if WASM_ENABLE_GC != 0
WASMRefType **out_ref_type,
#endif
uint32 *out_min_size, uint32 *out_max_size);
bool
wasm_runtime_get_export_func_type(const WASMModuleCommon *module_comm,
const WASMExport *export_, WASMType **out);
const WASMExport *export_,
WASMFuncType **out);
bool
wasm_runtime_get_export_global_type(const WASMModuleCommon *module_comm,
@ -1061,12 +1130,15 @@ wasm_runtime_get_export_memory_type(const WASMModuleCommon *module_comm,
bool
wasm_runtime_get_export_table_type(const WASMModuleCommon *module_comm,
const WASMExport *export_,
uint8 *out_elem_type, uint32 *out_min_size,
uint32 *out_max_size);
uint8 *out_elem_type,
#if WASM_ENABLE_GC != 0
WASMRefType **out_ref_type,
#endif
uint32 *out_min_size, uint32 *out_max_size);
bool
wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst,
void *func_ptr, WASMType *func_type,
void *func_ptr, WASMFuncType *func_type,
uint32 argc, uint32 *argv, bool with_env,
void *wasm_c_api_env);

399
core/iwasm/compilation/aot.c
View File

@ -114,9 +114,9 @@ aot_create_table_init_data_list(const WASMModule *module)
/* Create each table data segment */
for (i = 0; i < module->table_seg_count; i++) {
size =
offsetof(AOTTableInitData, func_indexes)
+ sizeof(uint32) * (uint64)module->table_segments[i].function_count;
size = offsetof(AOTTableInitData, init_values)
+ sizeof(InitializerExpression)
* (uint64)module->table_segments[i].value_count;
if (size >= UINT32_MAX
|| !(data_list[i] = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
@ -124,8 +124,7 @@ aot_create_table_init_data_list(const WASMModule *module)
}
data_list[i]->offset = module->table_segments[i].base_offset;
data_list[i]->func_index_count =
module->table_segments[i].function_count;
data_list[i]->value_count = module->table_segments[i].value_count;
data_list[i]->mode = module->table_segments[i].mode;
data_list[i]->elem_type = module->table_segments[i].elem_type;
/* runtime control it */
@ -133,12 +132,16 @@ aot_create_table_init_data_list(const WASMModule *module)
bh_memcpy_s(&data_list[i]->offset, sizeof(AOTInitExpr),
&module->table_segments[i].base_offset,
sizeof(AOTInitExpr));
data_list[i]->func_index_count =
module->table_segments[i].function_count;
bh_memcpy_s(data_list[i]->func_indexes,
sizeof(uint32) * module->table_segments[i].function_count,
module->table_segments[i].func_indexes,
sizeof(uint32) * module->table_segments[i].function_count);
data_list[i]->value_count = module->table_segments[i].value_count;
#if WASM_ENABLE_GC != 0
data_list[i]->elem_ref_type = module->table_segments[i].elem_ref_type;
#endif
bh_memcpy_s(data_list[i]->init_values,
sizeof(InitializerExpression)
* module->table_segments[i].value_count,
module->table_segments[i].init_values,
sizeof(InitializerExpression)
* module->table_segments[i].value_count);
}
return data_list;
@ -148,13 +151,60 @@ fail:
return NULL;
}
static void
get_value_type_size(uint8 value_type, bool gc_enabled, uint32 *p_size_64bit,
uint32 *p_size_32bit)
{
uint32 size_64bit = 0, size_32bit = 0;
if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_F32)
size_64bit = size_32bit = sizeof(int32);
else if (value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F64)
size_64bit = size_32bit = sizeof(int64);
else if (value_type == VALUE_TYPE_V128)
size_64bit = size_32bit = sizeof(int64) * 2;
else if (!gc_enabled
&& (value_type == VALUE_TYPE_FUNCREF
|| value_type == VALUE_TYPE_EXTERNREF))
size_64bit = size_32bit = sizeof(int32);
else if (gc_enabled
&& ((value_type >= (uint8)REF_TYPE_ARRAYREF
&& value_type <= (uint8)REF_TYPE_NULLFUNCREF)
|| (value_type >= (uint8)REF_TYPE_HT_NULLABLE
&& value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE)
#if WASM_ENABLE_STRINGREF != 0
|| (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8
&& value_type <= (uint8)REF_TYPE_STRINGREF)
|| (value_type >= (uint8)REF_TYPE_STRINGVIEWITER
&& value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16)
#endif
)) {
size_64bit = sizeof(uint64);
size_32bit = sizeof(uint32);
}
else if (gc_enabled && value_type == PACKED_TYPE_I8) {
size_64bit = size_32bit = sizeof(int8);
}
else if (gc_enabled && value_type == PACKED_TYPE_I16) {
size_64bit = size_32bit = sizeof(int16);
}
else {
bh_assert(0);
}
*p_size_64bit = size_64bit;
*p_size_32bit = size_32bit;
}
static AOTImportGlobal *
aot_create_import_globals(const WASMModule *module,
uint32 *p_import_global_data_size)
aot_create_import_globals(const WASMModule *module, bool gc_enabled,
uint32 *p_import_global_data_size_64bit,
uint32 *p_import_global_data_size_32bit)
{
AOTImportGlobal *import_globals;
uint64 size;
uint32 i, data_offset = 0;
uint32 i, data_offset_64bit = 0, data_offset_32bit = 0;
uint32 value_size_64bit, value_size_32bit;
/* Allocate memory */
size = sizeof(AOTImportGlobal) * (uint64)module->import_global_count;
@ -175,23 +225,37 @@ aot_create_import_globals(const WASMModule *module,
import_globals[i].is_mutable = import_global->is_mutable;
import_globals[i].global_data_linked =
import_global->global_data_linked;
import_globals[i].size = wasm_value_type_size(import_global->type);
/* Calculate data offset */
import_globals[i].data_offset = data_offset;
data_offset += wasm_value_type_size(import_global->type);
import_globals[i].data_offset_64bit = data_offset_64bit;
import_globals[i].data_offset_32bit = data_offset_32bit;
get_value_type_size(import_global->type, gc_enabled, &value_size_64bit,
&value_size_32bit);
import_globals[i].size_64bit = value_size_64bit;
import_globals[i].size_32bit = value_size_32bit;
data_offset_64bit += value_size_64bit;
data_offset_32bit += value_size_32bit;
}
*p_import_global_data_size = data_offset;
*p_import_global_data_size_64bit = data_offset_64bit;
*p_import_global_data_size_32bit = data_offset_32bit;
return import_globals;
}
static AOTGlobal *
aot_create_globals(const WASMModule *module, uint32 global_data_start_offset,
uint32 *p_global_data_size)
aot_create_globals(const WASMModule *module, bool gc_enabled,
uint32 global_data_start_offset_64bit,
uint32 global_data_start_offset_32bit,
uint32 *p_global_data_size_64bit,
uint32 *p_global_data_size_32bit)
{
AOTGlobal *globals;
uint64 size;
uint32 i, data_offset = global_data_start_offset;
uint32 i;
uint32 data_offset_64bit = global_data_start_offset_64bit;
uint32 data_offset_32bit = global_data_start_offset_32bit;
uint32 value_size_64bit, value_size_32bit;
/* Allocate memory */
size = sizeof(AOTGlobal) * (uint64)module->global_count;
@ -207,65 +271,28 @@ aot_create_globals(const WASMModule *module, uint32 global_data_start_offset,
WASMGlobal *global = &module->globals[i];
globals[i].type = global->type;
globals[i].is_mutable = global->is_mutable;
globals[i].size = wasm_value_type_size(global->type);
memcpy(&globals[i].init_expr, &global->init_expr,
sizeof(global->init_expr));
/* Calculate data offset */
globals[i].data_offset = data_offset;
data_offset += wasm_value_type_size(global->type);
globals[i].data_offset_64bit = data_offset_64bit;
globals[i].data_offset_32bit = data_offset_32bit;
get_value_type_size(global->type, gc_enabled, &value_size_64bit,
&value_size_32bit);
globals[i].size_64bit = value_size_64bit;
globals[i].size_32bit = value_size_32bit;
data_offset_64bit += value_size_64bit;
data_offset_32bit += value_size_32bit;
}
*p_global_data_size = data_offset - global_data_start_offset;
*p_global_data_size_64bit =
data_offset_64bit - global_data_start_offset_64bit;
*p_global_data_size_32bit =
data_offset_32bit - global_data_start_offset_32bit;
return globals;
}
static void
aot_destroy_func_types(AOTFuncType **func_types, uint32 count)
{
uint32 i;
for (i = 0; i < count; i++)
if (func_types[i])
wasm_runtime_free(func_types[i]);
wasm_runtime_free(func_types);
}
static AOTFuncType **
aot_create_func_types(const WASMModule *module)
{
AOTFuncType **func_types;
uint64 size;
uint32 i;
/* Allocate memory */
size = sizeof(AOTFuncType *) * (uint64)module->type_count;
if (size >= UINT32_MAX
|| !(func_types = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
return NULL;
}
memset(func_types, 0, size);
/* Create each function type */
for (i = 0; i < module->type_count; i++) {
size = offsetof(AOTFuncType, types)
+ (uint64)module->types[i]->param_count
+ (uint64)module->types[i]->result_count;
if (size >= UINT32_MAX
|| !(func_types[i] = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
memcpy(func_types[i], module->types[i], size);
}
return func_types;
fail:
aot_destroy_func_types(func_types, module->type_count);
return NULL;
}
static AOTImportFunc *
aot_create_import_funcs(const WASMModule *module)
{
@ -295,7 +322,7 @@ aot_create_import_funcs(const WASMModule *module)
import_funcs[i].call_conv_wasm_c_api = false;
/* Resolve function type index */
for (j = 0; j < module->type_count; j++)
if (import_func->func_type == module->types[j]) {
if (import_func->func_type == (WASMFuncType *)module->types[j]) {
import_funcs[i].func_type_index = j;
break;
}
@ -310,13 +337,16 @@ aot_destroy_funcs(AOTFunc **funcs, uint32 count)
uint32 i;
for (i = 0; i < count; i++)
if (funcs[i])
if (funcs[i]) {
if (funcs[i]->local_offsets)
wasm_runtime_free(funcs[i]->local_offsets);
wasm_runtime_free(funcs[i]);
}
wasm_runtime_free(funcs);
}
static AOTFunc **
aot_create_funcs(const WASMModule *module)
aot_create_funcs(const WASMModule *module, uint32 pointer_size)
{
AOTFunc **funcs;
uint64 size;
@ -334,28 +364,70 @@ aot_create_funcs(const WASMModule *module)
/* Create each function */
for (i = 0; i < module->function_count; i++) {
WASMFunction *func = module->functions[i];
AOTFuncType *func_type = NULL;
AOTFunc *aot_func = NULL;
uint64 total_size;
uint32 offset = 0;
size = sizeof(AOTFunc);
if (!(funcs[i] = wasm_runtime_malloc((uint32)size))) {
if (!(aot_func = funcs[i] = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
memset(aot_func, 0, sizeof(AOTFunc));
func_type = aot_func->func_type = func->func_type;
/* Resolve function type index */
for (j = 0; j < module->type_count; j++) {
if (func_type == (WASMFuncType *)module->types[j]) {
aot_func->func_type_index = j;
break;
}
}
total_size = sizeof(uint16)
* ((uint64)func_type->param_count + func->local_count);
if ((total_size > 0)
&& (total_size >= UINT32_MAX
|| !(aot_func->local_offsets =
wasm_runtime_malloc((uint32)total_size)))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
funcs[i]->func_type = func->func_type;
/* Resolve function type index */
for (j = 0; j < module->type_count; j++)
if (func->func_type == module->types[j]) {
funcs[i]->func_type_index = j;
break;
}
/* Resolve local variable info and code info */
funcs[i]->local_count = func->local_count;
funcs[i]->local_types = func->local_types;
funcs[i]->param_cell_num = func->param_cell_num;
funcs[i]->local_cell_num = func->local_cell_num;
funcs[i]->code = func->code;
funcs[i]->code_size = func->code_size;
aot_func->local_count = func->local_count;
aot_func->local_types_wp = func->local_types;
aot_func->code = func->code;
aot_func->code_size = func->code_size;
/* Resolve local offsets */
for (j = 0; j < func_type->param_count; j++) {
aot_func->local_offsets[j] = (uint16)offset;
offset += wasm_value_type_cell_num_internal(func_type->types[j],
pointer_size);
}
aot_func->param_cell_num = offset;
for (j = 0; j < func->local_count; j++) {
aot_func->local_offsets[func_type->param_count + j] =
(uint16)offset;
offset += wasm_value_type_cell_num_internal(func->local_types[j],
pointer_size);
}
aot_func->local_cell_num = offset - aot_func->param_cell_num;
aot_func->max_stack_cell_num = func->max_stack_cell_num;
/* We use max_stack_cell_num calculated from wasm_loader, which is based
* on wamrc's target type.
* - If the wamrc is compiled as 64bit, then the number is enough for
* both 32bit and 64bit runtime target
* - If the wamrc is compiled as 32bit, then we multiply this number by
* two to avoid overflow on 64bit runtime target */
if (sizeof(uintptr_t) == 4) {
aot_func->max_stack_cell_num *= 2;
}
}
return funcs;
@ -365,12 +437,94 @@ fail:
return NULL;
}
#if WASM_ENABLE_GC != 0
static void
calculate_struct_field_sizes_offsets(AOTCompData *comp_data, bool is_target_x86,
bool gc_enabled)
{
uint32 i;
for (i = 0; i < comp_data->type_count; i++) {
if (comp_data->types[i]->type_flag == WASM_TYPE_STRUCT) {
WASMStructType *struct_type = (WASMStructType *)comp_data->types[i];
WASMStructFieldType *fields = struct_type->fields;
uint32 field_offset_64bit, field_offset_32bit;
uint32 field_size_64bit, field_size_32bit, j;
/* offsetof(WASMStructObject, field_data) in 64-bit */
field_offset_64bit = sizeof(uint64);
/* offsetof(WASMStructObject, field_data) in 32-bit */
field_offset_32bit = sizeof(uint32);
for (j = 0; j < struct_type->field_count; j++) {
get_value_type_size(fields[j].field_type, gc_enabled,
&field_size_64bit, &field_size_32bit);
fields[j].field_size_64bit = field_size_64bit;
fields[j].field_size_32bit = field_size_32bit;
if (!is_target_x86) {
if (field_size_64bit == 2)
field_offset_64bit = align_uint(field_offset_64bit, 2);
else if (field_size_64bit >= 4)
field_offset_64bit = align_uint(field_offset_64bit, 4);
if (field_size_32bit == 2)
field_offset_32bit = align_uint(field_offset_32bit, 2);
else if (field_size_32bit >= 4)
field_offset_32bit = align_uint(field_offset_32bit, 4);
}
fields[j].field_offset_64bit = field_offset_64bit;
fields[j].field_offset_32bit = field_offset_32bit;
field_offset_64bit += field_size_64bit;
field_offset_32bit += field_size_32bit;
}
}
}
}
#endif
AOTCompData *
aot_create_comp_data(WASMModule *module)
aot_create_comp_data(WASMModule *module, const char *target_arch,
bool gc_enabled)
{
AOTCompData *comp_data;
uint32 import_global_data_size = 0, global_data_size = 0, i, j;
uint32 import_global_data_size_64bit = 0, global_data_size_64bit = 0, i, j;
uint32 import_global_data_size_32bit = 0, global_data_size_32bit = 0;
uint64 size;
bool is_64bit_target = false;
#if WASM_ENABLE_GC != 0
bool is_target_x86 = false;
#endif
#if WASM_ENABLE_GC != 0
if (!target_arch) {
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \
|| defined(BUILD_TARGET_X86_32)
is_target_x86 = true;
#endif
}
else {
if (!strncmp(target_arch, "x86_64", 6)
|| !strncmp(target_arch, "i386", 4))
is_target_x86 = true;
}
#endif
if (!target_arch) {
#if UINTPTR_MAX == UINT64_MAX
is_64bit_target = true;
#endif
}
else {
/* All 64bit targets contains "64" string in their target name */
if (strstr(target_arch, "64") != NULL) {
is_64bit_target = true;
}
}
/* Allocate memory */
if (!(comp_data = wasm_runtime_malloc(sizeof(AOTCompData)))) {
@ -457,6 +611,10 @@ aot_create_comp_data(WASMModule *module)
module->import_tables[i].u.table.init_size;
comp_data->tables[i].table_max_size =
module->import_tables[i].u.table.max_size;
#if WASM_ENABLE_GC != 0
comp_data->tables[i].elem_ref_type =
module->import_tables[i].u.table.elem_ref_type;
#endif
comp_data->tables[i].possible_grow =
module->import_tables[i].u.table.possible_grow;
}
@ -470,6 +628,16 @@ aot_create_comp_data(WASMModule *module)
module->tables[j].max_size;
comp_data->tables[i].possible_grow =
module->tables[j].possible_grow;
#if WASM_ENABLE_GC != 0
comp_data->tables[j].elem_ref_type =
module->tables[j].elem_ref_type;
/* Note: if the init_expr contains extra data for struct/array
* initialization information (init_expr.u.data), the pointer is
* copied.
* The pointers should still belong to wasm module, so DO NOT
* free the pointers copied to comp_data */
comp_data->tables[j].init_expr = module->tables[j].init_expr;
#endif
}
}
}
@ -484,24 +652,33 @@ aot_create_comp_data(WASMModule *module)
/* Create import globals */
comp_data->import_global_count = module->import_global_count;
if (comp_data->import_global_count > 0
&& !(comp_data->import_globals =
aot_create_import_globals(module, &import_global_data_size)))
&& !(comp_data->import_globals = aot_create_import_globals(
module, gc_enabled, &import_global_data_size_64bit,
&import_global_data_size_32bit)))
goto fail;
/* Create globals */
comp_data->global_count = module->global_count;
if (comp_data->global_count
&& !(comp_data->globals = aot_create_globals(
module, import_global_data_size, &global_data_size)))
module, gc_enabled, import_global_data_size_64bit,
import_global_data_size_32bit, &global_data_size_64bit,
&global_data_size_32bit)))
goto fail;
comp_data->global_data_size = import_global_data_size + global_data_size;
comp_data->global_data_size_64bit =
import_global_data_size_64bit + global_data_size_64bit;
comp_data->global_data_size_32bit =
import_global_data_size_32bit + global_data_size_32bit;
/* Create function types */
comp_data->func_type_count = module->type_count;
if (comp_data->func_type_count
&& !(comp_data->func_types = aot_create_func_types(module)))
goto fail;
/* Create types, they are checked by wasm loader */
comp_data->type_count = module->type_count;
comp_data->types = module->types;
#if WASM_ENABLE_GC != 0
/* Calculate the field sizes and field offsets for 64-bit and 32-bit
targets since they may vary in 32-bit target and 64-bit target */
calculate_struct_field_sizes_offsets(comp_data, is_target_x86, gc_enabled);
#endif
/* Create import functions */
comp_data->import_func_count = module->import_function_count;
@ -511,7 +688,9 @@ aot_create_comp_data(WASMModule *module)
/* Create functions */
comp_data->func_count = module->function_count;
if (comp_data->func_count && !(comp_data->funcs = aot_create_funcs(module)))
if (comp_data->func_count
&& !(comp_data->funcs =
aot_create_funcs(module, is_64bit_target ? 8 : 4)))
goto fail;
#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
@ -534,6 +713,12 @@ aot_create_comp_data(WASMModule *module)
comp_data->free_func_index = module->free_function;
comp_data->retain_func_index = module->retain_function;
#if WASM_ENABLE_STRINGREF != 0
comp_data->string_literal_count = module->string_literal_count;
comp_data->string_literal_ptrs_wp = module->string_literal_ptrs;
comp_data->string_literal_lengths_wp = module->string_literal_lengths;
#endif
comp_data->wasm_module = module;
return comp_data;
@ -576,10 +761,6 @@ aot_destroy_comp_data(AOTCompData *comp_data)
if (comp_data->globals)
wasm_runtime_free(comp_data->globals);
if (comp_data->func_types)
aot_destroy_func_types(comp_data->func_types,
comp_data->func_type_count);
if (comp_data->import_funcs)
wasm_runtime_free(comp_data->import_funcs);

76
core/iwasm/compilation/aot.h
View File

@ -39,7 +39,12 @@ extern const char *aot_stack_sizes_alias_name;
extern const char *aot_stack_sizes_section_name;
typedef InitializerExpression AOTInitExpr;
typedef WASMType AOTFuncType;
typedef WASMType AOTType;
typedef WASMFuncType AOTFuncType;
#if WASM_ENABLE_GC != 0
typedef WASMStructType AOTStructType;
typedef WASMArrayType AOTArrayType;
#endif
typedef WASMExport AOTExport;
#if WASM_ENABLE_DEBUG_AOT != 0
@ -117,22 +122,30 @@ typedef struct AOTMemInitData {
typedef struct AOTImportTable {
char *module_name;
char *table_name;
uint32 elem_type;
uint32 table_flags;
uint8 elem_type;
uint8 table_flags;
bool possible_grow;
uint32 table_init_size;
uint32 table_max_size;
bool possible_grow;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
#endif
} AOTImportTable;
/**
* Table
*/
typedef struct AOTTable {
uint32 elem_type;
uint32 table_flags;
uint8 elem_type;
uint8 table_flags;
bool possible_grow;
uint32 table_init_size;
uint32 table_max_size;
bool possible_grow;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
/* init expr for the whole table */
InitializerExpression init_expr;
#endif
} AOTTable;
/**
@ -143,14 +156,17 @@ typedef struct AOTTableInitData {
uint32 mode;
/* funcref or externref, elemkind will be considered as funcref */
uint32 elem_type;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
#endif
/* optional, only for active */
uint32 table_index;
/* Start address of init data */
AOTInitExpr offset;
/* Function index count */
uint32 func_index_count;
uint32 value_count;
/* Function index array */
uint32 func_indexes[1];
InitializerExpression init_values[1];
} AOTTableInitData;
/**
@ -165,6 +181,19 @@ typedef struct AOTImportGlobal {
uint32 size;
/* The data offset of current global in global data */
uint32 data_offset;
#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0
/*
* The data size and data offset of a wasm global may vary
* in 32-bit target and 64-bit target, e.g., the size of a
* GC obj is 4 bytes in the former and 8 bytes in the
* latter, the AOT compiler needs to use the correct data
* offset according to the target info.
*/
uint32 size_64bit;
uint32 size_32bit;
uint32 data_offset_64bit;
uint32 data_offset_32bit;
#endif
/* global data after linked */
WASMValue global_data_linked;
bool is_linked;
@ -180,6 +209,13 @@ typedef struct AOTGlobal {
uint32 size;
/* The data offset of current global in global data */
uint32 data_offset;
#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0
/* See comments in AOTImportGlobal */
uint32 size_64bit;
uint32 size_32bit;
uint32 data_offset_64bit;
uint32 data_offset_32bit;
#endif
AOTInitExpr init_expr;
} AOTGlobal;
@ -209,11 +245,15 @@ typedef struct AOTFunc {
AOTFuncType *func_type;
uint32 func_type_index;
uint32 local_count;
uint8 *local_types;
uint8 *local_types_wp;
uint16 param_cell_num;
uint16 local_cell_num;
uint32 max_stack_cell_num;
uint32 code_size;
uint8 *code;
/* offset of each local, including function parameters
and local variables */
uint16 *local_offsets;
} AOTFunc;
typedef struct AOTCompData {
@ -250,8 +290,8 @@ typedef struct AOTCompData {
AOTGlobal *globals;
/* Function types */
uint32 func_type_count;
AOTFuncType **func_types;
uint32 type_count;
AOTType **types;
/* Import functions */
uint32 import_func_count;
@ -267,7 +307,8 @@ typedef struct AOTCompData {
uint8 *aot_name_section_buf;
uint32 aot_name_section_size;
uint32 global_data_size;
uint32 global_data_size_64bit;
uint32 global_data_size_32bit;
uint32 start_func_index;
uint32 malloc_func_index;
@ -282,6 +323,12 @@ typedef struct AOTCompData {
uint32 aux_stack_bottom;
uint32 aux_stack_size;
#if WASM_ENABLE_STRINGREF != 0
uint32 string_literal_count;
uint32 *string_literal_lengths_wp;
const uint8 **string_literal_ptrs_wp;
#endif
WASMModule *wasm_module;
#if WASM_ENABLE_DEBUG_AOT != 0
dwarf_extractor_handle_t extractor;
@ -295,7 +342,8 @@ typedef struct AOTNativeSymbol {
} AOTNativeSymbol;
AOTCompData *
aot_create_comp_data(WASMModule *module);
aot_create_comp_data(WASMModule *module, const char *target_arch,
bool gc_enabled);
void
aot_destroy_comp_data(AOTCompData *comp_data);

1411
core/iwasm/compilation/aot_compiler.c
View File

File diff suppressed because it is too large Load Diff

451
core/iwasm/compilation/aot_compiler.h
View File

@ -8,6 +8,8 @@
#include "aot.h"
#include "aot_llvm.h"
#include "../interpreter/wasm_interp.h"
#include "../aot/aot_runtime.h"
#ifdef __cplusplus
extern "C" {
@ -78,8 +80,33 @@ typedef enum FloatArithmetic {
FLOAT_MAX,
} FloatArithmetic;
/**
* Check whether a value type is a GC reference type,
* don't use wasm_is_type_reftype since it requires
* GC feature and may result in compilation error when
* GC feature isn't compiled
*/
static inline bool
check_type_compatible(uint8 src_type, uint8 dst_type)
aot_is_type_gc_reftype(uint8 type)
{
return ((type >= (uint8)REF_TYPE_ARRAYREF
&& type <= (uint8)REF_TYPE_NULLFUNCREF)
|| (type >= (uint8)REF_TYPE_HT_NULLABLE
&& type <= (uint8)REF_TYPE_HT_NON_NULLABLE)
#if WASM_ENABLE_STRINGREF != 0
|| (type >= (uint8)REF_TYPE_STRINGVIEWWTF8
&& type <= (uint8)REF_TYPE_STRINGREF)
|| (type >= (uint8)REF_TYPE_STRINGVIEWITER
&& type <= (uint8)REF_TYPE_STRINGVIEWWTF16)
#endif
)
? true
: false;
}
static inline bool
check_type_compatible(const AOTCompContext *comp_ctx, uint8 src_type,
uint8 dst_type)
{
if (src_type == dst_type) {
return true;
@ -92,20 +119,332 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
/* i32 <==> func.ref, i32 <==> extern.ref */
if (src_type == VALUE_TYPE_I32
&& (dst_type == VALUE_TYPE_EXTERNREF
|| dst_type == VALUE_TYPE_FUNCREF)) {
&& (comp_ctx->enable_ref_types
&& (dst_type == VALUE_TYPE_EXTERNREF
|| dst_type == VALUE_TYPE_FUNCREF))) {
return true;
}
if (dst_type == VALUE_TYPE_I32
&& (src_type == VALUE_TYPE_FUNCREF
|| src_type == VALUE_TYPE_EXTERNREF)) {
&& (comp_ctx->enable_ref_types
&& (src_type == VALUE_TYPE_FUNCREF
|| src_type == VALUE_TYPE_EXTERNREF))) {
return true;
}
return false;
}
/**
* Operations for AOTCompFrame
*/
/**
* Get the offset from frame pointer to the n-th local variable slot.
*
* @param n the index to the local variable array
*
* @return the offset from frame pointer to the local variable slot
*/
static inline uint32
offset_of_local(AOTCompContext *comp_ctx, unsigned n)
{
if (!comp_ctx->is_jit_mode)
/* In AOTFrame, there are 7 pointers before field lp */
return comp_ctx->pointer_size
* (offsetof(AOTFrame, lp) / sizeof(uintptr_t))
+ sizeof(uint32) * n;
else
return offsetof(WASMInterpFrame, lp) + sizeof(uint32) * n;
}
uint32
offset_of_local_in_outs_area(AOTCompContext *comp_ctx, unsigned n);
/**
* Get the offset from frame pointer to the n-th local variable's
* reference flag slot.
*
* @param n the index to the local variable array
*
* @return the offset from frame pointer to the local variable slot
*/
static inline unsigned
offset_of_ref(AOTCompContext *comp_ctx, unsigned n)
{
AOTCompFrame *frame = comp_ctx->aot_frame;
uint32 all_cell_num = frame->max_local_cell_num + frame->max_stack_cell_num;
return offset_of_local(comp_ctx, all_cell_num) + n;
}
/**
* Generate instructions to commit computation result to the frame.
* The general principle is to only commit values that will be used
* through the frame.
*
* @param frame the frame information
*/
bool
aot_gen_commit_values(AOTCompFrame *frame);
/**
* Generate instructions to commit SP and IP pointers to the frame.
*
* @param frame the frame information
*/
bool
aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip);
bool
aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value,
uint8 value_type, LLVMValueRef cur_frame, uint32 offset);
static inline void
push_32bit(AOTCompFrame *frame, AOTValue *aot_value)
{
frame->sp->value = aot_value->value;
frame->sp->type = aot_value->type;
frame->sp->dirty = 1;
frame->sp++;
}
static inline void
push_64bit(AOTCompFrame *frame, AOTValue *aot_value)
{
push_32bit(frame, aot_value);
push_32bit(frame, aot_value);
}
static inline void
push_i32(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_I32
|| aot_value->type == VALUE_TYPE_I1);
push_32bit(frame, aot_value);
}
static inline void
push_i64(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_I64);
push_64bit(frame, aot_value);
}
static inline void
push_f32(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_F32);
push_32bit(frame, aot_value);
}
static inline void
push_f64(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_F64);
push_64bit(frame, aot_value);
}
static inline void
push_v128(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_V128);
push_64bit(frame, aot_value);
push_64bit(frame, aot_value);
}
static inline void
push_ref(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(frame->comp_ctx->enable_ref_types);
push_32bit(frame, aot_value);
}
#if WASM_ENABLE_GC != 0
static inline void
push_gc_ref(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(frame->comp_ctx->enable_gc);
bh_assert(aot_value->type == VALUE_TYPE_GC_REF);
if (frame->comp_ctx->pointer_size == sizeof(uint64)) {
push_64bit(frame, aot_value);
(frame->sp - 1)->ref = (frame->sp - 2)->ref = 1;
}
else {
push_32bit(frame, aot_value);
(frame->sp - 1)->ref = 1;
}
}
#endif
/* Clear value slots except ref and committed_ref */
static inline void
clear_frame_value_slots(AOTValueSlot *slots, uint32 n)
{
uint32 i;
for (i = 0; i < n; i++) {
slots[i].value = 0;
slots[i].type = 0;
slots[i].dirty = 0;
}
}
static inline void
pop_i32(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_I32
|| (frame->sp - 1)->type == VALUE_TYPE_I1);
frame->sp--;
clear_frame_value_slots(frame->sp, 1);
}
static inline void
pop_i64(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 2);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_I64
&& (frame->sp - 2)->type == VALUE_TYPE_I64);
frame->sp -= 2;
clear_frame_value_slots(frame->sp, 2);
}
static inline void
pop_f32(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_F32);
frame->sp--;
clear_frame_value_slots(frame->sp, 1);
}
static inline void
pop_f64(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 2);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_F64
&& (frame->sp - 2)->type == VALUE_TYPE_F64);
frame->sp -= 2;
clear_frame_value_slots(frame->sp, 2);
}
static inline void
pop_v128(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 4);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_V128
&& (frame->sp - 2)->type == VALUE_TYPE_V128
&& (frame->sp - 3)->type == VALUE_TYPE_V128
&& (frame->sp - 4)->type == VALUE_TYPE_V128);
frame->sp -= 4;
clear_frame_value_slots(frame->sp, 4);
}
static inline void
pop_ref(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_FUNCREF
|| (frame->sp - 1)->type == VALUE_TYPE_EXTERNREF);
frame->sp -= 1;
clear_frame_value_slots(frame->sp, 1);
}
#if WASM_ENABLE_GC != 0
static inline void
pop_gc_ref(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_GC_REF);
frame->sp -= 1;
clear_frame_value_slots(frame->sp, 1);
frame->sp->ref = 0;
if (frame->comp_ctx->pointer_size == sizeof(uint64)) {
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_GC_REF);
frame->sp -= 1;
clear_frame_value_slots(frame->sp, 1);
frame->sp->ref = 0;
}
}
#endif
static inline void
set_local_i32(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_I32;
frame->lp[n].dirty = 1;
}
static inline void
set_local_i64(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_I64;
frame->lp[n].dirty = 1;
frame->lp[n + 1].value = value;
frame->lp[n + 1].type = VALUE_TYPE_I64;
frame->lp[n + 1].dirty = 1;
}
static inline void
set_local_f32(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_F32;
frame->lp[n].dirty = 1;
}
static inline void
set_local_f64(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_F64;
frame->lp[n].dirty = 1;
frame->lp[n + 1].value = value;
frame->lp[n + 1].type = VALUE_TYPE_F64;
frame->lp[n + 1].dirty = 1;
}
static inline void
set_local_v128(AOTCompFrame *frame, int n, LLVMValueRef value)
{
uint32 i;
for (i = 0; i < 4; i++) {
frame->lp[n + i].value = value;
frame->lp[n + i].type = VALUE_TYPE_V128;
frame->lp[n + i].dirty = 1;
}
}
static inline void
set_local_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
{
bh_assert(frame->comp_ctx->enable_ref_types);
frame->lp[n].value = value;
frame->lp[n].type = ref_type;
frame->lp[n].dirty = 1;
}
#if WASM_ENABLE_GC != 0
static inline void
set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
{
bh_assert(frame->comp_ctx->enable_gc);
bh_assert(ref_type == VALUE_TYPE_GC_REF);
frame->lp[n].value = value;
frame->lp[n].type = ref_type;
frame->lp[n].dirty = 1;
frame->lp[n].ref = 1;
if (frame->comp_ctx->pointer_size == sizeof(uint64)) {
frame->lp[n + 1].value = value;
frame->lp[n + 1].type = ref_type;
frame->lp[n + 1].dirty = 1;
frame->lp[n + 1].ref = 1;
}
}
#endif
#define CHECK_STACK() \
do { \
if (!func_ctx->block_stack.block_list_end) { \
@ -119,37 +458,61 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
} \
} while (0)
#if WASM_ENABLE_GC != 0
#define GET_GC_REF_FROM_STACK(llvm_value) \
do { \
AOTValue *aot_value; \
CHECK_STACK(); \
aot_value = \
func_ctx->block_stack.block_list_end->value_stack.value_list_end; \
if (aot_value->type != VALUE_TYPE_GC_REF) { \
aot_set_last_error("WASM stack data type is not reference"); \
goto fail; \
} \
llvm_value = aot_value->value; \
} while (0)
#endif
#define POP(llvm_value, value_type) \
do { \
AOTValue *aot_value; \
uint8 val_type_to_pop = value_type; \
CHECK_STACK(); \
aot_value = aot_value_stack_pop( \
&func_ctx->block_stack.block_list_end->value_stack); \
if (!check_type_compatible(aot_value->type, value_type)) { \
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); \
if (comp_ctx->enable_gc && aot_is_type_gc_reftype(value_type)) \
val_type_to_pop = VALUE_TYPE_GC_REF; \
if (!check_type_compatible(comp_ctx, aot_value->type, \
val_type_to_pop)) { \
aot_set_last_error("invalid WASM stack data type."); \
wasm_runtime_free(aot_value); \
goto fail; \
} \
if (aot_value->type == value_type) \
if (aot_value->type == val_type_to_pop) \
llvm_value = aot_value->value; \
else { \
if (aot_value->type == VALUE_TYPE_I1) { \
if (!(llvm_value = \
LLVMBuildZExt(comp_ctx->builder, aot_value->value, \
I32_TYPE, "i1toi32"))) { \
aot_set_last_error("invalid WASM stack " \
"data type."); \
aot_set_last_error("invalid WASM stack data type."); \
wasm_runtime_free(aot_value); \
goto fail; \
} \
} \
else { \
bh_assert(aot_value->type == VALUE_TYPE_I32 \
|| aot_value->type == VALUE_TYPE_FUNCREF \
|| aot_value->type == VALUE_TYPE_EXTERNREF); \
bh_assert(value_type == VALUE_TYPE_I32 \
|| value_type == VALUE_TYPE_FUNCREF \
|| value_type == VALUE_TYPE_EXTERNREF); \
bh_assert( \
aot_value->type == VALUE_TYPE_I32 \
|| (comp_ctx->enable_ref_types \
&& (aot_value->type == VALUE_TYPE_FUNCREF \
|| aot_value->type == VALUE_TYPE_EXTERNREF))); \
bh_assert( \
val_type_to_pop == VALUE_TYPE_I32 \
|| (comp_ctx->enable_ref_types \
&& (val_type_to_pop == VALUE_TYPE_FUNCREF \
|| val_type_to_pop == VALUE_TYPE_EXTERNREF))); \
llvm_value = aot_value->value; \
} \
} \
@ -163,13 +526,14 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define POP_V128(v) POP(v, VALUE_TYPE_V128)
#define POP_FUNCREF(v) POP(v, VALUE_TYPE_FUNCREF)
#define POP_EXTERNREF(v) POP(v, VALUE_TYPE_EXTERNREF)
#define POP_GC_REF(v) POP(v, VALUE_TYPE_GC_REF)
#define POP_COND(llvm_value) \
do { \
AOTValue *aot_value; \
CHECK_STACK(); \
aot_value = aot_value_stack_pop( \
&func_ctx->block_stack.block_list_end->value_stack); \
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); \
if (aot_value->type != VALUE_TYPE_I1 \
&& aot_value->type != VALUE_TYPE_I32) { \
aot_set_last_error("invalid WASM stack data type."); \
@ -190,23 +554,27 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
wasm_runtime_free(aot_value); \
} while (0)
#define PUSH(llvm_value, value_type) \
do { \
AOTValue *aot_value; \
if (!func_ctx->block_stack.block_list_end) { \
aot_set_last_error("WASM block stack underflow."); \
goto fail; \
} \
aot_value = wasm_runtime_malloc(sizeof(AOTValue)); \
if (!aot_value) { \
aot_set_last_error("allocate memory failed."); \
goto fail; \
} \
memset(aot_value, 0, sizeof(AOTValue)); \
aot_value->type = value_type; \
aot_value->value = llvm_value; \
aot_value_stack_push( \
&func_ctx->block_stack.block_list_end->value_stack, aot_value); \
#define PUSH(llvm_value, value_type) \
do { \
AOTValue *aot_value; \
if (!func_ctx->block_stack.block_list_end) { \
aot_set_last_error("WASM block stack underflow."); \
goto fail; \
} \
aot_value = wasm_runtime_malloc(sizeof(AOTValue)); \
if (!aot_value) { \
aot_set_last_error("allocate memory failed."); \
goto fail; \
} \
memset(aot_value, 0, sizeof(AOTValue)); \
if (comp_ctx->enable_gc && aot_is_type_gc_reftype(value_type)) \
aot_value->type = VALUE_TYPE_GC_REF; \
else \
aot_value->type = value_type; \
aot_value->value = llvm_value; \
aot_value_stack_push( \
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack, \
aot_value); \
} while (0)
#define PUSH_I32(v) PUSH(v, VALUE_TYPE_I32)
@ -217,9 +585,10 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define PUSH_COND(v) PUSH(v, VALUE_TYPE_I1)
#define PUSH_FUNCREF(v) PUSH(v, VALUE_TYPE_FUNCREF)
#define PUSH_EXTERNREF(v) PUSH(v, VALUE_TYPE_EXTERNREF)
#define PUSH_GC_REF(v) PUSH(v, VALUE_TYPE_GC_REF)
#define TO_LLVM_TYPE(wasm_type) \
wasm_type_to_llvm_type(&comp_ctx->basic_types, wasm_type)
wasm_type_to_llvm_type(comp_ctx, &comp_ctx->basic_types, wasm_type)
#define I32_TYPE comp_ctx->basic_types.int32_type
#define I64_TYPE comp_ctx->basic_types.int64_type
@ -229,15 +598,19 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define INT1_TYPE comp_ctx->basic_types.int1_type
#define INT8_TYPE comp_ctx->basic_types.int8_type
#define INT16_TYPE comp_ctx->basic_types.int16_type
#define INTPTR_T_TYPE comp_ctx->basic_types.intptr_t_type
#define MD_TYPE comp_ctx->basic_types.meta_data_type
#define INT8_PTR_TYPE comp_ctx->basic_types.int8_ptr_type
#define INT16_PTR_TYPE comp_ctx->basic_types.int16_ptr_type
#define INT32_PTR_TYPE comp_ctx->basic_types.int32_ptr_type
#define INT64_PTR_TYPE comp_ctx->basic_types.int64_ptr_type
#define INTPTR_T_PTR_TYPE comp_ctx->basic_types.intptr_t_ptr_type
#define F32_PTR_TYPE comp_ctx->basic_types.float32_ptr_type
#define F64_PTR_TYPE comp_ctx->basic_types.float64_ptr_type
#define FUNC_REF_TYPE comp_ctx->basic_types.funcref_type
#define EXTERN_REF_TYPE comp_ctx->basic_types.externref_type
#define GC_REF_TYPE comp_ctx->basic_types.gc_ref_type
#define GC_REF_PTR_TYPE comp_ctx->basic_types.gc_ref_ptr_type
#define INT8_PTR_TYPE_GS comp_ctx->basic_types.int8_ptr_type_gs
#define INT16_PTR_TYPE_GS comp_ctx->basic_types.int16_ptr_type_gs
@ -253,7 +626,10 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define I8_CONST(v) LLVMConstInt(INT8_TYPE, v, true)
#define LLVM_CONST(name) (comp_ctx->llvm_consts.name)
#define I1_ZERO LLVM_CONST(i1_zero)
#define I1_ONE LLVM_CONST(i1_one)
#define I8_ZERO LLVM_CONST(i8_zero)
#define I8_ONE LLVM_CONST(i8_one)
#define I32_ZERO LLVM_CONST(i32_zero)
#define I64_ZERO LLVM_CONST(i64_zero)
#define F32_ZERO LLVM_CONST(f32_zero)
@ -267,6 +643,9 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define I32_SEVEN LLVM_CONST(i32_seven)
#define I32_EIGHT LLVM_CONST(i32_eight)
#define I32_NINE LLVM_CONST(i32_nine)
#define I32_TEN LLVM_CONST(i32_ten)
#define I32_ELEVEN LLVM_CONST(i32_eleven)
#define I32_TWELVE LLVM_CONST(i32_twelve)
#define I32_NEG_ONE LLVM_CONST(i32_neg_one)
#define I64_NEG_ONE LLVM_CONST(i64_neg_one)
#define I32_MIN LLVM_CONST(i32_min)
@ -276,6 +655,8 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define I64_63 LLVM_CONST(i64_63)
#define I64_64 LLVM_CONST(i64_64)
#define REF_NULL I32_NEG_ONE
#define GC_REF_NULL LLVM_CONST(gc_ref_null)
#define I8_PTR_NULL LLVM_CONST(i8_ptr_null)
#define V128_TYPE comp_ctx->basic_types.v128_type
#define V128_PTR_TYPE comp_ctx->basic_types.v128_ptr_type

1078
core/iwasm/compilation/aot_emit_aot_file.c
View File

File diff suppressed because it is too large Load Diff

24
core/iwasm/compilation/aot_emit_compare.c
View File

@ -230,3 +230,27 @@ aot_compile_op_f64_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
fail:
return false;
}
bool
aot_compile_op_ref_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef gc_obj1 = NULL, gc_obj2 = NULL, res;
POP_GC_REF(gc_obj1);
POP_GC_REF(gc_obj2);
/* LLVM pointer values pointers are compared using LLVMBuildICmp */
res = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, gc_obj1, gc_obj2,
"cmp_gc_obj_eq");
if (!res) {
aot_set_last_error("llvm build compare failed.");
return false;
}
PUSH_COND(res);
return true;
fail:
return false;
}

7
core/iwasm/compilation/aot_emit_compare.h
View File

@ -28,6 +28,13 @@ bool
aot_compile_op_f64_compare(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
FloatCond cond);
#if WASM_ENABLE_GC != 0
bool
aot_compile_op_ref_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
#endif
#ifdef __cplusplus
} /* end of extern "C" */
#endif

587
core/iwasm/compilation/aot_emit_control.c
View File

@ -6,6 +6,9 @@
#include "aot_emit_control.h"
#include "aot_compiler.h"
#include "aot_emit_exception.h"
#if WASM_ENABLE_GC != 0
#include "aot_emit_gc.h"
#endif
#include "../aot/aot_runtime.h"
#include "../interpreter/wasm_loader.h"
@ -155,12 +158,81 @@ get_target_block(AOTFuncContext *func_ctx, uint32 br_depth)
return block;
}
static void
clear_frame_locals(AOTCompFrame *aot_frame)
{
uint32 i;
for (i = 0; i < aot_frame->max_local_cell_num; i++) {
aot_frame->lp[i].dirty = 0;
aot_frame->lp[i].value = NULL;
if (aot_frame->comp_ctx->enable_gc)
/* Mark the ref flag as committed */
aot_frame->lp[i].committed_ref = aot_frame->lp[i].ref + 1;
}
}
static void
restore_frame_sp_for_op_else(AOTBlock *block, AOTCompFrame *aot_frame)
{
uint32 all_cell_num =
aot_frame->max_local_cell_num + aot_frame->max_stack_cell_num;
AOTValueSlot *p_end = aot_frame->lp + all_cell_num, *p;
/* Reset all the value slots from current frame sp for the else
branch since they be the same as starting to translate the
if branch */
for (p = block->frame_sp_begin; p < p_end; p++) {
p->dirty = 0;
p->value = NULL;
p->type = 0;
if (aot_frame->comp_ctx->enable_gc) {
p->ref = 0;
p->committed_ref = 1;
}
}
bh_assert(aot_frame->sp >= block->frame_sp_begin);
aot_frame->sp = block->frame_sp_begin;
}
static void
restore_frame_sp_for_op_end(AOTBlock *block, AOTCompFrame *aot_frame)
{
uint32 all_cell_num =
aot_frame->max_local_cell_num + aot_frame->max_stack_cell_num;
AOTValueSlot *p_end = aot_frame->lp + all_cell_num, *p;
bh_assert(block->frame_sp_max_reached >= block->frame_sp_begin);
/* Reset all the value slots from current frame sp to be same as
starting to translate this block, except for the frame ref
flags: set the flags to uncommitted before the max frame sp
ever reached, set the flags to committed non-ref after that */
for (p = block->frame_sp_begin; p < p_end; p++) {
p->dirty = 0;
p->value = NULL;
p->type = 0;
if (aot_frame->comp_ctx->enable_gc) {
p->ref = 0;
if (p < block->frame_sp_max_reached)
p->committed_ref = 0;
else
p->committed_ref = 1;
}
}
bh_assert(aot_frame->sp >= block->frame_sp_begin);
aot_frame->sp = block->frame_sp_begin;
}
static bool
handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint8 **p_frame_ip)
{
AOTBlock *block = func_ctx->block_stack.block_list_end;
AOTBlock *block_prev;
AOTCompFrame *aot_frame = comp_ctx->aot_frame;
uint8 *frame_ip = NULL;
uint32 i;
AOTFuncType *func_type;
@ -177,10 +249,22 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
comp_ctx, func_ctx,
(*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code);
#endif
if (aot_frame) {
/* Clear frame local variables since they have been committed */
clear_frame_locals(aot_frame);
}
if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block
&& *p_frame_ip <= block->wasm_code_else) {
/* Clear value stack and start to translate else branch */
aot_value_stack_destroy(&block->value_stack);
aot_value_stack_destroy(comp_ctx, &block->value_stack);
if (aot_frame) {
/* Restore the frame sp */
restore_frame_sp_for_op_else(block, aot_frame);
}
/* Recover parameters of else branch */
for (i = 0; i < block->param_count; i++)
PUSH(block->else_param_phis[i], block->param_types[i]);
@ -197,7 +281,13 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
if (block->llvm_else_block && !block->skip_wasm_code_else
&& *p_frame_ip <= block->wasm_code_else) {
/* Clear value stack and start to translate else branch */
aot_value_stack_destroy(&block->value_stack);
aot_value_stack_destroy(comp_ctx, &block->value_stack);
if (aot_frame) {
/* Restore the frame sp */
restore_frame_sp_for_op_else(block, aot_frame);
}
SET_BUILDER_POS(block->llvm_else_block);
*p_frame_ip = block->wasm_code_else + 1;
/* Push back the block */
@ -215,7 +305,7 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
frame_ip = block->wasm_code_end;
aot_block_destroy(block);
aot_block_destroy(comp_ctx, block);
block = block_prev;
}
@ -228,7 +318,13 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
&& !block->skip_wasm_code_else
&& *p_frame_ip <= block->wasm_code_else) {
/* Clear value stack and start to translate else branch */
aot_value_stack_destroy(&block->value_stack);
aot_value_stack_destroy(comp_ctx, &block->value_stack);
if (aot_frame) {
/* Restore the frame sp */
restore_frame_sp_for_op_else(block, aot_frame);
}
/* Recover parameters of else branch */
for (i = 0; i < block->param_count; i++)
PUSH(block->else_param_phis[i], block->param_types[i]);
@ -242,6 +338,12 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* Pop block, push its return value, and destroy the block */
block = aot_block_stack_pop(&func_ctx->block_stack);
if (aot_frame) {
/* Restore the frame sp */
restore_frame_sp_for_op_end(block, aot_frame);
}
func_type = func_ctx->aot_func->func_type;
for (i = 0; i < block->result_count; i++) {
bh_assert(block->result_phis[i]);
@ -285,7 +387,7 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
#endif
}
}
aot_block_destroy(block);
aot_block_destroy(comp_ctx, block);
return true;
fail:
return false;
@ -381,6 +483,10 @@ push_aot_block_to_stack_and_pass_params(AOTCompContext *comp_ctx,
/* Push the new block to block stack */
aot_block_stack_push(&func_ctx->block_stack, block);
if (comp_ctx->aot_frame) {
block->frame_sp_begin = block->frame_sp_max_reached =
comp_ctx->aot_frame->sp;
}
/* Push param phis to the new block */
for (i = 0; i < block->param_count; i++) {
@ -473,6 +579,13 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
block->block_index = func_ctx->block_stack.block_index[label_type];
func_ctx->block_stack.block_index[label_type]++;
if (comp_ctx->aot_frame) {
if (label_type != LABEL_TYPE_BLOCK && comp_ctx->enable_gc
&& !aot_gen_commit_values(comp_ctx->aot_frame)) {
goto fail;
}
}
if (label_type == LABEL_TYPE_BLOCK || label_type == LABEL_TYPE_LOOP) {
/* Create block */
format_block_name(name, sizeof(name), block->block_index, label_type,
@ -500,7 +613,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
false, NULL, NULL))) {
goto fail;
}
aot_block_destroy(block);
aot_block_destroy(comp_ctx, block);
return aot_handle_next_reachable_block(comp_ctx, func_ctx,
p_frame_ip);
}
@ -580,7 +693,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
else {
/* skip the block */
aot_block_destroy(block);
aot_block_destroy(comp_ctx, block);
*p_frame_ip = end_addr + 1;
}
}
@ -593,7 +706,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return true;
fail:
aot_block_destroy(block);
aot_block_destroy(comp_ctx, block);
return false;
}
@ -603,6 +716,7 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
{
AOTBlock *block = func_ctx->block_stack.block_list_end;
LLVMValueRef value;
AOTCompFrame *aot_frame = comp_ctx->aot_frame;
char name[32];
uint32 i, result_index;
@ -638,14 +752,26 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
ADD_TO_RESULT_PHIS(block, value, result_index);
}
if (aot_frame) {
bh_assert(block->frame_sp_begin == aot_frame->sp);
if (comp_ctx->enable_gc && !aot_gen_commit_values(aot_frame)) {
goto fail;
}
}
/* Jump to end block */
BUILD_BR(block->llvm_end_block);
if (!block->skip_wasm_code_else && block->llvm_else_block) {
/* Clear value stack, recover param values
* and start to translate else branch.
*/
aot_value_stack_destroy(&block->value_stack);
and start to translate else branch. */
aot_value_stack_destroy(comp_ctx, &block->value_stack);
if (comp_ctx->aot_frame) {
clear_frame_locals(aot_frame);
restore_frame_sp_for_op_else(block, aot_frame);
}
for (i = 0; i < block->param_count; i++)
PUSH(block->else_param_phis[i], block->param_types[i]);
SET_BUILDER_POS(block->llvm_else_block);
@ -685,6 +811,13 @@ aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block);
}
if (comp_ctx->aot_frame) {
if (block->label_type != LABEL_TYPE_FUNCTION && comp_ctx->enable_gc
&& !aot_gen_commit_values(comp_ctx->aot_frame)) {
return false;
}
}
/* Handle block result values */
CREATE_RESULT_VALUE_PHIS(block);
for (i = 0; i < block->result_count; i++) {
@ -695,6 +828,10 @@ aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
ADD_TO_RESULT_PHIS(block, value, result_index);
}
if (comp_ctx->aot_frame) {
bh_assert(comp_ctx->aot_frame->sp == block->frame_sp_begin);
}
/* Jump to the end block */
BUILD_BR(block->llvm_end_block);
@ -704,9 +841,9 @@ fail:
return false;
}
#if WASM_ENABLE_THREAD_MGR != 0
bool
check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
bool check_terminate_and_suspend)
{
LLVMValueRef terminate_addr, terminate_flags, flag, offset, res;
LLVMBasicBlockRef terminate_block, non_terminate_block;
@ -774,7 +911,6 @@ check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
fail:
return false;
}
#endif /* End of WASM_ENABLE_THREAD_MGR */
bool
aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
@ -786,18 +922,35 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
char name[32];
uint32 i, param_index, result_index;
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (comp_ctx->enable_thread_mgr) {
if (!check_suspend_flags(comp_ctx, func_ctx))
return false;
}
#endif
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
return false;
}
if (comp_ctx->aot_frame) {
if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame))
return false;
if (block_dst->label_type == LABEL_TYPE_LOOP) {
if (comp_ctx->enable_thread_mgr) {
/* Commit sp when GC is enabled, don't commit ip */
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame,
comp_ctx->enable_gc, false))
return false;
}
}
else {
if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached)
block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp;
}
}
/* Terminate or suspend current thread only when this is a backward jump */
if (comp_ctx->enable_thread_mgr
&& block_dst->label_type == LABEL_TYPE_LOOP) {
if (!check_suspend_flags(comp_ctx, func_ctx, true))
return false;
}
if (block_dst->label_type == LABEL_TYPE_LOOP) {
/* Dest block is Loop block */
/* Handle Loop parameters */
@ -838,26 +991,47 @@ fail:
return false;
}
bool
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 br_depth, uint8 **p_frame_ip)
static bool
aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 br_depth, LLVMValueRef value_cmp,
uint8 **p_frame_ip)
{
AOTBlock *block_dst;
LLVMValueRef value_cmp, value, *values = NULL;
LLVMValueRef value, *values = NULL;
LLVMBasicBlockRef llvm_else_block, next_llvm_end_block;
char name[32];
uint32 i, param_index, result_index;
uint64 size;
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (comp_ctx->enable_thread_mgr) {
if (!check_suspend_flags(comp_ctx, func_ctx))
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
return false;
}
if (comp_ctx->aot_frame) {
if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame))
return false;
if (block_dst->label_type == LABEL_TYPE_LOOP) {
if (comp_ctx->enable_thread_mgr) {
/* Commit sp when GC is enabled, don't commit ip */
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame,
comp_ctx->enable_gc, false))
return false;
}
}
else {
if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached)
block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp;
}
}
/* Terminate or suspend current thread only when this is
a backward jump */
if (comp_ctx->enable_thread_mgr
&& block_dst->label_type == LABEL_TYPE_LOOP) {
if (!check_suspend_flags(comp_ctx, func_ctx, true))
return false;
}
#endif
POP_COND(value_cmp);
if (LLVMIsUndef(value_cmp)
#if LLVM_VERSION_NUMBER >= 12
@ -873,9 +1047,6 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
if (!LLVMIsEfficientConstInt(value_cmp)) {
/* Compare value is not constant, create condition br IR */
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
return false;
}
/* Create llvm else block */
CREATE_BLOCK(llvm_else_block, "br_if_else");
@ -972,6 +1143,20 @@ fail:
return false;
}
bool
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 br_depth, uint8 **p_frame_ip)
{
LLVMValueRef value_cmp;
POP_COND(value_cmp);
return aot_compile_conditional_br(comp_ctx, func_ctx, br_depth, value_cmp,
p_frame_ip);
fail:
return false;
}
bool
aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 *br_depths, uint32 br_count, uint8 **p_frame_ip)
@ -986,14 +1171,6 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint64 size;
char name[32];
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (comp_ctx->enable_thread_mgr) {
if (!check_suspend_flags(comp_ctx, func_ctx))
return false;
}
#endif
POP_I32(value_cmp);
if (LLVMIsUndef(value_cmp)
@ -1009,6 +1186,46 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
if (!LLVMIsEfficientConstInt(value_cmp)) {
if (comp_ctx->aot_frame) {
if (comp_ctx->enable_gc
&& !aot_gen_commit_values(comp_ctx->aot_frame))
return false;
if (comp_ctx->enable_thread_mgr) {
/* Commit sp when GC is enabled, don't commit ip */
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame,
comp_ctx->enable_gc, false))
return false;
}
for (i = 0; i <= br_count; i++) {
target_block = get_target_block(func_ctx, br_depths[i]);
if (!target_block)
return false;
if (target_block->label_type != LABEL_TYPE_LOOP) {
if (comp_ctx->aot_frame->sp
> target_block->frame_sp_max_reached)
target_block->frame_sp_max_reached =
comp_ctx->aot_frame->sp;
}
}
}
if (comp_ctx->enable_thread_mgr) {
for (i = 0; i <= br_count; i++) {
target_block = get_target_block(func_ctx, br_depths[i]);
if (!target_block)
return false;
/* Terminate or suspend current thread only when this is a
backward jump */
if (target_block->label_type == LABEL_TYPE_LOOP) {
if (!check_suspend_flags(comp_ctx, func_ctx, true))
return false;
break;
}
}
}
/* Compare value is not constant, create switch IR */
for (i = 0; i <= br_count; i++) {
target_block = get_target_block(func_ctx, br_depths[i]);
@ -1137,6 +1354,7 @@ aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
comp_ctx, func_ctx,
(*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code);
#endif
if (block_func->result_count) {
/* Store extra result values to function parameters */
for (i = 0; i < block_func->result_count - 1; i++) {
@ -1194,3 +1412,284 @@ aot_handle_next_reachable_block(AOTCompContext *comp_ctx,
{
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
}
#if WASM_ENABLE_GC != 0
static bool
commit_gc_and_check_suspend_flags(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 br_depth)
{
AOTBlock *block_dst;
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
return false;
}
if (comp_ctx->aot_frame) {
/* Note that GC is enabled, no need to check it again */
if (!aot_gen_commit_values(comp_ctx->aot_frame))
return false;
if (block_dst->label_type == LABEL_TYPE_LOOP) {
if (comp_ctx->enable_thread_mgr) {
/* Note that GC is enabled, no need to check it again */
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, false))
return false;
}
}
else {
if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached)
block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp;
}
}
/* Terminate or suspend current thread only when this is
a backward jump */
if (comp_ctx->enable_thread_mgr
&& block_dst->label_type == LABEL_TYPE_LOOP) {
if (!check_suspend_flags(comp_ctx, func_ctx, true))
return false;
}
return true;
}
static bool
compile_gc_cond_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 br_depth, LLVMValueRef value_cmp)
{
AOTBlock *block_dst;
LLVMValueRef value, *values = NULL;
LLVMBasicBlockRef llvm_else_block, next_llvm_end_block;
char name[32];
uint32 i, param_index, result_index;
uint64 size;
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
return false;
}
/* Create llvm else block */
CREATE_BLOCK(llvm_else_block, "br_if_else");
MOVE_BLOCK_AFTER_CURR(llvm_else_block);
if (block_dst->label_type == LABEL_TYPE_LOOP) {
/* Dest block is Loop block */
/* Handle Loop parameters */
if (block_dst->param_count) {
size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count;
if (size >= UINT32_MAX
|| !(values = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
for (i = 0; i < block_dst->param_count; i++) {
param_index = block_dst->param_count - 1 - i;
POP(value, block_dst->param_types[param_index]);
ADD_TO_PARAM_PHIS(block_dst, value, param_index);
values[param_index] = value;
}
for (i = 0; i < block_dst->param_count; i++) {
PUSH(values[i], block_dst->param_types[i]);
}
wasm_runtime_free(values);
values = NULL;
}
BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block, llvm_else_block);
/* Move builder to else block */
SET_BUILDER_POS(llvm_else_block);
}
else {
/* Dest block is Block/If/Function block */
/* Create the end block */
if (!block_dst->llvm_end_block) {
format_block_name(name, sizeof(name), block_dst->block_index,
block_dst->label_type, LABEL_END);
CREATE_BLOCK(block_dst->llvm_end_block, name);
if ((next_llvm_end_block = find_next_llvm_end_block(block_dst)))
MOVE_BLOCK_BEFORE(block_dst->llvm_end_block,
next_llvm_end_block);
}
/* Set reachable flag and create condition br IR */
block_dst->is_reachable = true;
/* Handle result values */
if (block_dst->result_count) {
size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count;
if (size >= UINT32_MAX
|| !(values = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
CREATE_RESULT_VALUE_PHIS(block_dst);
for (i = 0; i < block_dst->result_count; i++) {
result_index = block_dst->result_count - 1 - i;
POP(value, block_dst->result_types[result_index]);
values[result_index] = value;
ADD_TO_RESULT_PHIS(block_dst, value, result_index);
}
for (i = 0; i < block_dst->result_count; i++) {
PUSH(values[i], block_dst->result_types[i]);
}
wasm_runtime_free(values);
values = NULL;
}
/* Condition jump to end block */
BUILD_COND_BR(value_cmp, block_dst->llvm_end_block, llvm_else_block);
/* Move builder to else block */
SET_BUILDER_POS(llvm_else_block);
}
return true;
fail:
if (values)
wasm_runtime_free(values);
return false;
}
bool
aot_compile_op_br_on_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 br_depth, uint8 **p_frame_ip)
{
LLVMValueRef gc_obj, value_cmp;
if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) {
return false;
}
POP_GC_REF(gc_obj);
if (!(value_cmp =
LLVMBuildIsNull(comp_ctx->builder, gc_obj, "cmp_gc_obj"))) {
aot_set_last_error("llvm build isnull failed.");
goto fail;
}
if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, value_cmp)) {
goto fail;
}
PUSH_GC_REF(gc_obj);
return true;
fail:
return false;
}
bool
aot_compile_op_br_on_non_null(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 br_depth,
uint8 **p_frame_ip)
{
LLVMValueRef gc_obj, value_cmp;
if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) {
return false;
}
GET_GC_REF_FROM_STACK(gc_obj);
if (!(value_cmp =
LLVMBuildIsNotNull(comp_ctx->builder, gc_obj, "cmp_gc_obj"))) {
aot_set_last_error("llvm build isnotnull failed.");
goto fail;
}
if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, value_cmp)) {
goto fail;
}
POP_GC_REF(gc_obj);
return true;
fail:
return false;
}
bool
aot_compile_op_br_on_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
int32 heap_type, bool nullable, bool br_on_fail,
uint32 br_depth, uint8 **p_frame_ip)
{
LLVMValueRef gc_obj, is_null, castable, not_castable, br_if_phi;
LLVMBasicBlockRef block_curr, block_non_null, block_br_if;
if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) {
return false;
}
GET_GC_REF_FROM_STACK(gc_obj);
block_curr = CURR_BLOCK();
CREATE_BLOCK(block_non_null, "obj_non_null");
MOVE_BLOCK_AFTER_CURR(block_non_null);
CREATE_BLOCK(block_br_if, "br_if");
MOVE_BLOCK_AFTER(block_br_if, block_non_null);
SET_BUILDER_POS(block_br_if);
if (!(br_if_phi =
LLVMBuildPhi(comp_ctx->builder, INT1_TYPE, "br_if_phi"))) {
aot_set_last_error("llvm build phi failed.");
goto fail;
}
SET_BUILDER_POS(block_curr);
if (!(is_null = LLVMBuildIsNull(comp_ctx->builder, gc_obj, "is_null"))) {
aot_set_last_error("llvm build isnull failed.");
goto fail;
}
BUILD_COND_BR(is_null, block_br_if, block_non_null);
if ((!br_on_fail && nullable) || (br_on_fail && !nullable)) {
LLVMAddIncoming(br_if_phi, &I1_ONE, &block_curr, 1);
}
else { /* (!br_on_fail && !nullable) || (br_on_fail && nullable)) */
LLVMAddIncoming(br_if_phi, &I1_ZERO, &block_curr, 1);
}
SET_BUILDER_POS(block_non_null);
if (heap_type >= 0) {
if (!aot_call_aot_obj_is_instance_of(comp_ctx, func_ctx, gc_obj,
I32_CONST(heap_type), &castable))
goto fail;
}
else {
if (!aot_call_wasm_obj_is_type_of(comp_ctx, func_ctx, gc_obj,
I32_CONST(heap_type), &castable))
goto fail;
}
if (!br_on_fail) {
if (!(castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, castable,
I8_ZERO, "castable"))) {
aot_set_last_error("llvm build icmp failed.");
return false;
}
LLVMAddIncoming(br_if_phi, &castable, &block_non_null, 1);
}
else {
if (!(not_castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
castable, I8_ZERO, "castable"))) {
aot_set_last_error("llvm build icmp failed.");
return false;
}
LLVMAddIncoming(br_if_phi, &not_castable, &block_non_null, 1);
}
BUILD_BR(block_br_if);
SET_BUILDER_POS(block_br_if);
if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, br_if_phi)) {
goto fail;
}
return true;
fail:
return false;
}
#endif /* End of WASM_ENABLE_GC != 0 */

20
core/iwasm/compilation/aot_emit_control.h
View File

@ -50,9 +50,25 @@ bool
aot_handle_next_reachable_block(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint8 **p_frame_ip);
#if WASM_ENABLE_THREAD_MGR != 0
bool
check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
bool check_terminate_and_suspend);
#if WASM_ENABLE_GC != 0
bool
aot_compile_op_br_on_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 br_depth, uint8 **p_frame_ip);
bool
aot_compile_op_br_on_non_null(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 br_depth,
uint8 **p_frame_ip);
bool
aot_compile_op_br_on_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
int32 heap_type, bool nullable, bool br_on_fail,
uint32 br_depth, uint8 **p_frame_ip);
#endif
#ifdef __cplusplus

88
core/iwasm/compilation/aot_emit_exception.c
View File

@ -7,6 +7,46 @@
#include "../interpreter/wasm_runtime.h"
#include "../aot/aot_runtime.h"
static bool
commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef exce_ip, bool is_64bit)
{
LLVMValueRef cur_frame = func_ctx->cur_frame;
LLVMValueRef value_offset, value_addr, value_ptr;
uint32 offset_ip;
if (!comp_ctx->is_jit_mode)
offset_ip = comp_ctx->pointer_size * 4;
else
offset_ip = offsetof(WASMInterpFrame, ip);
if (!(value_offset = I32_CONST(offset_ip))) {
aot_set_last_error("llvm build const failed");
return false;
}
if (!(value_addr =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame,
&value_offset, 1, "ip_addr"))) {
aot_set_last_error("llvm build in bounds gep failed");
return false;
}
if (!(value_ptr = LLVMBuildBitCast(
comp_ctx->builder, value_addr,
is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) {
aot_set_last_error("llvm build bit cast failed");
return false;
}
if (!LLVMBuildStore(comp_ctx->builder, exce_ip, value_ptr)) {
aot_set_last_error("llvm build store failed");
return false;
}
return true;
}
bool
aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
int32 exception_id, bool is_cond_br, LLVMValueRef cond_br_if,
@ -16,6 +56,7 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef exce_id = I32_CONST((uint32)exception_id), func_const, func;
LLVMTypeRef param_types[2], ret_type, func_type, func_ptr_type;
LLVMValueRef param_values[2];
bool is_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false;
bh_assert(exception_id >= 0 && exception_id < EXCE_NUM);
@ -32,13 +73,23 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMPositionBuilderAtEnd(comp_ctx->builder,
func_ctx->got_exception_block);
/* Create exection id phi */
/* Create exception id phi */
if (!(func_ctx->exception_id_phi = LLVMBuildPhi(
comp_ctx->builder, I32_TYPE, "exception_id_phi"))) {
aot_set_last_error("llvm build phi failed.");
return false;
}
if (comp_ctx->aot_frame) {
/* Create exception ip phi */
if (!(func_ctx->exception_ip_phi = LLVMBuildPhi(
comp_ctx->builder, is_64bit ? I64_TYPE : I32_TYPE,
"exception_ip_phi"))) {
aot_set_last_error("llvm build phi failed.");
return false;
}
}
/* Call aot_set_exception_with_id() to throw exception */
param_types[0] = INT8_PTR_TYPE;
param_types[1] = I32_TYPE;
@ -103,6 +154,12 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return false;
}
if (comp_ctx->aot_frame) {
if (!commit_ip(comp_ctx, func_ctx, func_ctx->exception_ip_phi,
is_64bit))
return false;
}
/* Create return IR */
AOTFuncType *aot_func_type = func_ctx->aot_func->func_type;
if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) {
@ -116,6 +173,35 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* Add phi incoming value to got_exception block */
LLVMAddIncoming(func_ctx->exception_id_phi, &exce_id, &block_curr, 1);
if (comp_ctx->aot_frame) {
const uint8 *ip = comp_ctx->aot_frame->frame_ip;
LLVMValueRef exce_ip = NULL;
if (!comp_ctx->is_jit_mode) {
WASMModule *module = comp_ctx->comp_data->wasm_module;
if (is_64bit)
exce_ip =
I64_CONST((uint64)(uintptr_t)(ip - module->load_addr));
else
exce_ip =
I32_CONST((uint32)(uintptr_t)(ip - module->load_addr));
}
else {
if (is_64bit)
exce_ip = I64_CONST((uint64)(uintptr_t)ip);
else
exce_ip = I32_CONST((uint32)(uintptr_t)ip);
}
if (!exce_ip) {
aot_set_last_error("llvm build const failed");
return false;
}
/* Add phi incoming value to got_exception block */
LLVMAddIncoming(func_ctx->exception_ip_phi, &exce_ip, &block_curr, 1);
}
if (!is_cond_br) {
/* not condition br, create br IR */
if (!LLVMBuildBr(comp_ctx->builder, func_ctx->got_exception_block)) {

1473
core/iwasm/compilation/aot_emit_function.c
View File

File diff suppressed because it is too large Load Diff

6
core/iwasm/compilation/aot_emit_function.h
View File

@ -30,6 +30,12 @@ bool
aot_compile_op_ref_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 func_idx);
#if WASM_ENABLE_GC != 0
bool
aot_compile_op_call_ref(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_idx, bool tail_call);
#endif
#ifdef __cplusplus
} /* end of extern "C" */
#endif

2144
core/iwasm/compilation/aot_emit_gc.c Normal file
View File

File diff suppressed because it is too large Load Diff

119
core/iwasm/compilation/aot_emit_gc.h Normal file
View File

@ -0,0 +1,119 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _AOT_EMIT_GC_H_
#define _AOT_EMIT_GC_H_
#include "aot_compiler.h"
#include "aot_runtime.h"
#ifdef __cplusplus
extern "C" {
#endif
#if WASM_ENABLE_GC != 0
bool
aot_call_aot_create_func_obj(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef func_idx, LLVMValueRef *p_gc_obj);
bool
aot_call_aot_obj_is_instance_of(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, LLVMValueRef gc_obj,
LLVMValueRef heap_type, LLVMValueRef *castable);
bool
aot_call_wasm_obj_is_type_of(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef gc_obj, LLVMValueRef heap_type,
LLVMValueRef *castable);
bool
aot_call_aot_rtt_type_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef type_index, LLVMValueRef *rtt_type);
bool
aot_compile_op_ref_as_non_null(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_struct_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index, bool init_with_default);
bool
aot_compile_op_struct_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index, uint32 field_idx, bool sign);
bool
aot_compile_op_struct_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index, uint32 field_idx);
bool
aot_compile_op_array_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index, bool init_with_default,
bool fixed_size, uint32 array_len);
bool
aot_compile_op_array_new_data(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 type_index,
uint32 data_seg_index);
bool
aot_array_obj_length(AOTCompContext *comp_ctx, LLVMValueRef array_obj,
LLVMValueRef *p_array_len);
bool
aot_array_obj_elem_addr(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef array_obj, LLVMValueRef elem_idx,
LLVMValueRef *p_elem_data, uint8 array_elem_type);
bool
aot_compile_op_array_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index, bool sign);
bool
aot_compile_op_array_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index);
bool
aot_compile_op_array_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index);
bool
aot_compile_op_array_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 type_index, uint32 src_type_index);
bool
aot_compile_op_array_len(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_op_i31_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_op_i31_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
bool sign);
bool
aot_compile_op_ref_test(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
int32 heap_type, bool nullable);
bool
aot_compile_op_ref_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
int32 heap_type, bool nullable);
bool
aot_compile_op_extern_internalize(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_extern_externalize(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
#endif
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* end of _AOT_EMIT_GC_H_ */

15
core/iwasm/compilation/aot_emit_memory.c
View File

@ -338,7 +338,7 @@ fail:
} \
} while (0)
#if WASM_ENABLE_SHARED_MEMORY != 0
#if WASM_ENABLE_SHARED_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0
bool
check_memory_alignment(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef addr, uint32 align)
@ -376,7 +376,9 @@ check_memory_alignment(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
fail:
return false;
}
#endif /* WASM_ENABLE_SHARED_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 */
#if WASM_ENABLE_SHARED_MEMORY != 0
#define BUILD_ATOMIC_LOAD(align, data_type) \
do { \
if (!(check_memory_alignment(comp_ctx, func_ctx, maddr, align))) { \
@ -874,9 +876,8 @@ fail:
return false;
}
#if WASM_ENABLE_BULK_MEMORY != 0
static LLVMValueRef
#if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0
LLVMValueRef
check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef offset, LLVMValueRef bytes)
{
@ -971,7 +972,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
fail:
return NULL;
}
#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 or WASM_ENABLE_STRINGREF != 0 */
#if WASM_ENABLE_BULK_MEMORY != 0
bool
aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 seg_index)
@ -1501,13 +1504,11 @@ aot_compile_op_atomic_wait(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
PUSH_I32(ret_value);
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (comp_ctx->enable_thread_mgr) {
if (!check_suspend_flags(comp_ctx, func_ctx))
if (!check_suspend_flags(comp_ctx, func_ctx, false))
return false;
}
#endif
return true;
fail:

8
core/iwasm/compilation/aot_emit_memory.h
View File

@ -61,6 +61,14 @@ aot_compile_op_memory_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
check_memory_alignment(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef addr, uint32 align);
LLVMValueRef
check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef offset, LLVMValueRef bytes);
#if WASM_ENABLE_BULK_MEMORY != 0
bool
aot_compile_op_memory_init(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,

43
core/iwasm/compilation/aot_emit_parametric.c
View File

@ -21,8 +21,8 @@ pop_value_from_wasm_stack(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return false;
}
aot_value =
aot_value_stack_pop(&func_ctx->block_stack.block_list_end->value_stack);
aot_value = aot_value_stack_pop(
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack);
type = aot_value->type;
if (aot_value->type == VALUE_TYPE_I1) {
@ -44,19 +44,34 @@ pop_value_from_wasm_stack(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
wasm_runtime_free(aot_value);
/* is_32: i32, f32, ref.func, ref.extern, v128 */
if (is_32
&& !(type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32
|| type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF
|| type == VALUE_TYPE_V128)) {
aot_set_last_error("invalid WASM stack data type.");
return false;
if (is_32) {
/* is_32: i32, f32, ref.func, ref.extern, v128,
or GC ref types */
if (!(type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32
|| type == VALUE_TYPE_V128
|| (comp_ctx->enable_ref_types
&& (type == VALUE_TYPE_FUNCREF
|| type == VALUE_TYPE_EXTERNREF))
#if WASM_ENABLE_GC != 0
|| (comp_ctx->enable_gc && type == VALUE_TYPE_GC_REF)
#endif
)) {
aot_set_last_error("invalid WASM stack data type.");
return false;
}
}
/* !is_32: i64, f64 */
if (!is_32 && !(type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64)) {
aot_set_last_error("invalid WASM stack data type.");
return false;
else {
/* !is_32: i64, f64, or GC ref types */
if (!(type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64
#if WASM_ENABLE_GC != 0
|| (comp_ctx->enable_gc && type == VALUE_TYPE_GC_REF)
/* may be i32 which denotes funcref/externref */
|| (!comp_ctx->enable_gc && type == VALUE_TYPE_I32)
#endif
)) {
aot_set_last_error("invalid WASM stack data type.");
return false;
}
}
return true;

1443
core/iwasm/compilation/aot_emit_stringref.c Normal file
View File

File diff suppressed because it is too large Load Diff

112
core/iwasm/compilation/aot_emit_stringref.h Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _AOT_EMIT_STRINGREF_H_
#define _AOT_EMIT_STRINGREF_H_
#include "aot_compiler.h"
#ifdef __cplusplus
extern "C" {
#endif
bool
aot_compile_op_string_new(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 encoding);
bool
aot_compile_op_string_const(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 contents);
bool
aot_compile_op_string_measure(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 encoding);
bool
aot_compile_op_string_encode(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 mem_idx, uint32 encoding);
bool
aot_compile_op_string_concat(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_string_eq(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_op_string_is_usv_sequence(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_string_as_wtf8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_wtf8_advance(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_wtf8_encode(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 mem_idx,
uint32 encoding);
bool
aot_compile_op_stringview_wtf8_slice(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_string_as_wtf16(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_wtf16_length(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_wtf16_get_codeunit(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_wtf16_encode(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint32 mem_idx);
bool
aot_compile_op_stringview_wtf16_slice(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_string_as_iter(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_iter_next(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_iter_advance(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_iter_rewind(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_stringview_iter_slice(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_op_string_new_array(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 encoding);
bool
aot_compile_op_string_encode_array(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 encoding);
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* end of _AOT_EMIT_STRINGREF_H_ */

203
core/iwasm/compilation/aot_emit_table.c
View File

@ -6,6 +6,9 @@
#include "aot_emit_table.h"
#include "aot_emit_exception.h"
#include "../aot/aot_runtime.h"
#if WASM_ENABLE_GC != 0
#include "aot_emit_gc.h"
#endif
uint64
get_tbl_inst_offset(const AOTCompContext *comp_ctx,
@ -18,13 +21,16 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx,
offset =
offsetof(AOTModuleInstance, global_table_data.bytes)
+ (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance)
+ comp_ctx->comp_data->global_data_size;
/* Get global data size according to target info */
+ (comp_ctx->pointer_size == sizeof(uint64)
? comp_ctx->comp_data->global_data_size_64bit
: comp_ctx->comp_data->global_data_size_32bit);
while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) {
offset += offsetof(AOTTableInstance, elems);
/* avoid loading from current AOTTableInstance */
offset +=
sizeof(uint32)
(uint64)comp_ctx->pointer_size
* aot_get_imp_tbl_data_slots(imp_tbls + i, comp_ctx->is_jit_mode);
++i;
}
@ -38,7 +44,7 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx,
while (i < tbl_idx && i < comp_ctx->comp_data->table_count) {
offset += offsetof(AOTTableInstance, elems);
/* avoid loading from current AOTTableInstance */
offset += sizeof(uint32)
offset += (uint64)comp_ctx->pointer_size
* aot_get_tbl_data_slots(tbls + i, comp_ctx->is_jit_mode);
++i;
}
@ -58,7 +64,7 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx)
return offset_32;
}
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
LLVMValueRef
aot_compile_get_tbl_inst(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
@ -183,7 +189,8 @@ bool
aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 tbl_idx)
{
LLVMValueRef elem_idx, offset, table_elem, func_idx;
LLVMValueRef elem_idx, offset, func_idx;
LLVMValueRef table_elem_base, table_elem_addr, table_elem;
POP_I32(elem_idx);
@ -198,34 +205,66 @@ aot_compile_op_table_get(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
goto fail;
}
if (!(table_elem = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
"table_elem_i8p"))) {
if (!(table_elem_base = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset,
1, "table_elem_base_i8p"))) {
aot_set_last_error("llvm build add failed.");
goto fail;
}
if (!(table_elem = LLVMBuildBitCast(comp_ctx->builder, table_elem,
INT32_PTR_TYPE, "table_elem_i32p"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* Load function object reference or function index */
if (comp_ctx->enable_gc) {
if (!(table_elem_base =
LLVMBuildBitCast(comp_ctx->builder, table_elem_base,
GC_REF_PTR_TYPE, "table_elem_base"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* Load function index */
if (!(table_elem =
LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, table_elem,
&elem_idx, 1, "table_elem"))) {
HANDLE_FAILURE("LLVMBuildNUWAdd");
goto fail;
}
if (!(table_elem_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, GC_REF_TYPE, table_elem_base, &elem_idx, 1,
"table_elem_addr"))) {
HANDLE_FAILURE("LLVMBuildNUWAdd");
goto fail;
}
if (!(func_idx = LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, table_elem,
"func_idx"))) {
HANDLE_FAILURE("LLVMBuildLoad");
goto fail;
}
if (!(table_elem = LLVMBuildLoad2(comp_ctx->builder, GC_REF_TYPE,
table_elem_addr, "table_elem"))) {
HANDLE_FAILURE("LLVMBuildLoad");
goto fail;
}
PUSH_I32(func_idx);
PUSH_GC_REF(table_elem);
}
else {
if (!(table_elem_base =
LLVMBuildBitCast(comp_ctx->builder, table_elem_base,
INTPTR_T_PTR_TYPE, "table_elem_base"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
if (!(table_elem_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, INTPTR_T_TYPE, table_elem_base, &elem_idx,
1, "table_elem_addr"))) {
HANDLE_FAILURE("LLVMBuildNUWAdd");
goto fail;
}
if (!(table_elem = LLVMBuildLoad2(comp_ctx->builder, INTPTR_T_TYPE,
table_elem_addr, "table_elem"))) {
HANDLE_FAILURE("LLVMBuildLoad");
goto fail;
}
if (!(func_idx = LLVMBuildIntCast2(comp_ctx->builder, table_elem,
I32_TYPE, true, "func_idx"))) {
HANDLE_FAILURE("LLVMBuildIntCast");
goto fail;
}
PUSH_I32(func_idx);
}
return true;
fail:
@ -236,44 +275,72 @@ bool
aot_compile_op_table_set(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 tbl_idx)
{
LLVMValueRef val, elem_idx, offset, table_elem;
LLVMValueRef val = NULL, elem_idx, offset, table_elem_base, table_elem_addr;
if (comp_ctx->enable_gc)
POP_GC_REF(val);
else {
POP_I32(val);
if (!(val = LLVMBuildIntCast2(comp_ctx->builder, val, INTPTR_T_TYPE,
true, "val_intptr_t"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
}
POP_I32(val);
POP_I32(elem_idx);
if (!aot_check_table_access(comp_ctx, func_ctx, tbl_idx, elem_idx)) {
goto fail;
}
/* load data as i32* */
/* load data as gc_obj_ref* or i32* */
if (!(offset = I32_CONST(get_tbl_inst_offset(comp_ctx, func_ctx, tbl_idx)
+ offsetof(AOTTableInstance, elems)))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
if (!(table_elem = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
"table_elem_i8p"))) {
if (!(table_elem_base = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset,
1, "table_elem_base_i8p"))) {
HANDLE_FAILURE("LLVMBuildInBoundsGEP");
goto fail;
}
if (!(table_elem = LLVMBuildBitCast(comp_ctx->builder, table_elem,
INT32_PTR_TYPE, "table_elem_i32p"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
if (comp_ctx->enable_gc) {
if (!(table_elem_base =
LLVMBuildBitCast(comp_ctx->builder, table_elem_base,
GC_REF_PTR_TYPE, "table_elem_base"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
if (!(table_elem_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, GC_REF_TYPE, table_elem_base, &elem_idx, 1,
"table_elem_addr"))) {
HANDLE_FAILURE("LLVMBuildInBoundsGEP");
goto fail;
}
}
else {
if (!(table_elem_base =
LLVMBuildBitCast(comp_ctx->builder, table_elem_base,
INTPTR_T_PTR_TYPE, "table_elem_base"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
if (!(table_elem_addr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, INTPTR_T_TYPE, table_elem_base, &elem_idx,
1, "table_elem_addr"))) {
HANDLE_FAILURE("LLVMBuildInBoundsGEP");
goto fail;
}
}
/* Load function index */
if (!(table_elem =
LLVMBuildInBoundsGEP2(comp_ctx->builder, I32_TYPE, table_elem,
&elem_idx, 1, "table_elem"))) {
HANDLE_FAILURE("LLVMBuildInBoundsGEP");
goto fail;
}
if (!(LLVMBuildStore(comp_ctx->builder, val, table_elem))) {
if (!(LLVMBuildStore(comp_ctx->builder, val, table_elem_addr))) {
HANDLE_FAILURE("LLVMBuildStore");
goto fail;
}
@ -434,7 +501,7 @@ aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[0] = INT8_PTR_TYPE;
param_types[1] = I32_TYPE;
param_types[2] = I32_TYPE;
param_types[3] = I32_TYPE;
param_types[3] = INT8_PTR_TYPE;
ret_type = I32_TYPE;
if (comp_ctx->is_jit_mode)
@ -452,7 +519,25 @@ aot_compile_op_table_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* n */
POP_I32(param_values[2]);
/* v */
POP_I32(param_values[3]);
if (comp_ctx->enable_gc) {
POP_GC_REF(param_values[3]);
if (!(param_values[3] =
LLVMBuildBitCast(comp_ctx->builder, param_values[3],
INT8_PTR_TYPE, "table_elem_i8p"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
}
else {
POP_I32(param_values[3]);
if (!(param_values[3] =
LLVMBuildIntToPtr(comp_ctx->builder, param_values[3],
INT8_PTR_TYPE, "table_elem_i8p"))) {
HANDLE_FAILURE("LLVMBuildIntToPtr");
goto fail;
}
}
if (!(ret = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values,
4, "table_grow"))) {
@ -477,7 +562,7 @@ aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
param_types[0] = INT8_PTR_TYPE;
param_types[1] = I32_TYPE;
param_types[2] = I32_TYPE;
param_types[3] = I32_TYPE;
param_types[3] = INT8_PTR_TYPE;
param_types[4] = I32_TYPE;
ret_type = VOID_TYPE;
@ -496,7 +581,25 @@ aot_compile_op_table_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
/* n */
POP_I32(param_values[2]);
/* v */
POP_I32(param_values[3]);
if (comp_ctx->enable_gc) {
POP_GC_REF(param_values[3]);
if (!(param_values[3] =
LLVMBuildBitCast(comp_ctx->builder, param_values[3],
INT8_PTR_TYPE, "table_elem_i8p"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
}
else {
POP_I32(param_values[3]);
if (!(param_values[3] =
LLVMBuildIntToPtr(comp_ctx->builder, param_values[3],
INT8_PTR_TYPE, "table_elem_i8p"))) {
HANDLE_FAILURE("LLVMBuildIntToPtr");
goto fail;
}
}
/* i */
POP_I32(param_values[4]);
@ -512,4 +615,4 @@ fail:
return false;
}
#endif /* WASM_ENABLE_REF_TYPES != 0 */
#endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC !=0 */

134
core/iwasm/compilation/aot_emit_variable.c
View File

@ -17,13 +17,21 @@
} while (0)
static uint8
get_local_type(AOTFuncContext *func_ctx, uint32 local_idx)
get_local_type(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 local_idx)
{
AOTFunc *aot_func = func_ctx->aot_func;
uint32 param_count = aot_func->func_type->param_count;
return local_idx < param_count
? aot_func->func_type->types[local_idx]
: aot_func->local_types[local_idx - param_count];
uint8 local_type;
local_type = local_idx < param_count
? aot_func->func_type->types[local_idx]
: aot_func->local_types_wp[local_idx - param_count];
if (comp_ctx->enable_gc && aot_is_type_gc_reftype(local_type))
local_type = VALUE_TYPE_GC_REF;
return local_type;
}
bool
@ -37,7 +45,7 @@ aot_compile_op_get_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
CHECK_LOCAL(local_idx);
local_type = get_local_type(func_ctx, local_idx);
local_type = get_local_type(comp_ctx, func_ctx, local_idx);
snprintf(name, sizeof(name), "%s%d%s", "local", local_idx, "#");
if (!(value = LLVMBuildLoad2(comp_ctx->builder, TO_LLVM_TYPE(local_type),
@ -58,15 +66,56 @@ fail:
return false;
}
bool
aot_compile_op_set_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 local_idx)
static bool
aot_compile_op_set_or_tee_local(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx, uint32 local_idx,
bool is_tee_local)
{
LLVMValueRef value;
uint8 local_type;
uint32 n;
CHECK_LOCAL(local_idx);
POP(value, get_local_type(func_ctx, local_idx));
local_type = get_local_type(comp_ctx, func_ctx, local_idx);
POP(value, local_type);
if (comp_ctx->aot_frame) {
/* Get the slot index */
n = func_ctx->aot_func->local_offsets[local_idx];
bh_assert(comp_ctx->aot_frame->lp[n].type == local_type);
switch (local_type) {
case VALUE_TYPE_I32:
set_local_i32(comp_ctx->aot_frame, n, value);
break;
case VALUE_TYPE_I64:
set_local_i64(comp_ctx->aot_frame, n, value);
break;
case VALUE_TYPE_F32:
set_local_f32(comp_ctx->aot_frame, n, value);
break;
case VALUE_TYPE_F64:
set_local_f64(comp_ctx->aot_frame, n, value);
break;
case VALUE_TYPE_V128:
set_local_v128(comp_ctx->aot_frame, n, value);
break;
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
set_local_ref(comp_ctx->aot_frame, n, value, local_type);
break;
#if WASM_ENABLE_GC != 0
case VALUE_TYPE_GC_REF:
set_local_gc_ref(comp_ctx->aot_frame, n, value, local_type);
break;
#endif
default:
bh_assert(0);
break;
}
}
if (!LLVMBuildStore(comp_ctx->builder, value,
func_ctx->locals[local_idx])) {
@ -74,6 +123,10 @@ aot_compile_op_set_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return false;
}
if (is_tee_local) {
PUSH(value, local_type);
}
aot_checked_addr_list_del(func_ctx, local_idx);
return true;
@ -81,31 +134,19 @@ fail:
return false;
}
bool
aot_compile_op_set_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 local_idx)
{
return aot_compile_op_set_or_tee_local(comp_ctx, func_ctx, local_idx,
false);
}
bool
aot_compile_op_tee_local(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
uint32 local_idx)
{
LLVMValueRef value;
uint8 type;
CHECK_LOCAL(local_idx);
type = get_local_type(func_ctx, local_idx);
POP(value, type);
if (!LLVMBuildStore(comp_ctx->builder, value,
func_ctx->locals[local_idx])) {
aot_set_last_error("llvm build store fail");
return false;
}
PUSH(value, type);
aot_checked_addr_list_del(func_ctx, local_idx);
return true;
fail:
return false;
return aot_compile_op_set_or_tee_local(comp_ctx, func_ctx, local_idx, true);
}
static bool
@ -127,17 +168,29 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
bh_assert(global_idx < import_global_count + comp_data->global_count);
if (global_idx < import_global_count) {
global_offset = global_base_offset
+ comp_data->import_globals[global_idx].data_offset;
global_offset =
global_base_offset
/* Get global data offset according to target info */
+ (comp_ctx->pointer_size == sizeof(uint64)
? comp_data->import_globals[global_idx].data_offset_64bit
: comp_data->import_globals[global_idx].data_offset_32bit);
global_type = comp_data->import_globals[global_idx].type;
}
else {
global_offset =
global_base_offset
+ comp_data->globals[global_idx - import_global_count].data_offset;
/* Get global data offset according to target info */
+ (comp_ctx->pointer_size == sizeof(uint64)
? comp_data->globals[global_idx - import_global_count]
.data_offset_64bit
: comp_data->globals[global_idx - import_global_count]
.data_offset_32bit);
global_type = comp_data->globals[global_idx - import_global_count].type;
}
if (comp_ctx->enable_gc && aot_is_type_gc_reftype(global_type))
global_type = VALUE_TYPE_GC_REF;
offset = I32_CONST(global_offset);
if (!(global_ptr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
@ -150,20 +203,25 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
case VALUE_TYPE_I32:
case VALUE_TYPE_EXTERNREF:
case VALUE_TYPE_FUNCREF:
ptr_type = comp_ctx->basic_types.int32_ptr_type;
ptr_type = INT32_PTR_TYPE;
break;
case VALUE_TYPE_I64:
ptr_type = comp_ctx->basic_types.int64_ptr_type;
ptr_type = INT64_PTR_TYPE;
break;
case VALUE_TYPE_F32:
ptr_type = comp_ctx->basic_types.float32_ptr_type;
ptr_type = F32_PTR_TYPE;
break;
case VALUE_TYPE_F64:
ptr_type = comp_ctx->basic_types.float64_ptr_type;
ptr_type = F64_PTR_TYPE;
break;
case VALUE_TYPE_V128:
ptr_type = comp_ctx->basic_types.v128_ptr_type;
ptr_type = V128_PTR_TYPE;
break;
#if WASM_ENABLE_GC != 0
case VALUE_TYPE_GC_REF:
ptr_type = GC_REF_PTR_TYPE;
break;
#endif
default:
bh_assert("unknown type");
break;

388
core/iwasm/compilation/aot_llvm.c
View File

@ -10,6 +10,7 @@
#include "aot_emit_table.h"
#include "../aot/aot_runtime.h"
#include "../aot/aot_intrinsic.h"
#include "../interpreter/wasm_runtime.h"
#if WASM_ENABLE_DEBUG_AOT != 0
#include "debug/dwarf_extractor.h"
@ -25,13 +26,20 @@ create_native_stack_top_min(const AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
LLVMTypeRef
wasm_type_to_llvm_type(const AOTLLVMTypes *llvm_types, uint8 wasm_type)
wasm_type_to_llvm_type(const AOTCompContext *comp_ctx,
const AOTLLVMTypes *llvm_types, uint8 wasm_type)
{
switch (wasm_type) {
case VALUE_TYPE_I32:
return llvm_types->int32_type;
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
return llvm_types->int32_type;
if (comp_ctx->enable_ref_types)
return llvm_types->int32_type;
else {
bh_assert(comp_ctx->enable_gc);
return llvm_types->gc_ref_type;
}
case VALUE_TYPE_I64:
return llvm_types->int64_type;
case VALUE_TYPE_F32:
@ -42,9 +50,31 @@ wasm_type_to_llvm_type(const AOTLLVMTypes *llvm_types, uint8 wasm_type)
return llvm_types->i64x2_vec_type;
case VALUE_TYPE_VOID:
return llvm_types->void_type;
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_NULLREF:
/* case REF_TYPE_FUNCREF: */
/* case REF_TYPE_EXTERNREF: */
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
case VALUE_TYPE_GC_REF:
bh_assert(comp_ctx->enable_gc);
return llvm_types->gc_ref_type;
default:
break;
}
bh_assert(0);
return NULL;
}
@ -245,18 +275,6 @@ aot_estimate_stack_usage_for_function_call(const AOTCompContext *comp_ctx,
return size;
}
static uint32
get_inst_extra_offset(AOTCompContext *comp_ctx)
{
const AOTCompData *comp_data = comp_ctx->comp_data;
uint32 table_count = comp_data->import_table_count + comp_data->table_count;
uint64 offset = get_tbl_inst_offset(comp_ctx, NULL, table_count);
uint32 offset_32 = (uint32)offset;
bh_assert(offset <= UINT32_MAX);
offset_32 = align_uint((uint32)offset_32, 8);
return offset_32;
}
/*
* a "precheck" function performs a few things before calling wrapped_func.
*
@ -353,7 +371,7 @@ aot_build_precheck_function(AOTCompContext *comp_ctx, LLVMModuleRef module,
LLVMValueRef offset;
LLVMValueRef stack_sizes_p;
offset_u32 = get_inst_extra_offset(comp_ctx);
offset_u32 = get_module_inst_extra_offset(comp_ctx);
offset_u32 += offsetof(AOTModuleInstanceExtra, stack_sizes);
offset = I32_CONST(offset_u32);
if (!offset) {
@ -552,6 +570,44 @@ fail:
return false;
}
static bool
check_wasm_type(AOTCompContext *comp_ctx, uint8 type)
{
if (type == VALUE_TYPE_FUNCREF || type == VALUE_TYPE_EXTERNREF) {
if (!comp_ctx->enable_ref_types && !comp_ctx->enable_gc) {
aot_set_last_error("funcref or externref type was found, "
"try removing --disable-ref-types option "
"or adding --enable-gc option.");
return false;
}
else
return true;
}
else if (aot_is_type_gc_reftype(type)) {
if (!comp_ctx->enable_gc) {
aot_set_last_error("GC reference type was found, "
"try adding --enable-gc option.");
return false;
}
else
return true;
}
else if (type == VALUE_TYPE_V128) {
if (!comp_ctx->enable_simd) {
aot_set_last_error("SIMD type was found, try removing "
" --disable-simd option.");
return false;
}
return true;
}
else if (type != VALUE_TYPE_I32 && type != VALUE_TYPE_I64
&& type != VALUE_TYPE_F32 && type != VALUE_TYPE_F64) {
bh_assert(0);
}
return true;
}
/**
* Add LLVM function
*/
@ -560,6 +616,8 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module,
const AOTFuncType *aot_func_type, uint32 func_index,
LLVMTypeRef *p_func_type, LLVMValueRef *p_precheck_func)
{
WASMFunction *aot_func =
comp_ctx->comp_data->wasm_module->functions[func_index];
LLVMValueRef func = NULL;
LLVMTypeRef *param_types, ret_type, func_type;
LLVMTypeRef func_type_wrapper;
@ -570,6 +628,18 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module,
uint32 i, j = 0, param_count = (uint64)aot_func_type->param_count;
uint32 backend_thread_num, compile_thread_num;
/* Check function parameter types and result types */
for (i = 0; i < aot_func_type->param_count + aot_func_type->result_count;
i++) {
if (!check_wasm_type(comp_ctx, aot_func_type->types[i]))
return NULL;
}
/* Check function local types */
for (i = 0; i < aot_func->local_count; i++) {
if (!check_wasm_type(comp_ctx, aot_func->local_types[i]))
return NULL;
}
/* exec env as first parameter */
param_count++;
@ -921,6 +991,49 @@ create_aux_stack_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
return true;
}
static bool
create_aux_stack_frame(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef wasm_stack_top_bound_ptr, offset;
offset = I32_ONE;
if (!(func_ctx->cur_frame_ptr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1,
"cur_frame_ptr"))) {
aot_set_last_error("llvm build in bounds gep failed");
return false;
}
if (!(func_ctx->cur_frame =
LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
func_ctx->cur_frame_ptr, "cur_frame"))) {
aot_set_last_error("llvm build load failed");
return false;
}
/* Get exec_env->wasm_stack.top_boundary and its address */
offset = I32_TEN;
if (!(wasm_stack_top_bound_ptr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1,
"wasm_stack_top_bound_ptr"))
|| !(func_ctx->wasm_stack_top_bound = LLVMBuildLoad2(
comp_ctx->builder, INT8_PTR_TYPE, wasm_stack_top_bound_ptr,
"wasm_stack_top_bound"))) {
aot_set_last_error("load wasm_stack.top_boundary failed");
return false;
}
offset = I32_ELEVEN;
if (!(func_ctx->wasm_stack_top_ptr = LLVMBuildInBoundsGEP2(
comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1,
"wasm_stack_top_ptr"))) {
aot_set_last_error("llvm build inbounds gep failed");
return false;
}
return true;
}
static bool
create_native_symbol(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
@ -955,7 +1068,8 @@ create_local_variables(const AOTCompData *comp_data,
const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
const AOTFunc *func)
{
AOTFuncType *aot_func_type = comp_data->func_types[func->func_type_index];
AOTFuncType *aot_func_type =
(AOTFuncType *)comp_data->types[func->func_type_index];
char local_name[32];
uint32 i, j = 1;
@ -980,14 +1094,14 @@ create_local_variables(const AOTCompData *comp_data,
LLVMValueRef local_value = NULL;
snprintf(local_name, sizeof(local_name), "l%d",
aot_func_type->param_count + i);
local_type = TO_LLVM_TYPE(func->local_types[i]);
local_type = TO_LLVM_TYPE(func->local_types_wp[i]);
func_ctx->locals[aot_func_type->param_count + i] =
LLVMBuildAlloca(comp_ctx->builder, local_type, local_name);
if (!func_ctx->locals[aot_func_type->param_count + i]) {
aot_set_last_error("llvm build alloca failed.");
return false;
}
switch (func->local_types[i]) {
switch (func->local_types_wp[i]) {
case VALUE_TYPE_I32:
local_value = I32_ZERO;
break;
@ -1005,8 +1119,33 @@ create_local_variables(const AOTCompData *comp_data,
break;
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
local_value = REF_NULL;
if (!comp_ctx->enable_gc)
local_value = REF_NULL;
else
local_value = GC_REF_NULL;
break;
#if WASM_ENABLE_GC != 0
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_NULLREF:
/* case REF_TYPE_FUNCREF: */
/* case REF_TYPE_EXTERNREF: */
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
local_value = GC_REF_NULL;
break;
#endif
default:
bh_assert(0);
break;
@ -1539,7 +1678,8 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx,
AOTFunc *func, uint32 func_index)
{
AOTFuncContext *func_ctx;
AOTFuncType *aot_func_type = comp_data->func_types[func->func_type_index];
AOTFuncType *aot_func_type =
(AOTFuncType *)comp_data->types[func->func_type_index];
WASMModule *module = comp_ctx->comp_data->wasm_module;
WASMFunction *wasm_func = module->functions[func_index];
AOTBlock *aot_block;
@ -1597,6 +1737,11 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx,
goto fail;
}
if (comp_ctx->enable_aux_stack_frame
&& !create_aux_stack_frame(comp_ctx, func_ctx)) {
goto fail;
}
/* Create local variables */
if (!create_local_variables(comp_data, comp_ctx, func_ctx, func)) {
goto fail;
@ -1634,13 +1779,14 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx,
fail:
if (func_ctx->mem_info)
wasm_runtime_free(func_ctx->mem_info);
aot_block_stack_destroy(&func_ctx->block_stack);
aot_block_stack_destroy(comp_ctx, &func_ctx->block_stack);
wasm_runtime_free(func_ctx);
return NULL;
}
static void
aot_destroy_func_contexts(AOTFuncContext **func_ctxes, uint32 count)
aot_destroy_func_contexts(AOTCompContext *comp_ctx, AOTFuncContext **func_ctxes,
uint32 count)
{
uint32 i;
@ -1648,7 +1794,7 @@ aot_destroy_func_contexts(AOTFuncContext **func_ctxes, uint32 count)
if (func_ctxes[i]) {
if (func_ctxes[i]->mem_info)
wasm_runtime_free(func_ctxes[i]->mem_info);
aot_block_stack_destroy(&func_ctxes[i]->block_stack);
aot_block_stack_destroy(comp_ctx, &func_ctxes[i]->block_stack);
aot_checked_addr_list_destroy(func_ctxes[i]);
wasm_runtime_free(func_ctxes[i]);
}
@ -1685,7 +1831,8 @@ aot_create_func_contexts(const AOTCompData *comp_data, AOTCompContext *comp_ctx)
AOTFunc *func = comp_data->funcs[i];
if (!(func_ctxes[i] =
aot_create_func_context(comp_data, comp_ctx, func, i))) {
aot_destroy_func_contexts(func_ctxes, comp_data->func_count);
aot_destroy_func_contexts(comp_ctx, func_ctxes,
comp_data->func_count);
return NULL;
}
}
@ -1694,7 +1841,8 @@ aot_create_func_contexts(const AOTCompData *comp_data, AOTCompContext *comp_ctx)
}
static bool
aot_set_llvm_basic_types(AOTLLVMTypes *basic_types, LLVMContextRef context)
aot_set_llvm_basic_types(AOTLLVMTypes *basic_types, LLVMContextRef context,
int pointer_size)
{
basic_types->int1_type = LLVMInt1TypeInContext(context);
basic_types->int8_type = LLVMInt8TypeInContext(context);
@ -1759,15 +1907,29 @@ aot_set_llvm_basic_types(AOTLLVMTypes *basic_types, LLVMContextRef context)
basic_types->funcref_type = LLVMInt32TypeInContext(context);
basic_types->externref_type = LLVMInt32TypeInContext(context);
if (pointer_size == 4) {
basic_types->intptr_t_type = basic_types->int32_type;
basic_types->intptr_t_ptr_type = basic_types->int32_ptr_type;
}
else {
basic_types->intptr_t_type = basic_types->int64_type;
basic_types->intptr_t_ptr_type = basic_types->int64_ptr_type;
}
basic_types->gc_ref_type = LLVMPointerType(basic_types->void_type, 0);
basic_types->gc_ref_ptr_type = LLVMPointerType(basic_types->gc_ref_type, 0);
return (basic_types->int8_ptr_type && basic_types->int8_pptr_type
&& basic_types->int16_ptr_type && basic_types->int32_ptr_type
&& basic_types->int64_ptr_type && basic_types->float32_ptr_type
&& basic_types->int64_ptr_type && basic_types->intptr_t_type
&& basic_types->intptr_t_ptr_type && basic_types->float32_ptr_type
&& basic_types->float64_ptr_type && basic_types->i8x16_vec_type
&& basic_types->i16x8_vec_type && basic_types->i32x4_vec_type
&& basic_types->i64x2_vec_type && basic_types->f32x4_vec_type
&& basic_types->f64x2_vec_type && basic_types->i1x2_vec_type
&& basic_types->meta_data_type && basic_types->funcref_type
&& basic_types->externref_type)
&& basic_types->externref_type && basic_types->gc_ref_type
&& basic_types->gc_ref_ptr_type)
? true
: false;
}
@ -1787,6 +1949,9 @@ aot_create_llvm_consts(AOTLLVMConsts *consts, AOTCompContext *comp_ctx)
if (!(consts->i8_zero = I8_CONST(0)))
return false;
if (!(consts->i8_one = I8_CONST(1)))
return false;
if (!(consts->f32_zero = F32_CONST(0)))
return false;
@ -1857,6 +2022,13 @@ aot_create_llvm_consts(AOTLLVMConsts *consts, AOTCompContext *comp_ctx)
CREATE_VEC_ZERO_MASK(2)
#undef CREATE_VEC_ZERO_MASK
if (!(consts->gc_ref_null =
LLVMConstNull(comp_ctx->basic_types.gc_ref_type)))
return false;
if (!(consts->i8_ptr_null =
LLVMConstNull(comp_ctx->basic_types.int8_ptr_type)))
return false;
return true;
}
@ -2369,6 +2541,12 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
if (option->enable_aux_stack_frame)
comp_ctx->enable_aux_stack_frame = true;
if (option->enable_perf_profiling)
comp_ctx->enable_perf_profiling = true;
if (option->enable_memory_profiling)
comp_ctx->enable_memory_profiling = true;
if (option->enable_aux_stack_check)
comp_ctx->enable_aux_stack_check = true;
@ -2399,6 +2577,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
if (option->builtin_intrinsics)
comp_ctx->builtin_intrinsics = option->builtin_intrinsics;
if (option->enable_gc)
comp_ctx->enable_gc = true;
comp_ctx->opt_level = option->opt_level;
comp_ctx->size_level = option->size_level;
@ -2885,6 +3066,29 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
}
LLVMDisposeMessage(triple);
#if WASM_ENABLE_WAMR_COMPILER != 0
WASMModule *wasm_module = (WASMModule *)comp_data->wasm_module;
/* Return error if SIMD is disabled by command line but SIMD instructions
* are used */
if (!option->enable_simd && wasm_module->is_simd_used) {
aot_set_last_error("SIMD is disabled by --disable-simd but SIMD "
"instructions are used in this module");
goto fail;
}
/* Disable features when they are not actually used */
if (!wasm_module->is_simd_used) {
option->enable_simd = comp_ctx->enable_simd = false;
}
if (!wasm_module->is_ref_types_used) {
option->enable_ref_types = comp_ctx->enable_ref_types = false;
}
if (!wasm_module->is_bulk_memory_used) {
option->enable_bulk_memory = comp_ctx->enable_bulk_memory = false;
}
#endif
if (option->enable_simd && strcmp(comp_ctx->target_arch, "x86_64") != 0
&& strncmp(comp_ctx->target_arch, "aarch64", 7) != 0) {
/* Disable simd if it isn't supported by target arch */
@ -2935,7 +3139,8 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
goto fail;
}
if (!aot_set_llvm_basic_types(&comp_ctx->basic_types, comp_ctx->context)) {
if (!aot_set_llvm_basic_types(&comp_ctx->basic_types, comp_ctx->context,
comp_ctx->pointer_size)) {
aot_set_last_error("create LLVM basic types failed.");
goto fail;
}
@ -3013,7 +3218,7 @@ aot_destroy_comp_context(AOTCompContext *comp_ctx)
LLVMOrcDisposeLLLazyJIT(comp_ctx->orc_jit);
if (comp_ctx->func_ctxes)
aot_destroy_func_contexts(comp_ctx->func_ctxes,
aot_destroy_func_contexts(comp_ctx, comp_ctx->func_ctxes,
comp_ctx->func_ctx_count);
if (bh_list_length(&comp_ctx->native_symbols) > 0) {
@ -3030,6 +3235,10 @@ aot_destroy_comp_context(AOTCompContext *comp_ctx)
wasm_runtime_free(comp_ctx->target_cpu);
}
if (comp_ctx->aot_frame) {
wasm_runtime_free(comp_ctx->aot_frame);
}
wasm_runtime_free(comp_ctx);
}
@ -3108,7 +3317,8 @@ aot_get_native_symbol_index(AOTCompContext *comp_ctx, const char *symbol)
}
void
aot_value_stack_push(AOTValueStack *stack, AOTValue *value)
aot_value_stack_push(const AOTCompContext *comp_ctx, AOTValueStack *stack,
AOTValue *value)
{
if (!stack->value_list_head)
stack->value_list_head = stack->value_list_end = value;
@ -3117,10 +3327,44 @@ aot_value_stack_push(AOTValueStack *stack, AOTValue *value)
value->prev = stack->value_list_end;
stack->value_list_end = value;
}
if (comp_ctx->aot_frame) {
switch (value->type) {
case VALUE_TYPE_I32:
case VALUE_TYPE_I1:
push_i32(comp_ctx->aot_frame, value);
break;
case VALUE_TYPE_I64:
push_i64(comp_ctx->aot_frame, value);
break;
case VALUE_TYPE_F32:
push_f32(comp_ctx->aot_frame, value);
break;
case VALUE_TYPE_F64:
push_f64(comp_ctx->aot_frame, value);
break;
case VALUE_TYPE_V128:
push_v128(comp_ctx->aot_frame, value);
break;
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
push_ref(comp_ctx->aot_frame, value);
break;
#if WASM_ENABLE_GC != 0
case VALUE_TYPE_GC_REF:
bh_assert(comp_ctx->enable_gc);
push_gc_ref(comp_ctx->aot_frame, value);
break;
#endif
default:
bh_assert(0);
break;
}
}
}
AOTValue *
aot_value_stack_pop(AOTValueStack *stack)
aot_value_stack_pop(const AOTCompContext *comp_ctx, AOTValueStack *stack)
{
AOTValue *value = stack->value_list_end;
@ -3134,11 +3378,49 @@ aot_value_stack_pop(AOTValueStack *stack)
value->prev = NULL;
}
if (comp_ctx->aot_frame) {
bh_assert(value);
bh_assert(value->value == (comp_ctx->aot_frame->sp - 1)->value);
bh_assert(value->type == (comp_ctx->aot_frame->sp - 1)->type);
switch (value->type) {
case VALUE_TYPE_I32:
case VALUE_TYPE_I1:
pop_i32(comp_ctx->aot_frame);
break;
case VALUE_TYPE_I64:
pop_i64(comp_ctx->aot_frame);
break;
case VALUE_TYPE_F32:
pop_f32(comp_ctx->aot_frame);
break;
case VALUE_TYPE_F64:
pop_f64(comp_ctx->aot_frame);
break;
case VALUE_TYPE_V128:
pop_v128(comp_ctx->aot_frame);
break;
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
pop_ref(comp_ctx->aot_frame);
break;
#if WASM_ENABLE_GC != 0
case VALUE_TYPE_GC_REF:
bh_assert(comp_ctx->enable_gc);
pop_gc_ref(comp_ctx->aot_frame);
break;
#endif
default:
bh_assert(0);
break;
}
}
return value;
}
void
aot_value_stack_destroy(AOTValueStack *stack)
aot_value_stack_destroy(AOTCompContext *comp_ctx, AOTValueStack *stack)
{
AOTValue *value = stack->value_list_head, *p;
@ -3183,14 +3465,14 @@ aot_block_stack_pop(AOTBlockStack *stack)
}
void
aot_block_stack_destroy(AOTBlockStack *stack)
aot_block_stack_destroy(AOTCompContext *comp_ctx, AOTBlockStack *stack)
{
AOTBlock *block = stack->block_list_head, *p;
while (block) {
p = block->next;
aot_value_stack_destroy(&block->value_stack);
aot_block_destroy(block);
aot_value_stack_destroy(comp_ctx, &block->value_stack);
aot_block_destroy(comp_ctx, block);
block = p;
}
@ -3199,9 +3481,9 @@ aot_block_stack_destroy(AOTBlockStack *stack)
}
void
aot_block_destroy(AOTBlock *block)
aot_block_destroy(AOTCompContext *comp_ctx, AOTBlock *block)
{
aot_value_stack_destroy(&block->value_stack);
aot_value_stack_destroy(comp_ctx, &block->value_stack);
if (block->param_types)
wasm_runtime_free(block->param_types);
if (block->param_phis)
@ -3316,8 +3598,38 @@ aot_build_zero_function_ret(const AOTCompContext *comp_ctx,
break;
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
ret = LLVMBuildRet(comp_ctx->builder, REF_NULL);
if (comp_ctx->enable_ref_types)
ret = LLVMBuildRet(comp_ctx->builder, REF_NULL);
#if WASM_ENABLE_GC != 0
else if (comp_ctx->enable_gc)
ret = LLVMBuildRet(comp_ctx->builder, GC_REF_NULL);
#endif
else
bh_assert(0);
break;
#if WASM_ENABLE_GC != 0
case REF_TYPE_NULLFUNCREF:
case REF_TYPE_NULLEXTERNREF:
case REF_TYPE_NULLREF:
/* case REF_TYPE_FUNCREF: */
/* case REF_TYPE_EXTERNREF: */
case REF_TYPE_ANYREF:
case REF_TYPE_EQREF:
case REF_TYPE_HT_NULLABLE:
case REF_TYPE_HT_NON_NULLABLE:
case REF_TYPE_I31REF:
case REF_TYPE_STRUCTREF:
case REF_TYPE_ARRAYREF:
#if WASM_ENABLE_STRINGREF != 0
case REF_TYPE_STRINGREF:
case REF_TYPE_STRINGVIEWWTF8:
case REF_TYPE_STRINGVIEWWTF16:
case REF_TYPE_STRINGVIEWITER:
#endif
bh_assert(comp_ctx->enable_gc);
ret = LLVMBuildRet(comp_ctx->builder, GC_REF_NULL);
break;
#endif
default:
bh_assert(0);
}

134
core/iwasm/compilation/aot_llvm.h
View File

@ -35,6 +35,7 @@
#endif
#include "aot_orc_extra.h"
#include "aot_comp_option.h"
#ifdef __cplusplus
extern "C" {
@ -64,6 +65,8 @@ extern "C" {
#undef DUMP_MODULE
#endif
struct AOTValueSlot;
/**
* Value in the WASM operation stack, each stack element
* is an LLVM value
@ -86,6 +89,53 @@ typedef struct AOTValueStack {
AOTValue *value_list_end;
} AOTValueStack;
/* Record information of a value slot of local variable or stack
during translation */
typedef struct AOTValueSlot {
/* The LLVM value of this slot */
LLVMValueRef value;
/* The value type of this slot */
uint8 type;
/* The dirty bit of the value slot. It's set if the value in
register is newer than the value in memory. */
uint32 dirty : 1;
/* Whether the new value in register is a reference, which is valid
only when the dirty bit is set. */
uint32 ref : 1;
/* Committed reference flag:
0: uncommitted, 1: not-reference, 2: reference */
uint32 committed_ref : 2;
} AOTValueSlot;
/* Frame information for translation */
typedef struct AOTCompFrame {
/* The current compilation context */
struct AOTCompContext *comp_ctx;
/* The current function context */
struct AOTFuncContext *func_ctx;
/* The current instruction pointer which is being compiled */
const uint8 *frame_ip;
/* Max local slot number */
uint32 max_local_cell_num;
/* Max operand stack slot number */
uint32 max_stack_cell_num;
/* Size of current AOTFrame/WASMInterpFrame */
uint32 cur_frame_size;
/* Stack top pointer */
AOTValueSlot *sp;
/* Local variables + stack operands */
AOTValueSlot lp[1];
} AOTCompFrame;
typedef struct AOTBlock {
struct AOTBlock *next;
struct AOTBlock *prev;
@ -124,6 +174,12 @@ typedef struct AOTBlock {
uint32 result_count;
uint8 *result_types;
LLVMValueRef *result_phis;
/* The begin frame stack pointer of this block */
AOTValueSlot *frame_sp_begin;
/* The max frame stack pointer that br/br_if/br_table/br_on_xxx
opcodes ever reached when they jumped to the end this block */
AOTValueSlot *frame_sp_max_reached;
} AOTBlock;
/**
@ -176,12 +232,19 @@ typedef struct AOTFuncContext {
LLVMValueRef cur_exception;
LLVMValueRef cur_frame;
LLVMValueRef cur_frame_ptr;
LLVMValueRef wasm_stack_top_bound;
LLVMValueRef wasm_stack_top_ptr;
bool mem_space_unchanged;
AOTCheckedAddrList checked_addr_list;
LLVMBasicBlockRef got_exception_block;
LLVMBasicBlockRef func_return_block;
LLVMValueRef exception_id_phi;
/* current ip when exception is thrown */
LLVMValueRef exception_ip_phi;
LLVMValueRef func_type_indexes;
#if WASM_ENABLE_DEBUG_AOT != 0
LLVMMetadataRef debug_func;
@ -198,6 +261,7 @@ typedef struct AOTLLVMTypes {
LLVMTypeRef int16_type;
LLVMTypeRef int32_type;
LLVMTypeRef int64_type;
LLVMTypeRef intptr_t_type;
LLVMTypeRef float32_type;
LLVMTypeRef float64_type;
LLVMTypeRef void_type;
@ -207,6 +271,7 @@ typedef struct AOTLLVMTypes {
LLVMTypeRef int16_ptr_type;
LLVMTypeRef int32_ptr_type;
LLVMTypeRef int64_ptr_type;
LLVMTypeRef intptr_t_ptr_type;
LLVMTypeRef float32_ptr_type;
LLVMTypeRef float64_ptr_type;
@ -233,12 +298,15 @@ typedef struct AOTLLVMTypes {
LLVMTypeRef funcref_type;
LLVMTypeRef externref_type;
LLVMTypeRef gc_ref_type;
LLVMTypeRef gc_ref_ptr_type;
} AOTLLVMTypes;
typedef struct AOTLLVMConsts {
LLVMValueRef i1_zero;
LLVMValueRef i1_one;
LLVMValueRef i8_zero;
LLVMValueRef i8_one;
LLVMValueRef i32_zero;
LLVMValueRef i64_zero;
LLVMValueRef f32_zero;
@ -282,6 +350,8 @@ typedef struct AOTLLVMConsts {
LLVMValueRef i32x8_zero;
LLVMValueRef i32x4_zero;
LLVMValueRef i32x2_zero;
LLVMValueRef gc_ref_null;
LLVMValueRef i8_ptr_null;
} AOTLLVMConsts;
/**
@ -339,6 +409,12 @@ typedef struct AOTCompContext {
/* Generate auxiliary stack frame */
bool enable_aux_stack_frame;
/* Function performance profiling */
bool enable_perf_profiling;
/* Memory usage profiling */
bool enable_memory_profiling;
/* Thread Manager */
bool enable_thread_mgr;
@ -380,6 +456,11 @@ typedef struct AOTCompContext {
/* Whether optimize the JITed code */
bool optimize;
bool emit_frame_pointer;
/* Enable GC */
bool enable_gc;
uint32 opt_level;
uint32 size_level;
@ -403,7 +484,6 @@ typedef struct AOTCompContext {
AOTLLVMConsts llvm_consts;
/* Function contexts */
/* TODO: */
AOTFuncContext **func_ctxes;
uint32 func_ctx_count;
char **custom_sections_wp;
@ -428,7 +508,8 @@ typedef struct AOTCompContext {
const char *llvm_passes;
const char *builtin_intrinsics;
bool emit_frame_pointer;
/* Current frame information for translation */
AOTCompFrame *aot_frame;
} AOTCompContext;
enum {
@ -438,41 +519,6 @@ enum {
AOT_LLVMIR_OPT_FILE,
};
/* always sync it with AOTCompOption in aot_export.h */
typedef struct AOTCompOption {
bool is_jit_mode;
bool is_indirect_mode;
char *target_arch;
char *target_abi;
char *target_cpu;
char *cpu_features;
bool is_sgx_platform;
bool enable_bulk_memory;
bool enable_thread_mgr;
bool enable_tail_call;
bool enable_simd;
bool enable_ref_types;
bool enable_aux_stack_check;
bool enable_aux_stack_frame;
bool disable_llvm_intrinsics;
bool disable_llvm_lto;
bool enable_llvm_pgo;
bool enable_stack_estimation;
bool quick_invoke_c_api_import;
char *use_prof_file;
uint32 opt_level;
uint32 size_level;
uint32 output_format;
uint32 bounds_checks;
uint32 stack_bounds_checks;
uint32 segue_flags;
char **custom_sections;
uint32 custom_sections_count;
const char *stack_usage_file;
const char *llvm_passes;
const char *builtin_intrinsics;
} AOTCompOption, *aot_comp_option_t;
bool
aot_compiler_init(void);
@ -498,13 +544,14 @@ void
aot_destroy_elf_file(uint8 *elf_file);
void
aot_value_stack_push(AOTValueStack *stack, AOTValue *value);
aot_value_stack_push(const AOTCompContext *comp_ctx, AOTValueStack *stack,
AOTValue *value);
AOTValue *
aot_value_stack_pop(AOTValueStack *stack);
aot_value_stack_pop(const AOTCompContext *comp_ctx, AOTValueStack *stack);
void
aot_value_stack_destroy(AOTValueStack *stack);
aot_value_stack_destroy(AOTCompContext *comp_ctx, AOTValueStack *stack);
void
aot_block_stack_push(AOTBlockStack *stack, AOTBlock *block);
@ -513,13 +560,14 @@ AOTBlock *
aot_block_stack_pop(AOTBlockStack *stack);
void
aot_block_stack_destroy(AOTBlockStack *stack);
aot_block_stack_destroy(AOTCompContext *comp_ctx, AOTBlockStack *stack);
void
aot_block_destroy(AOTBlock *block);
aot_block_destroy(AOTCompContext *comp_ctx, AOTBlock *block);
LLVMTypeRef
wasm_type_to_llvm_type(const AOTLLVMTypes *llvm_types, uint8 wasm_type);
wasm_type_to_llvm_type(const AOTCompContext *comp_ctx,
const AOTLLVMTypes *llvm_types, uint8 wasm_type);
bool
aot_checked_addr_list_add(AOTFuncContext *func_ctx, uint32 local_idx,

14
core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp
View File

@ -8461,7 +8461,7 @@ jit_codegen_compile_call_to_llvm_jit(const WASMType *func_type)
/* r10 = outs_area->lp */
{
x86::Mem m(regs_i64[hreg_info->exec_env_hreg_index],
(uint32)offsetof(WASMExecEnv, wasm_stack.s.top));
(uint32)offsetof(WASMExecEnv, wasm_stack.top));
a.mov(reg_lp, m);
a.add(reg_lp, (uint32)offsetof(WASMInterpFrame, lp));
}
@ -8701,15 +8701,15 @@ fast_jit_alloc_frame(WASMExecEnv *exec_env, uint32 param_cell_num,
* frame to store the function results from jit function to call,
* the second is the frame for the jit function
*/
if ((uint8 *)exec_env->wasm_stack.s.top + size_frame1 + size_frame2
> exec_env->wasm_stack.s.top_boundary) {
if ((uint8 *)exec_env->wasm_stack.top + size_frame1 + size_frame2
> exec_env->wasm_stack.top_boundary) {
wasm_set_exception(module_inst, "wasm operand stack overflow");
return NULL;
}
/* Allocate the frame */
frame = (WASMInterpFrame *)exec_env->wasm_stack.s.top;
exec_env->wasm_stack.s.top += size_frame1;
frame = (WASMInterpFrame *)exec_env->wasm_stack.top;
exec_env->wasm_stack.top += size_frame1;
frame->function = NULL;
frame->ip = NULL;
@ -9073,9 +9073,9 @@ jit_codegen_compile_call_to_fast_jit(const WASMModule *module, uint32 func_idx)
/* rdx = exec_env->cur_frame->prev_frame */
a.mov(x86::rdx,
x86::ptr(x86::rsi, (uint32)offsetof(WASMInterpFrame, prev_frame)));
/* exec_env->wasm_stack.s.top = cur_frame */
/* exec_env->wasm_stack.top = cur_frame */
{
x86::Mem m(x86::rdi, offsetof(WASMExecEnv, wasm_stack.s.top));
x86::Mem m(x86::rdi, offsetof(WASMExecEnv, wasm_stack.top));
a.mov(m, x86::rsi);
}
/* exec_env->cur_frame = prev_frame */

9
core/iwasm/fast-jit/fe/jit_emit_control.c
View File

@ -396,8 +396,9 @@ handle_func_return(JitCompContext *cc, JitBlock *block)
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
/* time_end = os_time_get_boot_us() */
if (!jit_emit_callnative(cc, os_time_get_boot_us, time_end, NULL, 0)) {
/* time_end = os_time_thread_cputime_us() */
if (!jit_emit_callnative(cc, os_time_thread_cputime_us, time_end, NULL,
0)) {
return false;
}
/* time_start = cur_frame->time_started */
@ -449,9 +450,9 @@ handle_func_return(JitCompContext *cc, JitBlock *block)
}
/* Free stack space of the current frame:
exec_env->wasm_stack.s.top = cur_frame */
exec_env->wasm_stack.top = cur_frame */
GEN_INSN(STPTR, cc->fp_reg, cc->exec_env_reg,
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top)));
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top)));
/* Set the prev_frame as the current frame:
exec_env->cur_frame = prev_frame */
GEN_INSN(STPTR, prev_frame, cc->exec_env_reg,

4
core/iwasm/fast-jit/fe/jit_emit_function.c
View File

@ -499,7 +499,9 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
if (UINTPTR_MAX == UINT64_MAX) {
offset_i32 = jit_cc_new_reg_I32(cc);
offset = jit_cc_new_reg_I64(cc);
GEN_INSN(SHL, offset_i32, elem_idx, NEW_CONST(I32, 2));
/* Calculate offset by pointer size (elem_idx *
* sizeof(table_elem_type_t)) */
GEN_INSN(SHL, offset_i32, elem_idx, NEW_CONST(I32, 3));
GEN_INSN(I32TOI64, offset, offset_i32);
}
else {

41
core/iwasm/fast-jit/fe/jit_emit_table.c
View File

@ -46,7 +46,8 @@ jit_compile_op_table_get(JitCompContext *cc, uint32 tbl_idx)
GEN_INSN(I32TOI64, elem_idx_long, elem_idx);
offset = jit_cc_new_reg_I64(cc);
GEN_INSN(MUL, offset, elem_idx_long, NEW_CONST(I64, sizeof(uint32)));
GEN_INSN(MUL, offset, elem_idx_long,
NEW_CONST(I64, sizeof(table_elem_type_t)));
res = jit_cc_new_reg_I32(cc);
tbl_elems = get_table_elems_reg(cc->jit_frame, tbl_idx);
@ -77,7 +78,8 @@ jit_compile_op_table_set(JitCompContext *cc, uint32 tbl_idx)
GEN_INSN(I32TOI64, elem_idx_long, elem_idx);
offset = jit_cc_new_reg_I64(cc);
GEN_INSN(MUL, offset, elem_idx_long, NEW_CONST(I64, sizeof(uint32)));
GEN_INSN(MUL, offset, elem_idx_long,
NEW_CONST(I64, sizeof(table_elem_type_t)));
tbl_elems = get_table_elems_reg(cc->jit_frame, tbl_idx);
GEN_INSN(STI32, elem_val, tbl_elems, offset);
@ -92,14 +94,15 @@ wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 seg_idx,
uint32 dst_offset, uint32 len, uint32 src_offset)
{
WASMTableInstance *tbl;
uint32 tbl_sz;
WASMTableSeg *tbl_seg = inst->module->table_segments + seg_idx;
uint32 *tbl_seg_elems = NULL, tbl_seg_len = 0;
InitializerExpression *tbl_seg_init_values = NULL, *init_values;
uint32 tbl_sz, tbl_seg_len = 0, i;
table_elem_type_t *addr;
if (!bh_bitmap_get_bit(inst->e->common.elem_dropped, seg_idx)) {
/* table segment isn't dropped */
tbl_seg_elems = tbl_seg->func_indexes;
tbl_seg_len = tbl_seg->function_count;
tbl_seg_init_values = tbl_seg->init_values;
tbl_seg_len = tbl_seg->value_count;
}
if (offset_len_out_of_bounds(src_offset, len, tbl_seg_len))
@ -113,10 +116,13 @@ wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 seg_idx,
if (!len)
return 0;
bh_memcpy_s((uint8 *)tbl + offsetof(WASMTableInstance, elems)
+ dst_offset * sizeof(uint32),
(uint32)((tbl_sz - dst_offset) * sizeof(uint32)),
tbl_seg_elems + src_offset, (uint32)(len * sizeof(uint32)));
addr =
(table_elem_type_t *)((uint8 *)tbl + offsetof(WASMTableInstance, elems)
+ dst_offset * sizeof(table_elem_type_t));
init_values = tbl_seg_init_values + src_offset;
for (i = 0; i < len; i++) {
addr[i] = (table_elem_type_t)(uintptr_t)init_values[+i].u.ref_index;
}
return 0;
out_of_bounds:
@ -175,12 +181,13 @@ wasm_copy_table(WASMModuleInstance *inst, uint32 src_tbl_idx,
if (offset_len_out_of_bounds(src_offset, len, src_tbl_sz))
goto out_of_bounds;
bh_memmove_s((uint8 *)dst_tbl + offsetof(WASMTableInstance, elems)
+ dst_offset * sizeof(uint32),
(uint32)((dst_tbl_sz - dst_offset) * sizeof(uint32)),
(uint8 *)src_tbl + offsetof(WASMTableInstance, elems)
+ src_offset * sizeof(uint32),
(uint32)(len * sizeof(uint32)));
bh_memmove_s(
(uint8 *)dst_tbl + offsetof(WASMTableInstance, elems)
+ dst_offset * sizeof(table_elem_type_t),
(uint32)((dst_tbl_sz - dst_offset) * sizeof(table_elem_type_t)),
(uint8 *)src_tbl + offsetof(WASMTableInstance, elems)
+ src_offset * sizeof(table_elem_type_t),
(uint32)(len * sizeof(table_elem_type_t)));
return 0;
out_of_bounds:
@ -272,7 +279,7 @@ fail:
static int
wasm_fill_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 dst_offset,
uint32 val, uint32 len)
uintptr_t val, uint32 len)
{
WASMTableInstance *tbl;
uint32 tbl_sz;

21
core/iwasm/fast-jit/jit_frontend.c
View File

@ -97,9 +97,9 @@ jit_frontend_get_table_inst_offset(const WASMModule *module, uint32 tbl_idx)
offset += (uint32)offsetof(WASMTableInstance, elems);
#if WASM_ENABLE_MULTI_MODULE != 0
offset += (uint32)sizeof(uint32) * table->max_size;
offset += (uint32)sizeof(table_elem_type_t) * table->max_size;
#else
offset += (uint32)sizeof(uint32)
offset += (uint32)sizeof(table_elem_type_t)
* (table->possible_grow ? table->max_size : table->init_size);
#endif
@ -1157,21 +1157,22 @@ init_func_translation(JitCompContext *cc)
func_inst = jit_cc_new_reg_ptr(cc);
#if WASM_ENABLE_PERF_PROFILING != 0
time_started = jit_cc_new_reg_I64(cc);
/* Call os_time_get_boot_us() to get time_started firstly
/* Call os_time_thread_cputime_us() to get time_started firstly
as there is stack frame switching below, calling native in them
may cause register spilling work inproperly */
if (!jit_emit_callnative(cc, os_time_get_boot_us, time_started, NULL, 0)) {
if (!jit_emit_callnative(cc, os_time_thread_cputime_us, time_started, NULL,
0)) {
return NULL;
}
#endif
#endif
/* top = exec_env->wasm_stack.s.top */
/* top = exec_env->wasm_stack.top */
GEN_INSN(LDPTR, top, cc->exec_env_reg,
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top)));
/* top_boundary = exec_env->wasm_stack.s.top_boundary */
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top)));
/* top_boundary = exec_env->wasm_stack.top_boundary */
GEN_INSN(LDPTR, top_boundary, cc->exec_env_reg,
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top_boundary)));
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top_boundary)));
/* frame_boundary = top + frame_size + outs_size */
GEN_INSN(ADD, frame_boundary, top, NEW_CONST(PTR, frame_size + outs_size));
/* if frame_boundary > top_boundary, throw stack overflow exception */
@ -1184,9 +1185,9 @@ init_func_translation(JitCompContext *cc)
/* Add first and then sub to reduce one used register */
/* new_top = frame_boundary - outs_size = top + frame_size */
GEN_INSN(SUB, new_top, frame_boundary, NEW_CONST(PTR, outs_size));
/* exec_env->wasm_stack.s.top = new_top */
/* exec_env->wasm_stack.top = new_top */
GEN_INSN(STPTR, new_top, cc->exec_env_reg,
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top)));
NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.top)));
/* frame_sp = frame->lp + local_size */
GEN_INSN(ADD, frame_sp, top,
NEW_CONST(PTR, offsetof(WASMInterpFrame, lp) + local_size));

46
core/iwasm/include/aot_comp_option.h Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef __AOT_COMP_OPTION_H__
#define __AOT_COMP_OPTION_H__
typedef struct AOTCompOption {
bool is_jit_mode;
bool is_indirect_mode;
char *target_arch;
char *target_abi;
char *target_cpu;
char *cpu_features;
bool is_sgx_platform;
bool enable_bulk_memory;
bool enable_thread_mgr;
bool enable_tail_call;
bool enable_simd;
bool enable_ref_types;
bool enable_gc;
bool enable_aux_stack_check;
bool enable_aux_stack_frame;
bool enable_perf_profiling;
bool enable_memory_profiling;
bool disable_llvm_intrinsics;
bool disable_llvm_lto;
bool enable_llvm_pgo;
bool enable_stack_estimation;
bool quick_invoke_c_api_import;
char *use_prof_file;
uint32_t opt_level;
uint32_t size_level;
uint32_t output_format;
uint32_t bounds_checks;
uint32_t stack_bounds_checks;
uint32_t segue_flags;
char **custom_sections;
uint32_t custom_sections_count;
const char *stack_usage_file;
const char *llvm_passes;
const char *builtin_intrinsics;
} AOTCompOption, *aot_comp_option_t;
#endif

40
core/iwasm/include/aot_export.h
View File

@ -9,6 +9,8 @@
#include <stdint.h>
#include <stdbool.h>
#include "aot_comp_option.h"
#ifdef __cplusplus
extern "C" {
#endif
@ -20,7 +22,8 @@ struct AOTCompContext;
typedef struct AOTCompContext *aot_comp_context_t;
aot_comp_data_t
aot_create_comp_data(void *wasm_module);
aot_create_comp_data(void *wasm_module, const char *target_arch,
bool gc_enabled);
void
aot_destroy_comp_data(aot_comp_data_t comp_data);
@ -38,41 +41,6 @@ enum {
AOT_LLVMIR_OPT_FILE,
};
/* always sync it with AOTCompOption in compilation/aot_llvm.h */
typedef struct AOTCompOption {
bool is_jit_mode;
bool is_indirect_mode;
char *target_arch;
char *target_abi;
char *target_cpu;
char *cpu_features;
bool is_sgx_platform;
bool enable_bulk_memory;
bool enable_thread_mgr;
bool enable_tail_call;
bool enable_simd;
bool enable_ref_types;
bool enable_aux_stack_check;
bool enable_aux_stack_frame;
bool disable_llvm_intrinsics;
bool disable_llvm_lto;
bool enable_llvm_pgo;
bool enable_stack_estimation;
bool quick_invoke_c_api_import;
char *use_prof_file;
uint32_t opt_level;
uint32_t size_level;
uint32_t output_format;
uint32_t bounds_checks;
uint32_t stack_bounds_checks;
uint32_t segue_flags;
char **custom_sections;
uint32_t custom_sections_count;
const char *stack_usage_file;
const char *llvm_passes;
const char *builtin_intrinsics;
} AOTCompOption, *aot_comp_option_t;
bool
aot_compiler_init(void);

954
core/iwasm/include/gc_export.h Normal file
View File

@ -0,0 +1,954 @@
/*
* Copyright (C) 2019 Intel Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _GC_EXPORT_H
#define _GC_EXPORT_H
#include "wasm_export.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef uint8_t wasm_value_type_t;
typedef enum wasm_value_type_enum {
VALUE_TYPE_I32 = 0x7F,
VALUE_TYPE_I64 = 0x7E,
VALUE_TYPE_F32 = 0x7D,
VALUE_TYPE_F64 = 0x7C,
VALUE_TYPE_V128 = 0x7B,
/* GC Types */
VALUE_TYPE_I8 = 0x78,
VALUE_TYPE_I16 = 0x77,
VALUE_TYPE_NULLFUNCREF = 0x73,
VALUE_TYPE_NULLEXTERNREF = 0x72,
VALUE_TYPE_NULLREF = 0x71,
VALUE_TYPE_FUNCREF = 0x70,
VALUE_TYPE_EXTERNREF = 0x6F,
VALUE_TYPE_ANYREF = 0x6E,
VALUE_TYPE_EQREF = 0x6D,
VALUE_TYPE_I31REF = 0x6C,
VALUE_TYPE_STRUCTREF = 0x6B,
VALUE_TYPE_ARRAYREF = 0x6A,
VALUE_TYPE_HT_NON_NULLABLE_REF = 0x64,
VALUE_TYPE_HT_NULLABLE_REF = 0x63,
/* Stringref Types */
VALUE_TYPE_STRINGREF = 0X67,
VALUE_TYPE_STRINGVIEWWTF8 = 0x66,
VALUE_TYPE_STRINGVIEWWTF16 = 0x62,
VALUE_TYPE_STRINGVIEWITER = 0x61
} wasm_value_type_enum;
typedef int32_t wasm_heap_type_t;
typedef enum wasm_heap_type_enum {
HEAP_TYPE_FUNC = -0x10,
HEAP_TYPE_EXTERN = -0x11,
HEAP_TYPE_ANY = -0x12,
HEAP_TYPE_EQ = -0x13,
HEAP_TYPE_I31 = -0x16,
HEAP_TYPE_NOFUNC = -0x17,
HEAP_TYPE_NOEXTERN = -0x18,
HEAP_TYPE_STRUCT = -0x19,
HEAP_TYPE_ARRAY = -0x1A,
HEAP_TYPE_NONE = -0x1B
} wasm_heap_type_enum;
struct WASMObject;
typedef struct WASMObject *wasm_obj_t;
#ifndef WASM_VALUE_DEFINED
#define WASM_VALUE_DEFINED
typedef union V128 {
int8_t i8x16[16];
int16_t i16x8[8];
int32_t i32x8[4];
int64_t i64x2[2];
float f32x4[4];
double f64x2[2];
} V128;
typedef union WASMValue {
int32_t i32;
uint32_t u32;
uint32_t global_index;
uint32_t ref_index;
int64_t i64;
uint64_t u64;
float f32;
double f64;
V128 v128;
wasm_obj_t gc_obj;
uint32_t type_index;
struct {
uint32_t type_index;
uint32_t length;
} array_new_default;
/* pointer to a memory space holding more data, current usage:
* struct.new init value: WASMStructNewInitValues *
* array.new init value: WASMArrayNewInitValues *
*/
void *data;
} WASMValue;
#endif /* end of WASM_VALUE_DEFINED */
typedef union WASMValue wasm_value_t;
/* Reference type, the layout is same as WasmRefType in wasm.h
* use wasm_ref_type_set_type_idx to initialize as concrete ref type
* use wasm_ref_type_set_heap_type to initialize as abstract ref type
*/
typedef struct wasm_ref_type_t {
wasm_value_type_t value_type;
bool nullable;
int32_t heap_type;
} wasm_ref_type_t;
/**
* Local object reference that can be traced when GC occurs. All
* native functions that need to hold WASM objects which may not be
* referenced from other elements of GC root set may be hold with
* this type of variable so that they can be traced when GC occurs.
* Before using such a variable, it must be pushed onto the stack
* (implemented as a chain) of such variables, and before leaving the
* frame of the variables, they must be popped from the stack.
*/
typedef struct WASMLocalObjectRef {
/* Previous local object reference variable on the stack */
struct WASMLocalObjectRef *prev;
/* The reference of WASM object hold by this variable */
wasm_obj_t val;
} WASMLocalObjectRef, wasm_local_obj_ref_t;
struct WASMType;
struct WASMFuncType;
struct WASMStructType;
struct WASMArrayType;
typedef struct WASMType *wasm_defined_type_t;
typedef struct WASMFuncType *wasm_func_type_t;
typedef struct WASMStructType *wasm_struct_type_t;
typedef struct WASMArrayType *wasm_array_type_t;
struct WASMExternrefObject;
struct WASMAnyrefObject;
struct WASMStructObject;
struct WASMArrayObject;
struct WASMFuncObject;
typedef struct WASMExternrefObject *wasm_externref_obj_t;
typedef struct WASMAnyrefObject *wasm_anyref_obj_t;
typedef struct WASMStructObject *wasm_struct_obj_t;
typedef struct WASMArrayObject *wasm_array_obj_t;
typedef struct WASMFuncObject *wasm_func_obj_t;
typedef struct WASMStringrefObject *wasm_stringref_obj_t;
typedef uintptr_t wasm_i31_obj_t;
typedef void (*wasm_obj_finalizer_t)(const wasm_obj_t obj, void *data);
/* Defined type related operations */
/**
* Get number of defined types in the given wasm module
*
* @param module the wasm module
*
* @return defined type count
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_get_defined_type_count(const wasm_module_t module);
/**
* Get defined type by type index
*
* @param module the wasm module
* @param index the type index
*
* @return defined type
*/
WASM_RUNTIME_API_EXTERN wasm_defined_type_t
wasm_get_defined_type(const wasm_module_t module, uint32_t index);
/**
* Get defined type of the GC managed object, the object must be struct,
* array or func.
*
* @param obj the object
*
* @return defined type of the object.
*/
WASM_RUNTIME_API_EXTERN wasm_defined_type_t
wasm_obj_get_defined_type(const wasm_obj_t obj);
/**
* Get defined type index of the GC managed object, the object must be struct,
* array or func.
*
* @param obj the object
*
* @return defined type index of the object.
*/
WASM_RUNTIME_API_EXTERN int32_t
wasm_obj_get_defined_type_idx(const wasm_module_t module, const wasm_obj_t obj);
/**
* Check whether a defined type is a function type
*
* @param def_type the defined type to be checked
*
* @return true if the defined type is function type, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_defined_type_is_func_type(const wasm_defined_type_t def_type);
/**
* Check whether a defined type is a struct type
*
* @param def_type the defined type to be checked
*
* @return true if the defined type is struct type, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_defined_type_is_struct_type(const wasm_defined_type_t def_type);
/**
* Check whether a defined type is an array type
*
* @param def_type the defined type to be checked
*
* @return true if the defined type is array type, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_defined_type_is_array_type(const wasm_defined_type_t def_type);
/**
* Get parameter count of a function type
*
* @param func_type the specified function type
*
* @return the param count of the specified function type
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_func_type_get_param_count(const wasm_func_type_t func_type);
/**
* Get type of a specified parameter of a function type
*
* @param func_type the specified function type
* @param param_idx the specified param index
*
* @return the param type at the specified param index of the specified func
* type
*/
WASM_RUNTIME_API_EXTERN wasm_ref_type_t
wasm_func_type_get_param_type(const wasm_func_type_t func_type,
uint32_t param_idx);
/**
* Get result count of a function type
*
* @param func_type the specified function type
*
* @return the result count of the specified function type
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_func_type_get_result_count(const wasm_func_type_t func_type);
/**
* Get type of a specified result of a function type
*
* @param func_type the specified function type
* @param param_idx the specified result index
*
* @return the result type at the specified result index of the specified func
* type
*/
WASM_RUNTIME_API_EXTERN wasm_ref_type_t
wasm_func_type_get_result_type(const wasm_func_type_t func_type,
uint32_t result_idx);
/**
* Get field count of a struct type
*
* @param struct_type the specified struct type
*
* @return the field count of the specified struct type
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_struct_type_get_field_count(const wasm_struct_type_t struct_type);
/**
* Get type of a specified field of a struct type
*
* @param struct_type the specified struct type
* @param field_idx index of the specified field
* @param p_is_mutable if not NULL, output the mutability of the field
*
* @return the result type at the specified field index of the specified struct
*/
WASM_RUNTIME_API_EXTERN wasm_ref_type_t
wasm_struct_type_get_field_type(const wasm_struct_type_t struct_type,
uint32_t field_idx, bool *p_is_mutable);
/**
* Get element type of an array type
*
* @param array_type the specified array type
* @param p_is_mutable if not NULL, output the mutability of the element type
*
* @return the ref type of array's elem type
*/
WASM_RUNTIME_API_EXTERN wasm_ref_type_t
wasm_array_type_get_elem_type(const wasm_array_type_t array_type,
bool *p_is_mutable);
/**
* Check whether two defined types are equal
*
* @param def_type1 the specified defined type1
* @param def_type2 the specified defined type2
* @param module current wasm module
*
* @return true if the defined type1 is equal to the defined type2,
* false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_defined_type_equal(const wasm_defined_type_t def_type1,
const wasm_defined_type_t def_type2,
const wasm_module_t module);
/**
* Check whether def_type1 is subtype of def_type2
*
* @param def_type1 the specified defined type1
* @param def_type2 the specified defined type2
* @param module current wasm module
*
* @return true if the defined type1 is subtype of the defined type2,
* false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_defined_type_is_subtype_of(const wasm_defined_type_t def_type1,
const wasm_defined_type_t def_type2,
const wasm_module_t module);
/* ref type related operations */
/**
* Set the ref_type to be (ref null? type_idx)
*
* @param ref_type the ref_type to be set
* @param nullable whether the ref_type is nullable
* @param type_idx the type index
*/
WASM_RUNTIME_API_EXTERN void
wasm_ref_type_set_type_idx(wasm_ref_type_t *ref_type, bool nullable,
int32_t type_idx);
/**
* Set the ref_type to be (ref null? func/extern/any/eq/i31/struct/array/..)
*
* @param ref_type the ref_type to be set
* @param nullable whether the ref_type is nullable
* @param heap_type the heap type
*/
WASM_RUNTIME_API_EXTERN void
wasm_ref_type_set_heap_type(wasm_ref_type_t *ref_type, bool nullable,
int32_t heap_type);
/**
* Check whether two ref types are equal
*
* @param ref_type1 the specified ref type1
* @param ref_type2 the specified ref type2
* @param module current wasm module
*
* @return true if the ref type1 is equal to the ref type2,
* false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_ref_type_equal(const wasm_ref_type_t *ref_type1,
const wasm_ref_type_t *ref_type2,
const wasm_module_t module);
/**
* Check whether ref_type1 is subtype of ref_type2
*
* @param ref_type1 the specified ref type1
* @param ref_type2 the specified ref type2
* @param module current wasm module
*
* @return true if the ref type1 is subtype of the ref type2,
* false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_ref_type_is_subtype_of(const wasm_ref_type_t *ref_type1,
const wasm_ref_type_t *ref_type2,
const wasm_module_t module);
/* wasm object related operations */
/**
* Create a struct object with the index of defined type
*
* @param exec_env the execution environment
* @param type_idx index of the struct type
*
* @return wasm_struct_obj_t if create success, NULL otherwise
*/
WASM_RUNTIME_API_EXTERN wasm_struct_obj_t
wasm_struct_obj_new_with_typeidx(wasm_exec_env_t exec_env, uint32_t type_idx);
/**
* Create a struct object with the struct type
*
* @param exec_env the execution environment
* @param type defined struct type
*
* @return wasm_struct_obj_t if create success, NULL otherwise
*/
WASM_RUNTIME_API_EXTERN wasm_struct_obj_t
wasm_struct_obj_new_with_type(wasm_exec_env_t exec_env,
const wasm_struct_type_t type);
/**
* Set the field value of a struct object
*
* @param obj the struct object to set field
* @param field_idx the specified field index
* @param value wasm value to be set
*/
WASM_RUNTIME_API_EXTERN void
wasm_struct_obj_set_field(wasm_struct_obj_t obj, uint32_t field_idx,
const wasm_value_t *value);
/**
* Get the field value of a struct object
*
* @param obj the struct object to get field
* @param field_idx the specified field index
* @param sign_extend whether to sign extend for i8 and i16 element types
* @param value output the wasm value
*/
WASM_RUNTIME_API_EXTERN void
wasm_struct_obj_get_field(const wasm_struct_obj_t obj, uint32_t field_idx,
bool sign_extend, wasm_value_t *value);
/**
* Create an array object with the index of defined type, the obj's length is
* length, init value is init_value
*
* @param exec_env the execution environment
* @param type_idx the index of the specified type
* @param length the array's length
* @param init_value the array's init value
*
* @return the created array object
*/
WASM_RUNTIME_API_EXTERN wasm_array_obj_t
wasm_array_obj_new_with_typeidx(wasm_exec_env_t exec_env, uint32_t type_idx,
uint32_t length, wasm_value_t *init_value);
/**
* Create an array object with the array type, the obj's length is length, init
* value is init_value
*
* @param exec_env the execution environment
* @param type the array's specified type
* @param length the array's length
* @param init_value the array's init value
*
* @return the created array object
*/
WASM_RUNTIME_API_EXTERN wasm_array_obj_t
wasm_array_obj_new_with_type(wasm_exec_env_t exec_env,
const wasm_array_type_t type, uint32_t length,
wasm_value_t *init_value);
/**
* Set the specified element's value of an array object
*
* @param array_obj the array object to set element value
* @param elem_idx the specified element index
* @param value wasm value to be set
*/
WASM_RUNTIME_API_EXTERN void
wasm_array_obj_set_elem(wasm_array_obj_t array_obj, uint32_t elem_idx,
const wasm_value_t *value);
/**
* Get the specified element's value of an array object
*
* @param array_obj the array object to get element value
* @param elem_idx the specified element index
* @param sign_extend whether to sign extend for i8 and i16 element types
* @param value output the wasm value
*/
WASM_RUNTIME_API_EXTERN void
wasm_array_obj_get_elem(const wasm_array_obj_t array_obj, uint32_t elem_idx,
bool sign_extend, wasm_value_t *value);
/**
* Copy elements from one array to another
*
* @param dst_obj destination array object
* @param dst_idx target index in destination
* @param src_obj source array object
* @param src_idx start index in source
* @param len length of elements to copy
*/
WASM_RUNTIME_API_EXTERN void
wasm_array_obj_copy(wasm_array_obj_t dst_obj, uint32_t dst_idx,
const wasm_array_obj_t src_obj, uint32_t src_idx,
uint32_t len);
/**
* Return the length of an array object
*
* @param array_obj the array object to get length
*
* @return length of the array object
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_array_obj_length(const wasm_array_obj_t array_obj);
/**
* Get the address of the first element of an array object
*
* @param array_obj the array object to get element address
*
* @return address of the first element
*/
WASM_RUNTIME_API_EXTERN void *
wasm_array_obj_first_elem_addr(const wasm_array_obj_t array_obj);
/**
* Get the address of the i-th element of an array object
*
* @param array_obj the array object to get element address
* @param elem_idx the specified element index
*
* @return address of the specified element
*/
WASM_RUNTIME_API_EXTERN void *
wasm_array_obj_elem_addr(const wasm_array_obj_t array_obj, uint32_t elem_idx);
/**
* Create a function object with the index of defined type and the index of the
* function
*
* @param exec_env the execution environment
* @param type_idx the index of the specified type
* @param func_idx_bound the index of the function
*
* @return the created function object
*/
WASM_RUNTIME_API_EXTERN wasm_func_obj_t
wasm_func_obj_new_with_typeidx(wasm_exec_env_t exec_env, uint32_t type_idx,
uint32_t func_idx_bound);
/**
* Create a function object with the function type and the index of the function
*
* @param exec_env the execution environment
* @param type the specified type
* @param func_idx_bound the index of the function
*
* @return the created function object
*/
WASM_RUNTIME_API_EXTERN wasm_func_obj_t
wasm_func_obj_new_with_type(wasm_exec_env_t exec_env, wasm_func_type_t type,
uint32_t func_idx_bound);
/**
* Get the function index bound of a function object
*
* @param func_obj the function object
*
* @return the bound function index
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_func_obj_get_func_idx_bound(const wasm_func_obj_t func_obj);
/**
* Get the function type of a function object
*
* @param func_obj the function object
*
* @return defined function type
*/
WASM_RUNTIME_API_EXTERN wasm_func_type_t
wasm_func_obj_get_func_type(const wasm_func_obj_t func_obj);
/**
* Call the given WASM function object with arguments (bytecode and AoT).
*
* @param exec_env the execution environment to call the function,
* which must be created from wasm_create_exec_env()
* @param func_obj the function object to call
* @param argc total cell number that the function parameters occupy,
* a cell is a slot of the uint32 array argv[], e.g. i32/f32 argument
* occupies one cell, i64/f64 argument occupies two cells, note that
* it might be different from the parameter number of the function
* @param argv the arguments. If the function has return value,
* the first (or first two in case 64-bit return value) element of
* argv stores the return value of the called WASM function after this
* function returns.
*
* @return true if success, false otherwise and exception will be thrown,
* the caller can call wasm_runtime_get_exception to get the exception
* info.
*/
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_call_func_ref(wasm_exec_env_t exec_env,
const wasm_func_obj_t func_obj, uint32_t argc,
uint32_t argv[]);
/**
* Call the given WASM function object with provided results space
* and arguments (bytecode and AoT).
*
* @param exec_env the execution environment to call the function,
* which must be created from wasm_create_exec_env()
* @param func_obj the function object to call
* @param num_results the number of results
* @param results the pre-alloced pointer to get the results
* @param num_args the number of arguments
* @param args the arguments
*
* @return true if success, false otherwise and exception will be thrown,
* the caller can call wasm_runtime_get_exception to get the exception
* info.
*/
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_call_func_ref_a(wasm_exec_env_t exec_env,
const wasm_func_obj_t func_obj,
uint32_t num_results, wasm_val_t results[],
uint32_t num_args, wasm_val_t *args);
/**
* Call the given WASM function object with provided results space and
* variant arguments (bytecode and AoT).
*
* @param exec_env the execution environment to call the function,
* which must be created from wasm_create_exec_env()
* @param func_obj the function object to call
* @param num_results the number of results
* @param results the pre-alloced pointer to get the results
* @param num_args the number of arguments
* @param ... the variant arguments
*
* @return true if success, false otherwise and exception will be thrown,
* the caller can call wasm_runtime_get_exception to get the exception
* info.
*/
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_call_func_ref_v(wasm_exec_env_t exec_env,
const wasm_func_obj_t func_obj,
uint32_t num_results, wasm_val_t results[],
uint32_t num_args, ...);
/**
* Create an externref object with host object
*
* @param exec_env the execution environment
* @param host_obj host object pointer
*
* @return wasm_externref_obj_t if success, NULL otherwise
*/
WASM_RUNTIME_API_EXTERN wasm_externref_obj_t
wasm_externref_obj_new(wasm_exec_env_t exec_env, const void *host_obj);
/**
* Get the host value of an externref object
*
* @param externref_obj the externref object
*
* @return the stored host object pointer
*/
WASM_RUNTIME_API_EXTERN const void *
wasm_externref_obj_get_value(const wasm_externref_obj_t externref_obj);
/**
* Create an anyref object with host object
*
* @param exec_env the execution environment
* @param host_obj host object pointer
*
* @return wasm_anyref_obj_t if success, NULL otherwise
*/
WASM_RUNTIME_API_EXTERN wasm_anyref_obj_t
wasm_anyref_obj_new(wasm_exec_env_t exec_env, const void *host_obj);
/**
* Get the host object value of an anyref object
*
* @param anyref_obj the anyref object
*
* @return the stored host object pointer
*/
WASM_RUNTIME_API_EXTERN const void *
wasm_anyref_obj_get_value(const wasm_anyref_obj_t anyref_obj);
/**
* Get the internal object inside the externref object, same as
* the operation of opcode extern.internalize
*
* @param externref_obj the externref object
*
* @return internalized wasm_obj_t
*/
WASM_RUNTIME_API_EXTERN wasm_obj_t
wasm_externref_obj_to_internal_obj(const wasm_externref_obj_t externref_obj);
/**
* Create an externref object from an internal object, same as
* the operation of opcode extern.externalize
*
* @param exec_env the execution environment
* @param internal_obj the internal object
*
* @return wasm_externref_obj_t if create success, NULL othersise
*/
WASM_RUNTIME_API_EXTERN wasm_externref_obj_t
wasm_internal_obj_to_externref_obj(wasm_exec_env_t exec_env,
const wasm_obj_t internal_obj);
/**
* Create an i31 object
*
* @param i31_value the scalar value
*
* @return wasm_i31_obj_t
*/
WASM_RUNTIME_API_EXTERN wasm_i31_obj_t
wasm_i31_obj_new(uint32_t i31_value);
/**
* Get value from an i31 object
*
* @param i31_obj the i31 object
* @param sign_extend whether to sign extend the value
*
* @return wasm_i31_obj_t
*/
WASM_RUNTIME_API_EXTERN uint32_t
wasm_i31_obj_get_value(wasm_i31_obj_t i31_obj, bool sign_extend);
/**
* Pin an object to make it traced during GC
*
* @param exec_env the execution environment
* @param obj the object to pin
*
* @return true if success, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_pin_object(wasm_exec_env_t exec_env, wasm_obj_t obj);
/**
* Unpin an object
*
* @param exec_env the execution environment
* @param obj the object to unpin
*
* @return true if success, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_unpin_object(wasm_exec_env_t exec_env, wasm_obj_t obj);
/**
* Check whether an object is a struct objectc
*
* @param obj the object to check
*
* @return true if the object is a struct, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_struct_obj(const wasm_obj_t obj);
/**
* Check whether an object is an array object
*
* @param obj the object to check
*
* @return true if the object is a array, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_array_obj(const wasm_obj_t obj);
/**
* Check whether an object is a function object
*
* @param obj the object to check
*
* @return true if the object is a function, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_func_obj(const wasm_obj_t obj);
/**
* Check whether an object is an i31 object
*
* @param obj the object to check
*
* @return true if the object is an i32, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_i31_obj(const wasm_obj_t obj);
/**
* Check whether an object is an externref object
*
* @param obj the object to check
*
* @return true if the object is an externref, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_externref_obj(const wasm_obj_t obj);
/**
* Check whether an object is an anyref object
*
* @param obj the object to check
*
* @return true if the object is an anyref, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_anyref_obj(const wasm_obj_t obj);
/**
* Check whether an object is a struct object, or, an i31/struct/array object
*
* @param obj the object to check
*
* @return true if the object is an internal object, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_internal_obj(const wasm_obj_t obj);
/**
* Check whether an object is an eq object
*
* @param obj the object to check
*
* @return true if the object is an eq object, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_eq_obj(const wasm_obj_t obj);
/**
* Check whether an object is an instance of a defined type
*
* @param obj the object to check
* @param defined_type the defined type
* @param module current wasm module
*
* @return true if the object is instance of the defined type, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_instance_of_defined_type(const wasm_obj_t obj,
const wasm_defined_type_t defined_type,
const wasm_module_t module);
/**
* Check whether an object is an instance of a defined type with
* index type_idx
*
* @param obj the object to check
* @param type_idx the type index
* @param module current wasm module
*
* @return true if the object is instance of the defined type specified by
* type_idx, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_instance_of_type_idx(const wasm_obj_t obj, uint32_t type_idx,
const wasm_module_t module);
/**
* Check whether an object is an instance of a ref type
*
* @param obj the object to check
* @param ref_type the ref type
*
* @return true if the object is instance of the ref type, false otherwise
*/
WASM_RUNTIME_API_EXTERN bool
wasm_obj_is_instance_of_ref_type(const wasm_obj_t obj,
const wasm_ref_type_t *ref_type);
/**
* Push a local object ref into stack, note that we should set its value
* after pushing to retain it during GC, and should pop it from stack
* before returning from the current function
*
* @param exec_env the execution environment
* @param local_obj_ref the local object ref to push
*/
WASM_RUNTIME_API_EXTERN void
wasm_runtime_push_local_obj_ref(wasm_exec_env_t exec_env,
wasm_local_obj_ref_t *local_obj_ref);
/**
* Pop a local object ref from stack
*
* @param exec_env the execution environment
*
* @return the popped wasm_local_obj_ref_t
*/
WASM_RUNTIME_API_EXTERN wasm_local_obj_ref_t *
wasm_runtime_pop_local_obj_ref(wasm_exec_env_t exec_env);
/**
* Pop n local object refs from stack
*
* @param exec_env the execution environment
* @param n number to pop
*/
WASM_RUNTIME_API_EXTERN void
wasm_runtime_pop_local_obj_refs(wasm_exec_env_t exec_env, uint32_t n);
/**
* Get current local object ref from stack
*
* @param exec_env the execution environment
*
* @return the wasm_local_obj_ref_t obj from the top of the stack, not change
* the state of the stack
*/
WASM_RUNTIME_API_EXTERN wasm_local_obj_ref_t *
wasm_runtime_get_cur_local_obj_ref(wasm_exec_env_t exec_env);
/**
* Set finalizer to the given object, if another finalizer is set to the same
* object, the previous one will be cancelled
*
* @param exec_env the execution environment
* @param obj object to set finalizer
* @param cb finalizer function to be called before this object is freed
* @param data custom data to be passed to finalizer function
*
* @return true if success, false otherwise
*/
bool
wasm_obj_set_gc_finalizer(wasm_exec_env_t exec_env, const wasm_obj_t obj,
wasm_obj_finalizer_t cb, void *data);
/**
* Unset finalizer to the given object
*
* @param exec_env the execution environment
* @param obj object to unset finalizer
*/
void
wasm_obj_unset_gc_finalizer(wasm_exec_env_t exec_env, void *obj);
#ifdef __cplusplus
}
#endif
#endif /* end of _GC_EXPORT_H */

3
core/iwasm/include/wasm_export.h
View File

@ -161,6 +161,9 @@ typedef struct RuntimeInitArgs {
/* Fast JIT code cache size */
uint32_t fast_jit_code_cache_size;
/* Default GC heap size */
uint32_t gc_heap_size;
/* Default running mode of the runtime */
RunningMode running_mode;

634
core/iwasm/interpreter/wasm.h
View File

@ -9,12 +9,15 @@
#include "bh_platform.h"
#include "bh_hashmap.h"
#include "bh_assert.h"
#if WASM_ENABLE_GC != 0
#include "gc_export.h"
#endif
#ifdef __cplusplus
extern "C" {
#endif
/** Value Type */
/* Value Type */
#define VALUE_TYPE_I32 0x7F
#define VALUE_TYPE_I64 0X7E
#define VALUE_TYPE_F32 0x7D
@ -23,29 +26,99 @@ extern "C" {
#define VALUE_TYPE_FUNCREF 0x70
#define VALUE_TYPE_EXTERNREF 0x6F
#define VALUE_TYPE_VOID 0x40
/* Packed Types */
#define PACKED_TYPE_I8 0x78
#define PACKED_TYPE_I16 0x77
/* Reference Types */
#define REF_TYPE_NULLFUNCREF 0x73
#define REF_TYPE_NULLEXTERNREF 0x72
#define REF_TYPE_NULLREF 0x71
#define REF_TYPE_FUNCREF VALUE_TYPE_FUNCREF /* 0x70 */
#define REF_TYPE_EXTERNREF VALUE_TYPE_EXTERNREF /* 0x6F */
#define REF_TYPE_ANYREF 0x6E
#define REF_TYPE_EQREF 0x6D
#define REF_TYPE_I31REF 0x6C
#define REF_TYPE_STRUCTREF 0x6B
#define REF_TYPE_ARRAYREF 0x6A
#define REF_TYPE_HT_NON_NULLABLE 0x64
#define REF_TYPE_HT_NULLABLE 0x63
#define REF_TYPE_STRINGREF VALUE_TYPE_STRINGREF /* 0x67 */
#define REF_TYPE_STRINGVIEWWTF8 VALUE_TYPE_STRINGVIEWWTF8 /* 0x66 */
#define REF_TYPE_STRINGVIEWWTF16 VALUE_TYPE_STRINGVIEWWTF16 /* 0x62 */
#define REF_TYPE_STRINGVIEWITER VALUE_TYPE_STRINGVIEWITER /* 0x61 */
/* Heap Types */
#define HEAP_TYPE_NOFUNC (-0x0D)
#define HEAP_TYPE_NOEXTERN (-0x0E)
#define HEAP_TYPE_NONE (-0x0F)
#define HEAP_TYPE_FUNC (-0x10)
#define HEAP_TYPE_EXTERN (-0x11)
#define HEAP_TYPE_ANY (-0x12)
#define HEAP_TYPE_EQ (-0x13)
#define HEAP_TYPE_I31 (-0x14)
#define HEAP_TYPE_STRUCT (-0x15)
#define HEAP_TYPE_ARRAY (-0x16)
#define HEAP_TYPE_STRINGREF (-0x19)
#define HEAP_TYPE_STRINGVIEWWTF8 (-0x1A)
#define HEAP_TYPE_STRINGVIEWWTF16 (-0x1E)
#define HEAP_TYPE_STRINGVIEWITER (-0x1F)
/* Defined Types */
#define DEFINED_TYPE_FUNC 0x60
#define DEFINED_TYPE_STRUCT 0x5F
#define DEFINED_TYPE_ARRAY 0x5E
#define DEFINED_TYPE_SUB 0x50
#define DEFINED_TYPE_SUB_FINAL 0x4F
#define DEFINED_TYPE_REC 0x4E
/* Used by AOT */
#define VALUE_TYPE_I1 0x41
/* Used by loader to represent any type of i32/i64/f32/f64 */
/**
* Used by loader to represent any type of i32/i64/f32/f64/v128
* and ref types, including funcref, externref, anyref, eqref,
* (ref null $ht), (ref $ht), i31ref, structref, arrayref,
* nullfuncref, nullexternref, nullref and stringref
*/
#define VALUE_TYPE_ANY 0x42
/**
* Used by wamr compiler to represent object ref types,
* including func object ref, externref object ref,
* internal object ref, eq obect ref, i31 object ref,
* struct object ref, array obect ref
*/
#define VALUE_TYPE_GC_REF 0x43
#define DEFAULT_NUM_BYTES_PER_PAGE 65536
#define DEFAULT_MAX_PAGES 65536
#if WASM_ENABLE_GC == 0
typedef uintptr_t table_elem_type_t;
#define NULL_REF (0xFFFFFFFF)
#else
typedef void *table_elem_type_t;
#define NULL_REF (NULL)
#define REF_CELL_NUM ((uint32)sizeof(uintptr_t) / sizeof(uint32))
#endif
#define TABLE_MAX_SIZE (1024)
#define INIT_EXPR_NONE 0x00
#define INIT_EXPR_TYPE_I32_CONST 0x41
#define INIT_EXPR_TYPE_I64_CONST 0x42
#define INIT_EXPR_TYPE_F32_CONST 0x43
#define INIT_EXPR_TYPE_F64_CONST 0x44
#define INIT_EXPR_TYPE_V128_CONST 0xFD
/* = WASM_OP_REF_FUNC */
#define INIT_EXPR_TYPE_FUNCREF_CONST 0xD2
/* = WASM_OP_REF_NULL */
#define INIT_EXPR_TYPE_REFNULL_CONST 0xD0
#define INIT_EXPR_TYPE_GET_GLOBAL 0x23
#define INIT_EXPR_TYPE_ERROR 0xff
#define INIT_EXPR_TYPE_REFNULL_CONST 0xD0
#define INIT_EXPR_TYPE_FUNCREF_CONST 0xD2
#define INIT_EXPR_TYPE_STRUCT_NEW 0xD3
#define INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT 0xD4
#define INIT_EXPR_TYPE_ARRAY_NEW 0xD5
#define INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT 0xD6
#define INIT_EXPR_TYPE_ARRAY_NEW_FIXED 0xD7
#define INIT_EXPR_TYPE_I31_NEW 0xD8
#define INIT_EXPR_TYPE_ANY_CONVERT_EXTERN 0xD9
#define INIT_EXPR_TYPE_EXTERN_CONVERT_ANY 0xDA
#define WASM_MAGIC_NUMBER 0x6d736100
#define WASM_CURRENT_VERSION 1
@ -68,6 +141,9 @@ extern "C" {
#if WASM_ENABLE_TAGS != 0
#define SECTION_TYPE_TAG 13
#endif
#if WASM_ENABLE_STRINGREF != 0
#define SECTION_TYPE_STRINGREF 14
#endif
#define SUB_SECTION_TYPE_MODULE 0
#define SUB_SECTION_TYPE_FUNC 1
@ -99,6 +175,21 @@ extern "C" {
#define LABEL_TYPE_CATCH_ALL 6
#endif
#define WASM_TYPE_FUNC 0
#define WASM_TYPE_STRUCT 1
#define WASM_TYPE_ARRAY 2
#if WASM_ENABLE_STRINGREF != 0
#define WASM_TYPE_STRINGREF 3
#define WASM_TYPE_STRINGVIEWWTF8 4
#define WASM_TYPE_STRINGVIEWWTF16 5
#define WASM_TYPE_STRINGVIEWITER 6
#endif
/* In WasmGC, a table can start with [0x40 0x00] to indicate it has an
* initializer */
#define TABLE_INIT_EXPR_FLAG 0x40
typedef struct WASMModule WASMModule;
typedef struct WASMFunction WASMFunction;
typedef struct WASMGlobal WASMGlobal;
@ -106,6 +197,8 @@ typedef struct WASMGlobal WASMGlobal;
typedef struct WASMTag WASMTag;
#endif
#ifndef WASM_VALUE_DEFINED
#define WASM_VALUE_DEFINED
typedef union V128 {
int8 i8x16[16];
int16 i16x8[8];
@ -124,44 +217,268 @@ typedef union WASMValue {
uint64 u64;
float32 f32;
float64 f64;
uintptr_t addr;
V128 v128;
#if WASM_ENABLE_GC != 0
wasm_obj_t gc_obj;
uint32 type_index;
struct {
uint32 type_index;
uint32 length;
} array_new_default;
/* pointer to a memory space holding more data, current usage:
* struct.new init value: WASMStructNewInitValues *
* array.new init value: WASMArrayNewInitValues *
*/
void *data;
#endif
} WASMValue;
#endif /* end of WASM_VALUE_DEFINED */
typedef struct WASMStructNewInitValues {
uint8 type_idx;
uint32 count;
WASMValue fields[1];
} WASMStructNewInitValues;
typedef struct WASMArrayNewInitValues {
uint8 type_idx;
uint32 length;
WASMValue elem_data[1];
} WASMArrayNewInitValues;
typedef struct InitializerExpression {
/* type of INIT_EXPR_TYPE_XXX */
/* it actually is instr, in some places, requires constant only */
/* type of INIT_EXPR_TYPE_XXX, which is an instruction of
constant expression */
uint8 init_expr_type;
WASMValue u;
} InitializerExpression;
#if WASM_ENABLE_GC != 0
/**
* Reference type of (ref null ht) or (ref ht),
* and heap type is defined type (type i), i >= 0
*/
typedef struct RefHeapType_TypeIdx {
/* ref_type is REF_TYPE_HT_NULLABLE or
REF_TYPE_HT_NON_NULLABLE, (0x6C or 0x6B) */
uint8 ref_type;
/* true if ref_type is REF_TYPE_HT_NULLABLE */
bool nullable;
/* heap type is defined type: type_index >= 0 */
int32 type_idx;
} RefHeapType_TypeIdx;
/**
* Reference type of (ref null ht) or (ref ht),
* and heap type is non-defined type
*/
typedef struct RefHeapType_Common {
/* ref_type is REF_TYPE_HT_NULLABLE or
REF_TYPE_HT_NON_NULLABLE (0x6C or 0x6B) */
uint8 ref_type;
/* true if ref_type is REF_TYPE_HT_NULLABLE */
bool nullable;
/* Common heap type (not defined type):
-0x10 (func), -0x11 (extern), -0x12 (any), -0x13 (eq),
-0x16 (i31), -0x17 (nofunc), -0x18 (noextern),
-0x19 (struct), -0x20 (array), -0x21 (none) */
int32 heap_type;
} RefHeapType_Common;
/**
* Reference type
*/
typedef union WASMRefType {
uint8 ref_type;
RefHeapType_TypeIdx ref_ht_typeidx;
RefHeapType_Common ref_ht_common;
} WASMRefType;
typedef struct WASMRefTypeMap {
/**
* The type index of a type array, which only stores
* the first byte of the type, e.g. WASMFuncType.types,
* WASMStructType.fields
*/
uint16 index;
/* The full type info if the type cannot be described
with one byte */
WASMRefType *ref_type;
} WASMRefTypeMap;
#endif /* end of WASM_ENABLE_GC */
#if WASM_ENABLE_GC == 0
typedef struct WASMFuncType WASMType;
typedef WASMType *WASMTypePtr;
#else
/**
* Common type, store the same fields of
* WASMFuncType, WASMStructType and WASMArrayType
*/
typedef struct WASMType {
/**
* type_flag must be WASM_TYPE_FUNC/STRUCT/ARRAY to
* denote that it is a WASMFuncType, WASMStructType or
* WASMArrayType
*/
uint16 type_flag;
bool is_sub_final;
/* The inheritance depth */
uint32 inherit_depth;
/* The root type */
struct WASMType *root_type;
/* The parent type */
struct WASMType *parent_type;
uint32 parent_type_idx;
/* number of internal types in the current rec group, if the type is not in
* a recursive group, rec_count = 0 */
uint16 rec_count;
uint16 rec_idx;
} WASMType, *WASMTypePtr;
#endif /* end of WASM_ENABLE_GC */
/* Function type */
typedef struct WASMFuncType {
#if WASM_ENABLE_GC != 0
WASMType base_type;
#endif
uint16 param_count;
uint16 result_count;
uint16 param_cell_num;
uint16 ret_cell_num;
uint16 ref_count;
#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0
/* Code block to call llvm jit functions of this
kind of function type from fast jit jitted code */
void *call_to_llvm_jit_from_fast_jit;
#endif
#if WASM_ENABLE_GC != 0
uint16 ref_type_map_count;
WASMRefTypeMap *ref_type_maps;
WASMRefTypeMap *result_ref_type_maps;
/* minimal type index of the type equal to this type,
used in type equal check in call_indirect opcode */
uint32 min_type_idx_normalized;
#else
uint16 ref_count;
#endif
#if WASM_ENABLE_QUICK_AOT_ENTRY != 0
/* Quick AOT/JIT entry of this func type */
void *quick_aot_entry;
#endif
/* types of params and results */
/* types of params and results, only store the first byte
* of the type, if it cannot be described with one byte,
* then the full type info is stored in ref_type_maps */
uint8 types[1];
} WASMType;
} WASMFuncType;
#if WASM_ENABLE_GC != 0
typedef struct WASMStructFieldType {
uint16 field_flags;
uint8 field_type;
uint8 field_size;
uint32 field_offset;
#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0
/*
* The field size and field offset of a wasm struct may vary
* in 32-bit target and 64-bit target, e.g., the size of a
* GC reference is 4 bytes in the former and 8 bytes in the
* latter, the AOT compiler needs to use the correct field
* offset according to the target info.
*/
uint8 field_size_64bit;
uint8 field_size_32bit;
uint32 field_offset_64bit;
uint32 field_offset_32bit;
#endif
} WASMStructFieldType;
typedef struct WASMStructType {
WASMType base_type;
/* total size of this struct object */
uint32 total_size;
uint16 field_count;
uint16 ref_type_map_count;
WASMRefTypeMap *ref_type_maps;
/* Offsets of reference fields that need to be traced during GC.
The first element of the table is the number of such offsets. */
uint16 *reference_table;
/* Field info, note that fields[i]->field_type only stores
* the first byte of the field type, if it cannot be described
* with one byte, then the full field type info is stored in
* ref_type_maps */
WASMStructFieldType fields[1];
} WASMStructType;
typedef struct WASMArrayType {
WASMType base_type;
uint16 elem_flags;
uint8 elem_type;
/* The full elem type info if the elem type cannot be
described with one byte */
WASMRefType *elem_ref_type;
} WASMArrayType;
#if WASM_ENABLE_STRINGREF != 0
/* stringref representation, we define it as a void * pointer here, the
* stringref implementation can use any structure */
/*
WasmGC heap
+-----------------------+
| |
| stringref |
| +----------+ | external string representation
| | host_ptr |--------o------+----->+------------+
| +----------+ | | | |
| | | +------------+
| stringview_wtf8/16 | |
| +----------+ | |
| | host_ptr |--------o------+
| +----------+ | |
| | |
| stringview_iter | |
| +----------+ | |
| | host_ptr |--------o------+
| +----------+ |
| | pos | |
| +----------+ |
| |
+-----------------------+
*/
typedef void *WASMString;
#endif /* end of WASM_ENABLE_STRINGREF != 0 */
#endif /* end of WASM_ENABLE_GC != 0 */
typedef struct WASMTable {
uint8 elem_type;
uint32 flags;
/**
* 0: no max size and not shared
* 1: hax max size
* 2: shared
*/
uint8 flags;
bool possible_grow;
uint32 init_size;
/* specified if (flags & 1), else it is 0x10000 */
uint32 max_size;
bool possible_grow;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
/* init expr for the whole table */
InitializerExpression init_expr;
#endif
} WASMTable;
typedef struct WASMMemory {
@ -174,12 +491,16 @@ typedef struct WASMMemory {
typedef struct WASMTableImport {
char *module_name;
char *field_name;
/* 0: no max size, 1: has max size */
uint8 elem_type;
uint32 flags;
uint8 flags;
bool possible_grow;
uint32 init_size;
/* specified if (flags & 1), else it is 0x10000 */
uint32 max_size;
bool possible_grow;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
WASMModule *import_module;
WASMTable *import_table_linked;
@ -203,19 +524,23 @@ typedef struct WASMFunctionImport {
char *module_name;
char *field_name;
/* function type */
WASMType *func_type;
WASMFuncType *func_type;
/* native function pointer after linked */
void *func_ptr_linked;
/* signature from registered native symbols */
const char *signature;
/* attachment */
void *attachment;
#if WASM_ENABLE_GC != 0
/* the type index of this function's func_type */
uint32 type_idx;
#endif
bool call_conv_raw;
bool call_conv_wasm_c_api;
#if WASM_ENABLE_MULTI_MODULE != 0
WASMModule *import_module;
WASMFunction *import_func_linked;
#endif
bool call_conv_wasm_c_api;
} WASMFunctionImport;
#if WASM_ENABLE_TAGS != 0
@ -241,9 +566,12 @@ typedef struct WASMGlobalImport {
char *field_name;
uint8 type;
bool is_mutable;
bool is_linked;
/* global data after linked */
WASMValue global_data_linked;
bool is_linked;
#if WASM_ENABLE_GC != 0
WASMRefType *ref_type;
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
/* imported function pointer after linked */
/* TODO: remove if not needed */
@ -278,9 +606,13 @@ struct WASMFunction {
char *field_name;
#endif
/* the type of function */
WASMType *func_type;
WASMFuncType *func_type;
uint32 local_count;
uint8 *local_types;
#if WASM_ENABLE_GC != 0
uint16 local_ref_type_map_count;
WASMRefTypeMap *local_ref_type_maps;
#endif
/* cell num of parameters */
uint16 param_cell_num;
@ -303,6 +635,11 @@ struct WASMFunction {
uint32 const_cell_num;
#endif
#if WASM_ENABLE_GC != 0
/* the type index of this function's func_type */
uint32 type_idx;
#endif
#if WASM_ENABLE_EXCE_HANDLING != 0
uint32 exception_handler_count;
#endif
@ -347,6 +684,9 @@ struct WASMTag {
struct WASMGlobal {
uint8 type;
bool is_mutable;
#if WASM_ENABLE_GC != 0
WASMRefType *ref_type;
#endif
InitializerExpression init_expr;
#if WASM_ENABLE_FAST_JIT != 0
/* The data offset of current global in global data */
@ -365,11 +705,14 @@ typedef struct WASMTableSeg {
uint32 mode;
/* funcref or externref, elemkind will be considered as funcref */
uint32 elem_type;
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
#endif
/* optional, only for active */
uint32 table_index;
InitializerExpression base_offset;
uint32 function_count;
uint32 *func_indexes;
uint32 value_count;
InitializerExpression *init_values;
} WASMTableSeg;
typedef struct WASMDataSeg {
@ -481,6 +824,13 @@ struct WASMModule {
#if WASM_ENABLE_BULK_MEMORY != 0
/* data count read from datacount section */
uint32 data_seg_count1;
#endif
#if WASM_ENABLE_GC != 0
#if WASM_ENABLE_STRINGREF != 0
uint32 string_literal_count;
uint32 *string_literal_lengths;
const uint8 **string_literal_ptrs;
#endif
#endif
uint32 import_function_count;
@ -564,13 +914,31 @@ struct WASMModule {
bh_list import_module_list_head;
bh_list *import_module_list;
#endif
#if WASM_ENABLE_GC != 0
/* Ref types hash set */
HashMap *ref_type_set;
struct WASMRttType **rtt_types;
korp_mutex rtt_type_lock;
#if WASM_ENABLE_STRINGREF != 0
/* special rtts for stringref types
- stringref
- stringview_wtf8
- stringview_wtf16
- stringview_iter
*/
struct WASMRttType *stringref_rtts[4];
#endif
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0
bh_list fast_opcode_list;
uint8 *buf_code;
uint64 buf_code_size;
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_DEBUG_AOT != 0 \
|| WASM_ENABLE_FAST_JIT != 0
#if WASM_ENABLE_DEBUG_INTERP != 0 || WASM_ENABLE_FAST_JIT != 0 \
|| WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_JIT != 0
uint8 *load_addr;
uint64 load_size;
#endif
@ -671,6 +1039,12 @@ struct WASMModule {
functions in that group */
uint32 fast_jit_ready_groups;
#endif
#if WASM_ENABLE_WAMR_COMPILER != 0
bool is_simd_used;
bool is_ref_types_used;
bool is_bulk_memory_used;
#endif
};
typedef struct BlockType {
@ -679,8 +1053,13 @@ typedef struct BlockType {
* by a type index of module.
*/
union {
uint8 value_type;
WASMType *type;
struct {
uint8 type;
#if WASM_ENABLE_GC != 0
WASMRefTypeMap ref_type_map;
#endif
} value_type;
WASMFuncType *type;
} u;
bool is_value_type;
} BlockType;
@ -755,33 +1134,72 @@ wasm_string_equal(const char *s1, const char *s2)
}
/**
* Return the byte size of value type.
* Return the byte size of value type with specific pointer size.
*
* Note: Please use wasm_value_type_size for interpreter, only aot compiler
* can use this API directly to calculate type size for different target
*/
inline static uint32
wasm_value_type_size_internal(uint8 value_type, uint8 pointer_size)
{
if (value_type == VALUE_TYPE_VOID)
return 0;
else if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_F32
|| value_type == VALUE_TYPE_ANY)
return sizeof(int32);
else if (value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F64)
return sizeof(int64);
#if WASM_ENABLE_SIMD != 0
else if (value_type == VALUE_TYPE_V128)
return sizeof(int64) * 2;
#endif
#if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
else if (value_type == VALUE_TYPE_FUNCREF
|| value_type == VALUE_TYPE_EXTERNREF)
return sizeof(uint32);
#elif WASM_ENABLE_GC != 0
else if ((value_type >= (uint8)REF_TYPE_ARRAYREF /* 0x6A */
&& value_type <= (uint8)REF_TYPE_NULLFUNCREF) /* 0x73 */
|| (value_type >= (uint8)REF_TYPE_HT_NULLABLE /* 0x63 */
&& value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE) /* 0x64 */
#if WASM_ENABLE_STRINGREF != 0
|| (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8 /* 0x66 */
&& value_type <= (uint8)REF_TYPE_STRINGREF) /* 0x67 */
|| (value_type >= (uint8)REF_TYPE_STRINGVIEWITER /* 0x61 */
&& value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16) /* 0x62 */
#endif
)
return pointer_size;
else if (value_type == PACKED_TYPE_I8)
return sizeof(int8);
else if (value_type == PACKED_TYPE_I16)
return sizeof(int16);
#endif
else {
bh_assert(0);
}
return 0;
}
/**
* Return the cell num of value type with specific pointer size.
*
* Note: Please use wasm_value_type_cell_num for interpreter, only aot compiler
* can use this API directly to calculate type cell num for different target
*/
inline static uint16
wasm_value_type_cell_num_internal(uint8 value_type, uint8 pointer_size)
{
return wasm_value_type_size_internal(value_type, pointer_size) / 4;
}
/**
* Return the byte size of value type.
*/
inline static uint32
wasm_value_type_size(uint8 value_type)
{
switch (value_type) {
case VALUE_TYPE_I32:
case VALUE_TYPE_F32:
#if WASM_ENABLE_REF_TYPES != 0
case VALUE_TYPE_FUNCREF:
case VALUE_TYPE_EXTERNREF:
#endif
return sizeof(int32);
case VALUE_TYPE_I64:
case VALUE_TYPE_F64:
return sizeof(int64);
#if WASM_ENABLE_SIMD != 0
case VALUE_TYPE_V128:
return sizeof(int64) * 2;
#endif
case VALUE_TYPE_VOID:
return 0;
default:
bh_assert(0);
}
return 0;
return wasm_value_type_size_internal(value_type, sizeof(uintptr_t));
}
inline static uint16
@ -813,70 +1231,152 @@ wasm_value_type_cell_num_outside(uint8 value_type)
}
#endif
#if WASM_ENABLE_GC == 0
inline static bool
wasm_type_equal(const WASMType *type1, const WASMType *type2)
wasm_type_equal(const WASMType *type1, const WASMType *type2,
const WASMTypePtr *types, uint32 type_count)
{
const WASMFuncType *func_type1 = (const WASMFuncType *)type1;
const WASMFuncType *func_type2 = (const WASMFuncType *)type2;
if (type1 == type2) {
return true;
}
return (type1->param_count == type2->param_count
&& type1->result_count == type2->result_count
&& memcmp(type1->types, type2->types,
(uint32)(type1->param_count + type1->result_count))
return (func_type1->param_count == func_type2->param_count
&& func_type1->result_count == func_type2->result_count
&& memcmp(
func_type1->types, func_type2->types,
(uint32)(func_type1->param_count + func_type1->result_count))
== 0)
? true
: false;
(void)types;
(void)type_count;
}
#else
/* implemented in gc_type.c */
bool
wasm_type_equal(const WASMType *type1, const WASMType *type2,
const WASMTypePtr *types, uint32 type_count);
#endif
inline static uint32
wasm_get_smallest_type_idx(WASMType **types, uint32 type_count,
wasm_get_smallest_type_idx(const WASMTypePtr *types, uint32 type_count,
uint32 cur_type_idx)
{
uint32 i;
for (i = 0; i < cur_type_idx; i++) {
if (wasm_type_equal(types[cur_type_idx], types[i]))
if (wasm_type_equal(types[cur_type_idx], types[i], types, type_count))
return i;
}
(void)type_count;
return cur_type_idx;
}
#if WASM_ENABLE_GC == 0
static inline uint32
block_type_get_param_types(BlockType *block_type, uint8 **p_param_types)
#else
static inline uint32
block_type_get_param_types(BlockType *block_type, uint8 **p_param_types,
WASMRefTypeMap **p_param_reftype_maps,
uint32 *p_param_reftype_map_count)
#endif
{
uint32 param_count = 0;
if (!block_type->is_value_type) {
WASMType *wasm_type = block_type->u.type;
*p_param_types = wasm_type->types;
param_count = wasm_type->param_count;
WASMFuncType *func_type = block_type->u.type;
*p_param_types = func_type->types;
param_count = func_type->param_count;
#if WASM_ENABLE_GC != 0
*p_param_reftype_maps = func_type->ref_type_maps;
*p_param_reftype_map_count =
func_type->result_ref_type_maps - func_type->ref_type_maps;
#endif
}
else {
*p_param_types = NULL;
param_count = 0;
#if WASM_ENABLE_GC != 0
*p_param_reftype_maps = NULL;
*p_param_reftype_map_count = 0;
#endif
}
return param_count;
}
#if WASM_ENABLE_GC == 0
static inline uint32
block_type_get_result_types(BlockType *block_type, uint8 **p_result_types)
#else
static inline uint32
block_type_get_result_types(BlockType *block_type, uint8 **p_result_types,
WASMRefTypeMap **p_result_reftype_maps,
uint32 *p_result_reftype_map_count)
#endif
{
uint32 result_count = 0;
uint8 *result_types = NULL;
#if WASM_ENABLE_GC != 0
uint8 type;
uint32 result_reftype_map_count = 0;
WASMRefTypeMap *result_reftype_maps = NULL;
#endif
if (block_type->is_value_type) {
if (block_type->u.value_type != VALUE_TYPE_VOID) {
*p_result_types = &block_type->u.value_type;
if (block_type->u.value_type.type != VALUE_TYPE_VOID) {
result_types = &block_type->u.value_type.type;
result_count = 1;
#if WASM_ENABLE_GC != 0
type = block_type->u.value_type.type;
if (type == (uint8)REF_TYPE_HT_NULLABLE
|| type == (uint8)REF_TYPE_HT_NON_NULLABLE) {
result_reftype_maps = &block_type->u.value_type.ref_type_map;
result_reftype_map_count = 1;
}
#endif
}
}
else {
WASMType *wasm_type = block_type->u.type;
*p_result_types = wasm_type->types + wasm_type->param_count;
result_count = wasm_type->result_count;
WASMFuncType *func_type = block_type->u.type;
result_types = func_type->types + func_type->param_count;
result_count = func_type->result_count;
#if WASM_ENABLE_GC != 0
result_reftype_maps = func_type->result_ref_type_maps;
result_reftype_map_count = (uint32)(func_type->ref_type_map_count
- (func_type->result_ref_type_maps
- func_type->ref_type_maps));
#endif
}
*p_result_types = result_types;
#if WASM_ENABLE_GC != 0
*p_result_reftype_maps = result_reftype_maps;
*p_result_reftype_map_count = result_reftype_map_count;
#endif
return result_count;
}
static inline uint32
block_type_get_arity(const BlockType *block_type, uint8 label_type)
{
if (label_type == LABEL_TYPE_LOOP) {
if (block_type->is_value_type)
return 0;
else
return block_type->u.type->param_count;
}
else {
if (block_type->is_value_type) {
return block_type->u.value_type.type != VALUE_TYPE_VOID ? 1 : 0;
}
else
return block_type->u.type->result_count;
}
return 0;
}
#ifdef __cplusplus
} /* end of extern "C" */
#endif

22
core/iwasm/interpreter/wasm_interp.h
View File

@ -47,8 +47,11 @@ typedef struct WASMInterpFrame {
the callee will put return values here continuously */
uint32 ret_offset;
uint32 *lp;
#if WASM_ENABLE_GC != 0
uint8 *frame_ref;
#endif
uint32 operand[1];
#else
#else /* else of WASM_ENABLE_FAST_INTERP != 0 */
/* Operand stack top pointer of the current frame. The bottom of
the stack is the next cell after the last local variable. */
uint32 *sp_bottom;
@ -64,10 +67,12 @@ typedef struct WASMInterpFrame {
* lp: parameters and local variables
* sp_bottom to sp_boundary: wasm operand stack
* csp_bottom to csp_boundary: wasm label stack
* frame ref flags: only available for GC
* whether each cell in local and stack area is a GC obj
* jit spill cache: only available for fast jit
*/
uint32 lp[1];
#endif
#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */
} WASMInterpFrame;
/**
@ -84,7 +89,12 @@ wasm_interp_interp_frame_size(unsigned all_cell_num)
unsigned frame_size;
#if WASM_ENABLE_FAST_INTERP == 0
#if WASM_ENABLE_GC == 0
frame_size = (uint32)offsetof(WASMInterpFrame, lp) + all_cell_num * 4;
#else
frame_size =
(uint32)offsetof(WASMInterpFrame, lp) + align_uint(all_cell_num * 5, 4);
#endif
#else
frame_size = (uint32)offsetof(WASMInterpFrame, operand) + all_cell_num * 4;
#endif
@ -97,6 +107,14 @@ wasm_interp_call_wasm(struct WASMModuleInstance *module_inst,
struct WASMFunctionInstance *function, uint32 argc,
uint32 argv[]);
#if WASM_ENABLE_GC != 0
bool
wasm_interp_traverse_gc_rootset(struct WASMExecEnv *exec_env, void *heap);
uint8 *
wasm_interp_get_frame_ref(WASMInterpFrame *frame);
#endif
#ifdef __cplusplus
}
#endif

2368
core/iwasm/interpreter/wasm_interp_classic.c
View File

File diff suppressed because it is too large Load Diff

2214
core/iwasm/interpreter/wasm_interp_fast.c
View File

File diff suppressed because it is too large Load Diff

5828
core/iwasm/interpreter/wasm_loader.c
View File

File diff suppressed because it is too large Load Diff

577
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -194,6 +194,29 @@ loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
return mem;
}
static void *
memory_realloc(void *mem_old, uint32 size_old, uint32 size_new, char *error_buf,
uint32 error_buf_size)
{
uint8 *mem_new;
bh_assert(size_new > size_old);
if ((mem_new = loader_malloc(size_new, error_buf, error_buf_size))) {
bh_memcpy_s(mem_new, size_new, mem_old, size_old);
memset(mem_new + size_old, 0, size_new - size_old);
wasm_runtime_free(mem_old);
}
return mem_new;
}
#define MEM_REALLOC(mem, size_old, size_new) \
do { \
void *mem_new = memory_realloc(mem, size_old, size_new, error_buf, \
error_buf_size); \
if (!mem_new) \
goto fail; \
mem = mem_new; \
} while (0)
static char *
const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module,
bool is_load_from_file_buf, char *error_buf,
@ -252,7 +275,7 @@ const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module,
}
static void
destroy_wasm_type(WASMType *type)
destroy_wasm_type(WASMFuncType *type)
{
if (type->ref_count > 1) {
/* The type is referenced by other types
@ -271,80 +294,256 @@ destroy_wasm_type(WASMType *type)
}
static bool
load_init_expr(const uint8 **p_buf, const uint8 *buf_end,
check_function_index(const WASMModule *module, uint32 function_index,
char *error_buf, uint32 error_buf_size)
{
return (function_index
< module->import_function_count + module->function_count);
}
typedef struct InitValue {
uint8 type;
uint8 flag;
WASMValue value;
} InitValue;
typedef struct ConstExprContext {
uint32 sp;
uint32 size;
WASMModule *module;
InitValue *stack;
InitValue data[WASM_CONST_EXPR_STACK_SIZE];
} ConstExprContext;
static void
init_const_expr_stack(ConstExprContext *ctx, WASMModule *module)
{
ctx->sp = 0;
ctx->module = module;
ctx->stack = ctx->data;
ctx->size = WASM_CONST_EXPR_STACK_SIZE;
}
static bool
push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type,
WASMValue *value, char *error_buf, uint32 error_buf_size)
{
InitValue *cur_value;
if (ctx->sp >= ctx->size) {
if (ctx->stack != ctx->data) {
MEM_REALLOC(ctx->stack, ctx->size * sizeof(InitValue),
(ctx->size + 4) * sizeof(InitValue));
}
else {
if (!(ctx->stack =
loader_malloc((ctx->size + 4) * (uint64)sizeof(InitValue),
error_buf, error_buf_size))) {
return false;
}
}
ctx->size += 4;
}
cur_value = &ctx->stack[ctx->sp++];
cur_value->type = type;
cur_value->flag = flag;
cur_value->value = *value;
return true;
fail:
return false;
}
static bool
pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type,
WASMValue *p_value, char *error_buf, uint32 error_buf_size)
{
InitValue *cur_value;
if (ctx->sp == 0) {
return false;
}
cur_value = &ctx->stack[--ctx->sp];
if (cur_value->type != type) {
return false;
}
if (p_flag)
*p_flag = cur_value->flag;
if (p_value)
*p_value = cur_value->value;
return true;
}
static void
destroy_const_expr_stack(ConstExprContext *ctx)
{
if (ctx->stack != ctx->data) {
wasm_runtime_free(ctx->stack);
}
}
static bool
load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end,
InitializerExpression *init_expr, uint8 type, char *error_buf,
uint32 error_buf_size)
{
const uint8 *p = *p_buf, *p_end = buf_end;
uint8 flag, end_byte, *p_float;
uint8 flag, *p_float;
uint32 i;
ConstExprContext const_expr_ctx = { 0 };
WASMValue cur_value = { 0 };
init_const_expr_stack(&const_expr_ctx, module);
CHECK_BUF(p, p_end, 1);
init_expr->init_expr_type = read_uint8(p);
flag = init_expr->init_expr_type;
flag = read_uint8(p);
while (flag != WASM_OP_END) {
switch (flag) {
/* i32.const */
case INIT_EXPR_TYPE_I32_CONST:
read_leb_int32(p, p_end, cur_value.i32);
if (!push_const_expr_stack(&const_expr_ctx, flag,
VALUE_TYPE_I32, &cur_value,
error_buf, error_buf_size)) {
bh_assert(0);
}
break;
/* i64.const */
case INIT_EXPR_TYPE_I64_CONST:
read_leb_int64(p, p_end, cur_value.i64);
if (!push_const_expr_stack(&const_expr_ctx, flag,
VALUE_TYPE_I64, &cur_value,
error_buf, error_buf_size)) {
bh_assert(0);
}
break;
/* f32.const */
case INIT_EXPR_TYPE_F32_CONST:
CHECK_BUF(p, p_end, 4);
p_float = (uint8 *)&cur_value.f32;
for (i = 0; i < sizeof(float32); i++)
*p_float++ = *p++;
if (!push_const_expr_stack(&const_expr_ctx, flag,
VALUE_TYPE_F32, &cur_value,
error_buf, error_buf_size)) {
bh_assert(0);
}
break;
/* f64.const */
case INIT_EXPR_TYPE_F64_CONST:
CHECK_BUF(p, p_end, 8);
p_float = (uint8 *)&cur_value.f64;
for (i = 0; i < sizeof(float64); i++)
*p_float++ = *p++;
if (!push_const_expr_stack(&const_expr_ctx, flag,
VALUE_TYPE_F64, &cur_value,
error_buf, error_buf_size)) {
bh_assert(0);
}
break;
switch (flag) {
/* i32.const */
case INIT_EXPR_TYPE_I32_CONST:
bh_assert(type == VALUE_TYPE_I32);
read_leb_int32(p, p_end, init_expr->u.i32);
break;
/* i64.const */
case INIT_EXPR_TYPE_I64_CONST:
bh_assert(type == VALUE_TYPE_I64);
read_leb_int64(p, p_end, init_expr->u.i64);
break;
/* f32.const */
case INIT_EXPR_TYPE_F32_CONST:
bh_assert(type == VALUE_TYPE_F32);
CHECK_BUF(p, p_end, 4);
p_float = (uint8 *)&init_expr->u.f32;
for (i = 0; i < sizeof(float32); i++)
*p_float++ = *p++;
break;
/* f64.const */
case INIT_EXPR_TYPE_F64_CONST:
bh_assert(type == VALUE_TYPE_F64);
CHECK_BUF(p, p_end, 8);
p_float = (uint8 *)&init_expr->u.f64;
for (i = 0; i < sizeof(float64); i++)
*p_float++ = *p++;
break;
#if WASM_ENABLE_REF_TYPES != 0
case INIT_EXPR_TYPE_FUNCREF_CONST:
{
bh_assert(type == VALUE_TYPE_FUNCREF);
read_leb_uint32(p, p_end, init_expr->u.ref_index);
break;
/* ref.func */
case INIT_EXPR_TYPE_FUNCREF_CONST:
{
uint32 func_idx;
read_leb_uint32(p, p_end, func_idx);
cur_value.ref_index = func_idx;
if (!check_function_index(module, func_idx, error_buf,
error_buf_size)) {
bh_assert(0);
}
if (!push_const_expr_stack(&const_expr_ctx, flag,
VALUE_TYPE_FUNCREF, &cur_value,
error_buf, error_buf_size)) {
bh_assert(0);
}
break;
}
/* ref.null */
case INIT_EXPR_TYPE_REFNULL_CONST:
{
uint8 type1;
CHECK_BUF(p, p_end, 1);
type1 = read_uint8(p);
cur_value.ref_index = UINT32_MAX;
if (!push_const_expr_stack(&const_expr_ctx, flag, type1,
&cur_value, error_buf,
error_buf_size)) {
bh_assert(0);
}
break;
}
#endif /* end of WASM_ENABLE_REF_TYPES != 0 */
/* get_global */
case INIT_EXPR_TYPE_GET_GLOBAL:
{
uint32 global_idx;
uint8 global_type;
read_leb_uint32(p, p_end, cur_value.global_index);
global_idx = cur_value.global_index;
bh_assert(global_idx < module->import_global_count);
bh_assert(
!module->import_globals[global_idx].u.global.is_mutable);
if (global_idx < module->import_global_count) {
global_type =
module->import_globals[global_idx].u.global.type;
}
else {
global_type =
module
->globals[global_idx - module->import_global_count]
.type;
}
if (!push_const_expr_stack(&const_expr_ctx, flag, global_type,
&cur_value, error_buf,
error_buf_size))
bh_assert(0);
break;
}
default:
{
bh_assert(0);
}
}
case INIT_EXPR_TYPE_REFNULL_CONST:
{
uint8 reftype;
CHECK_BUF(p, p_end, 1);
reftype = read_uint8(p);
bh_assert(type == reftype);
init_expr->u.ref_index = NULL_REF;
(void)reftype;
break;
}
#endif /* WASM_ENABLE_REF_TYPES != 0 */
/* get_global */
case INIT_EXPR_TYPE_GET_GLOBAL:
read_leb_uint32(p, p_end, init_expr->u.global_index);
break;
default:
bh_assert(0);
break;
CHECK_BUF(p, p_end, 1);
flag = read_uint8(p);
}
CHECK_BUF(p, p_end, 1);
end_byte = read_uint8(p);
bh_assert(end_byte == 0x0b);
*p_buf = p;
(void)end_byte;
/* There should be only one value left on the init value stack */
if (!pop_const_expr_stack(&const_expr_ctx, &flag, type, &cur_value,
error_buf, error_buf_size)) {
bh_assert(0);
}
bh_assert(const_expr_ctx.sp == 0);
init_expr->init_expr_type = flag;
init_expr->u = cur_value;
*p_buf = p;
destroy_const_expr_stack(&const_expr_ctx);
return true;
}
@ -357,13 +556,13 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
uint32 param_cell_num, ret_cell_num;
uint64 total_size;
uint8 flag;
WASMType *type;
WASMFuncType *type;
read_leb_uint32(p, p_end, type_count);
if (type_count) {
module->type_count = type_count;
total_size = sizeof(WASMType *) * (uint64)type_count;
total_size = sizeof(WASMFuncType *) * (uint64)type_count;
if (!(module->types =
loader_malloc(total_size, error_buf, error_buf_size))) {
return false;
@ -386,7 +585,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
bh_assert(param_count <= UINT16_MAX && result_count <= UINT16_MAX);
total_size = offsetof(WASMType, types)
total_size = offsetof(WASMFuncType, types)
+ sizeof(uint8) * (uint64)(param_count + result_count);
if (!(type = module->types[i] =
loader_malloc(total_size, error_buf, error_buf_size))) {
@ -424,7 +623,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
/* If there is already a same type created, use it instead */
for (j = 0; j < i; ++j) {
if (wasm_type_equal(type, module->types[j])) {
if (wasm_type_equal(type, module->types[j], module->types, i)) {
bh_assert(module->types[j]->ref_count != UINT16_MAX);
destroy_wasm_type(type);
module->types[i] = module->types[j];
@ -444,8 +643,9 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
static void
adjust_table_max_size(uint32 init_size, uint32 max_size_flag, uint32 *max_size)
{
uint32 default_max_size =
init_size * 2 > TABLE_MAX_SIZE ? init_size * 2 : TABLE_MAX_SIZE;
uint32 default_max_size = init_size * 2 > WASM_TABLE_MAX_SIZE
? init_size * 2
: WASM_TABLE_MAX_SIZE;
if (max_size_flag) {
/* module defines the table limitation */
@ -471,7 +671,7 @@ load_function_import(const uint8 **p_buf, const uint8 *buf_end,
{
const uint8 *p = *p_buf, *p_end = buf_end;
uint32 declare_type_index = 0;
WASMType *declare_func_type = NULL;
WASMFuncType *declare_func_type = NULL;
WASMFunction *linked_func = NULL;
const char *linked_signature = NULL;
void *linked_attachment = NULL;
@ -911,7 +1111,7 @@ static bool
init_function_local_offsets(WASMFunction *func, char *error_buf,
uint32 error_buf_size)
{
WASMType *param_type = func->func_type;
WASMFuncType *param_type = func->func_type;
uint32 param_count = param_type->param_count;
uint8 *param_types = param_type->types;
uint32 local_count = func->local_count;
@ -1065,14 +1265,6 @@ load_function_section(const uint8 *buf, const uint8 *buf_end,
return true;
}
static bool
check_function_index(const WASMModule *module, uint32 function_index,
char *error_buf, uint32 error_buf_size)
{
return (function_index
< module->import_function_count + module->function_count);
}
static bool
load_table_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
char *error_buf, uint32 error_buf_size)
@ -1151,8 +1343,8 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
read_leb_uint32(p, p_end, global_count);
module->global_count = 0;
if (global_count) {
module->global_count = global_count;
total_size = sizeof(WASMGlobal) * (uint64)global_count;
if (!(module->globals =
loader_malloc(total_size, error_buf, error_buf_size))) {
@ -1169,8 +1361,8 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
global->is_mutable = mutable ? true : false;
/* initialize expression */
if (!load_init_expr(&p, p_end, &(global->init_expr), global->type,
error_buf, error_buf_size))
if (!load_init_expr(module, &p, p_end, &(global->init_expr),
global->type, error_buf, error_buf_size))
return false;
if (INIT_EXPR_TYPE_GET_GLOBAL == global->init_expr.init_expr_type) {
@ -1190,7 +1382,10 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
< module->import_function_count
+ module->function_count);
}
module->global_count++;
}
bh_assert(module->global_count == global_count);
}
bh_assert(p == p_end);
@ -1341,52 +1536,78 @@ load_elem_type(const uint8 **p_buf, const uint8 *buf_end, uint32 *p_elem_type,
static bool
load_func_index_vec(const uint8 **p_buf, const uint8 *buf_end,
WASMModule *module, WASMTableSeg *table_segment,
bool use_init_expr, char *error_buf, uint32 error_buf_size)
char *error_buf, uint32 error_buf_size)
{
const uint8 *p = *p_buf, *p_end = buf_end;
uint32 function_count, function_index = 0, i;
uint64 total_size;
read_leb_uint32(p, p_end, function_count);
table_segment->function_count = function_count;
total_size = sizeof(uint32) * (uint64)function_count;
table_segment->value_count = function_count;
total_size = sizeof(InitializerExpression) * (uint64)function_count;
if (total_size > 0
&& !(table_segment->func_indexes = (uint32 *)loader_malloc(
total_size, error_buf, error_buf_size))) {
&& !(table_segment->init_values =
(InitializerExpression *)loader_malloc(total_size, error_buf,
error_buf_size))) {
return false;
}
for (i = 0; i < function_count; i++) {
InitializerExpression init_expr = { 0 };
InitializerExpression *init_expr = &table_segment->init_values[i];
#if WASM_ENABLE_REF_TYPES != 0
if (!use_init_expr) {
read_leb_uint32(p, p_end, function_index);
}
else {
if (!load_init_expr(&p, p_end, &init_expr, table_segment->elem_type,
error_buf, error_buf_size))
return false;
function_index = init_expr.u.ref_index;
}
#else
read_leb_uint32(p, p_end, function_index);
#endif
/* since we are using -1 to indicate ref.null */
if (init_expr.init_expr_type != INIT_EXPR_TYPE_REFNULL_CONST
&& !check_function_index(module, function_index, error_buf,
error_buf_size)) {
if (!check_function_index(module, function_index, error_buf,
error_buf_size)) {
return false;
}
table_segment->func_indexes[i] = function_index;
init_expr->init_expr_type = INIT_EXPR_TYPE_FUNCREF_CONST;
init_expr->u.ref_index = function_index;
}
*p_buf = p;
return true;
}
#if WASM_ENABLE_REF_TYPES != 0
static bool
load_init_expr_vec(const uint8 **p_buf, const uint8 *buf_end,
WASMModule *module, WASMTableSeg *table_segment,
char *error_buf, uint32 error_buf_size)
{
const uint8 *p = *p_buf, *p_end = buf_end;
uint32 ref_count, i;
uint64 total_size;
read_leb_uint32(p, p_end, ref_count);
table_segment->value_count = ref_count;
total_size = sizeof(InitializerExpression) * (uint64)ref_count;
if (total_size > 0
&& !(table_segment->init_values =
(InitializerExpression *)loader_malloc(total_size, error_buf,
error_buf_size))) {
return false;
}
for (i = 0; i < ref_count; i++) {
InitializerExpression *init_expr = &table_segment->init_values[i];
if (!load_init_expr(module, &p, p_end, init_expr,
table_segment->elem_type, error_buf,
error_buf_size))
return false;
bh_assert(
(init_expr->init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL)
|| (init_expr->init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST)
|| (init_expr->init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST));
}
*p_buf = p;
return true;
}
#endif /* end of WASM_ENABLE_REF_TYPES != 0 */
static bool
load_table_segment_section(const uint8 *buf, const uint8 *buf_end,
WASMModule *module, char *error_buf,
@ -1426,16 +1647,25 @@ load_table_segment_section(const uint8 *buf, const uint8 *buf_end,
error_buf, error_buf_size))
return false;
if (!load_init_expr(&p, p_end, &table_segment->base_offset,
VALUE_TYPE_I32, error_buf,
error_buf_size))
if (!load_init_expr(
module, &p, p_end, &table_segment->base_offset,
VALUE_TYPE_I32, error_buf, error_buf_size))
return false;
if (!load_func_index_vec(&p, p_end, module, table_segment,
table_segment->mode == 0 ? false
: true,
error_buf, error_buf_size))
return false;
if (table_segment->mode == 0) {
/* vec(funcidx) */
if (!load_func_index_vec(&p, p_end, module,
table_segment, error_buf,
error_buf_size))
return false;
}
else {
/* vec(expr) */
if (!load_init_expr_vec(&p, p_end, module,
table_segment, error_buf,
error_buf_size))
return false;
}
break;
/* elemkind + passive/declarative */
case 1:
@ -1443,8 +1673,9 @@ load_table_segment_section(const uint8 *buf, const uint8 *buf_end,
if (!load_elem_type(&p, p_end, &table_segment->elem_type,
true, error_buf, error_buf_size))
return false;
/* vec(funcidx) */
if (!load_func_index_vec(&p, p_end, module, table_segment,
false, error_buf, error_buf_size))
error_buf, error_buf_size))
return false;
break;
/* elemkind/elemtype + table_idx + active */
@ -1454,27 +1685,37 @@ load_table_segment_section(const uint8 *buf, const uint8 *buf_end,
&table_segment->table_index,
error_buf, error_buf_size))
return false;
if (!load_init_expr(&p, p_end, &table_segment->base_offset,
VALUE_TYPE_I32, error_buf,
error_buf_size))
if (!load_init_expr(
module, &p, p_end, &table_segment->base_offset,
VALUE_TYPE_I32, error_buf, error_buf_size))
return false;
if (!load_elem_type(&p, p_end, &table_segment->elem_type,
table_segment->mode == 2 ? true : false,
error_buf, error_buf_size))
return false;
if (!load_func_index_vec(&p, p_end, module, table_segment,
table_segment->mode == 2 ? false
: true,
error_buf, error_buf_size))
return false;
if (table_segment->mode == 2) {
/* vec(funcidx) */
if (!load_func_index_vec(&p, p_end, module,
table_segment, error_buf,
error_buf_size))
return false;
}
else {
/* vec(expr) */
if (!load_init_expr_vec(&p, p_end, module,
table_segment, error_buf,
error_buf_size))
return false;
}
break;
case 5:
case 7:
if (!load_elem_type(&p, p_end, &table_segment->elem_type,
false, error_buf, error_buf_size))
return false;
if (!load_func_index_vec(&p, p_end, module, table_segment,
true, error_buf, error_buf_size))
/* vec(expr) */
if (!load_init_expr_vec(&p, p_end, module, table_segment,
error_buf, error_buf_size))
return false;
break;
default:
@ -1489,10 +1730,10 @@ load_table_segment_section(const uint8 *buf, const uint8 *buf_end,
&table_segment->table_index, error_buf,
error_buf_size))
return false;
if (!load_init_expr(&p, p_end, &table_segment->base_offset,
if (!load_init_expr(module, &p, p_end, &table_segment->base_offset,
VALUE_TYPE_I32, error_buf, error_buf_size))
return false;
if (!load_func_index_vec(&p, p_end, module, table_segment, false,
if (!load_func_index_vec(&p, p_end, module, table_segment,
error_buf, error_buf_size))
return false;
#endif /* WASM_ENABLE_REF_TYPES != 0 */
@ -1569,8 +1810,8 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end,
#if WASM_ENABLE_BULK_MEMORY != 0
if (!is_passive)
#endif
if (!load_init_expr(&p, p_end, &init_expr, VALUE_TYPE_I32,
error_buf, error_buf_size))
if (!load_init_expr(module, &p, p_end, &init_expr,
VALUE_TYPE_I32, error_buf, error_buf_size))
return false;
read_leb_uint32(p, p_end, data_seg_len);
@ -1650,7 +1891,7 @@ load_start_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
char *error_buf, uint32 error_buf_size)
{
const uint8 *p = buf, *p_end = buf_end;
WASMType *type;
WASMFuncType *type;
uint32 start_function;
read_leb_uint32(p, p_end, start_function);
@ -1843,6 +2084,7 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
AOTCompOption option = { 0 };
char *aot_last_error;
uint64 size;
bool gc_enabled = false; /* GC hasn't been enabled in mini loader */
if (module->function_count == 0)
return true;
@ -1869,7 +2111,7 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
(bool *)((uint8 *)module->func_ptrs
+ sizeof(void *) * module->function_count);
module->comp_data = aot_create_comp_data(module);
module->comp_data = aot_create_comp_data(module, NULL, gc_enabled);
if (!module->comp_data) {
aot_last_error = aot_get_last_error();
bh_assert(aot_last_error != NULL);
@ -1900,10 +2142,15 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
option.enable_ref_types = true;
#endif
option.enable_aux_stack_check = true;
#if (WASM_ENABLE_PERF_PROFILING != 0) || (WASM_ENABLE_DUMP_CALL_STACK != 0)
#if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \
|| WASM_ENABLE_AOT_STACK_FRAME != 0
option.enable_aux_stack_frame = true;
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
option.enable_perf_profiling = true;
#endif
#if WASM_ENABLE_MEMORY_PROFILING != 0
option.enable_memory_profiling = true;
option.enable_stack_estimation = true;
#endif
@ -2299,7 +2546,7 @@ load_from_sections(WASMModule *module, WASMSection *sections,
uint32 aux_stack_top = (uint32)-1, global_index, func_index, i;
uint32 aux_data_end_global_index = (uint32)-1;
uint32 aux_heap_base_global_index = (uint32)-1;
WASMType *func_type;
WASMFuncType *func_type;
/* Find code and function sections if have */
while (section) {
@ -2936,7 +3183,8 @@ wasm_loader_load(uint8 *buf, uint32 size, char *error_buf,
return NULL;
}
#if WASM_ENABLE_FAST_JIT != 0
#if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \
|| WASM_ENABLE_JIT != 0
module->load_addr = (uint8 *)buf;
module->load_size = size;
#endif
@ -3045,8 +3293,8 @@ wasm_loader_unload(WASMModule *module)
if (module->table_segments) {
for (i = 0; i < module->table_seg_count; i++) {
if (module->table_segments[i].func_indexes)
wasm_runtime_free(module->table_segments[i].func_indexes);
if (module->table_segments[i].init_values)
wasm_runtime_free(module->table_segments[i].init_values);
}
wasm_runtime_free(module->table_segments);
}
@ -3689,29 +3937,6 @@ typedef struct Const {
uint8 value_type;
} Const;
static void *
memory_realloc(void *mem_old, uint32 size_old, uint32 size_new, char *error_buf,
uint32 error_buf_size)
{
uint8 *mem_new;
bh_assert(size_new > size_old);
if ((mem_new = loader_malloc(size_new, error_buf, error_buf_size))) {
bh_memcpy_s(mem_new, size_new, mem_old, size_old);
memset(mem_new + size_old, 0, size_new - size_old);
wasm_runtime_free(mem_old);
}
return mem_new;
}
#define MEM_REALLOC(mem, size_old, size_new) \
do { \
void *mem_new = memory_realloc(mem, size_old, size_new, error_buf, \
error_buf_size); \
if (!mem_new) \
goto fail; \
mem = mem_new; \
} while (0)
#define CHECK_CSP_PUSH() \
do { \
if (ctx->frame_csp >= ctx->frame_csp_boundary) { \
@ -5393,7 +5618,7 @@ copy_params_to_dynamic_space(WASMLoaderContext *loader_ctx, bool is_if_block,
uint32 i;
BranchBlock *block = loader_ctx->frame_csp - 1;
BlockType *block_type = &block->block_type;
WASMType *wasm_type = block_type->u.type;
WASMFuncType *wasm_type = block_type->u.type;
uint32 param_count = block_type->u.type->param_count;
int16 condition_offset = 0;
bool disable_emit = false;
@ -5637,7 +5862,7 @@ re_scan:
* 0x40/0x7F/0x7E/0x7D/0x7C, take it as the type of
* the single return value. */
block_type.is_value_type = true;
block_type.u.value_type = value_type;
block_type.u.value_type.type = value_type;
}
else {
uint32 type_index;
@ -5658,7 +5883,7 @@ re_scan:
/* Pop block parameters from stack */
if (BLOCK_HAS_PARAM(block_type)) {
WASMType *wasm_type = block_type.u.type;
WASMFuncType *wasm_type = block_type.u.type;
BranchBlock *cur_block = loader_ctx->frame_csp - 1;
#if WASM_ENABLE_FAST_INTERP != 0
@ -5861,19 +6086,11 @@ re_scan:
uint32 block_param_count = 0, block_ret_count = 0;
uint8 *block_param_types = NULL, *block_ret_types = NULL;
BlockType *cur_block_type = &cur_block->block_type;
if (cur_block_type->is_value_type) {
if (cur_block_type->u.value_type != VALUE_TYPE_VOID) {
block_ret_count = 1;
block_ret_types = &cur_block_type->u.value_type;
}
}
else {
block_param_count = cur_block_type->u.type->param_count;
block_ret_count = cur_block_type->u.type->result_count;
block_param_types = cur_block_type->u.type->types;
block_ret_types =
cur_block_type->u.type->types + block_param_count;
}
block_param_count = block_type_get_param_types(
cur_block_type, &block_param_types);
block_ret_count = block_type_get_result_types(
cur_block_type, &block_ret_types);
bh_assert(block_param_count == block_ret_count
&& (!block_param_count
|| !memcmp(block_param_types, block_ret_types,
@ -5881,6 +6098,7 @@ re_scan:
(void)block_ret_types;
(void)block_ret_count;
(void)block_param_types;
(void)block_param_count;
}
POP_CSP();
@ -6043,7 +6261,7 @@ re_scan:
case WASM_OP_RETURN_CALL:
#endif
{
WASMType *func_type;
WASMFuncType *func_type;
uint32 func_idx;
int32 idx;
@ -6115,7 +6333,7 @@ re_scan:
#endif
{
int32 idx;
WASMType *func_type;
WASMFuncType *func_type;
uint32 type_idx, table_idx;
bh_assert(module->import_table_count + module->table_count > 0);
@ -6531,8 +6749,9 @@ re_scan:
is passive, active or declarative. */
for (i = 0; i < module->table_seg_count; i++, table_seg++) {
if (table_seg->elem_type == VALUE_TYPE_FUNCREF) {
for (j = 0; j < table_seg->function_count; j++) {
if (table_seg->func_indexes[j] == func_idx) {
for (j = 0; j < table_seg->value_count; j++) {
if (table_seg->init_values[j].u.ref_index
== func_idx) {
func_declared = true;
break;
}

140
core/iwasm/interpreter/wasm_opcode.h
View File

@ -35,9 +35,9 @@ typedef enum WASMOpcode {
WASM_OP_CALL_INDIRECT = 0x11, /* call_indirect */
WASM_OP_RETURN_CALL = 0x12, /* return_call */
WASM_OP_RETURN_CALL_INDIRECT = 0x13, /* return_call_indirect */
WASM_OP_CALL_REF = 0x14, /* call_ref */
WASM_OP_RETURN_CALL_REF = 0x15, /* return_call_ref */
WASM_OP_UNUSED_0x14 = 0x14,
WASM_OP_UNUSED_0x15 = 0x15,
WASM_OP_UNUSED_0x16 = 0x16,
WASM_OP_UNUSED_0x17 = 0x17,
@ -259,27 +259,124 @@ typedef enum WASMOpcode {
WASM_OP_IMPDEP = 0xcf,
WASM_OP_REF_NULL = 0xd0, /* ref.null */
WASM_OP_REF_IS_NULL = 0xd1, /* ref.is_null */
WASM_OP_REF_FUNC = 0xd2, /* ref.func */
WASM_OP_REF_NULL = 0xd0, /* ref.null */
WASM_OP_REF_IS_NULL = 0xd1, /* ref.is_null */
WASM_OP_REF_FUNC = 0xd2, /* ref.func */
WASM_OP_REF_EQ = 0xd3, /* ref.eq */
WASM_OP_REF_AS_NON_NULL = 0xd4, /* ref.as_non_null */
WASM_OP_BR_ON_NULL = 0xd5, /* br_on_null */
WASM_OP_BR_ON_NON_NULL = 0xd6, /* br_on_non_null */
EXT_OP_BLOCK = 0xd3, /* block with blocktype */
EXT_OP_LOOP = 0xd4, /* loop with blocktype */
EXT_OP_IF = 0xd5, /* if with blocktype */
EXT_OP_BR_TABLE_CACHE = 0xd6, /* br_table from cache */
EXT_OP_BLOCK = 0xd7, /* block with blocktype */
EXT_OP_LOOP = 0xd8, /* loop with blocktype */
EXT_OP_IF = 0xd9, /* if with blocktype */
EXT_OP_BR_TABLE_CACHE = 0xda, /* br_table from cache */
EXT_OP_TRY = 0xd7, /* try block with blocktype */
EXT_OP_TRY = 0xdb, /* try block with blocktype */
#if WASM_ENABLE_DEBUG_INTERP != 0
DEBUG_OP_BREAK = 0xd8, /* debug break point */
DEBUG_OP_BREAK = 0xdc, /* debug break point */
#endif
/* Post-MVP extend op prefix */
WASM_OP_GC_PREFIX = 0xfb,
WASM_OP_MISC_PREFIX = 0xfc,
WASM_OP_SIMD_PREFIX = 0xfd,
WASM_OP_ATOMIC_PREFIX = 0xfe,
} WASMOpcode;
typedef enum WASMGCEXTOpcode {
WASM_OP_STRUCT_NEW = 0x00, /* struct.new */
WASM_OP_STRUCT_NEW_DEFAULT = 0x01, /* struct.new_default */
WASM_OP_STRUCT_GET = 0x02, /* struct.get */
WASM_OP_STRUCT_GET_S = 0x03, /* struct.get_s */
WASM_OP_STRUCT_GET_U = 0x04, /* struct.get_u */
WASM_OP_STRUCT_SET = 0x05, /* struct.set */
WASM_OP_ARRAY_NEW = 0x06, /* array.new */
WASM_OP_ARRAY_NEW_DEFAULT = 0x07, /* array.new_default */
WASM_OP_ARRAY_NEW_FIXED = 0x08, /* array.new_fixed */
WASM_OP_ARRAY_NEW_DATA = 0x09, /* array.new_data */
WASM_OP_ARRAY_NEW_ELEM = 0x0A, /* array.new_elem */
WASM_OP_ARRAY_GET = 0x0B, /* array.get */
WASM_OP_ARRAY_GET_S = 0x0C, /* array.get_s */
WASM_OP_ARRAY_GET_U = 0x0D, /* array.get_u */
WASM_OP_ARRAY_SET = 0x0E, /* array.set */
WASM_OP_ARRAY_LEN = 0x0F, /* array.len */
WASM_OP_ARRAY_FILL = 0x10, /* array.fill */
WASM_OP_ARRAY_COPY = 0x11, /* array.copy */
WASM_OP_ARRAY_INIT_DATA = 0x12,
/* array.init_data */ /* TODO */
WASM_OP_ARRAY_INIT_ELEM = 0x13,
/* array.init_elem */ /* TODO */
WASM_OP_REF_TEST = 0x14, /* ref.test */
WASM_OP_REF_TEST_NULLABLE = 0x15, /* ref.test_nullable */
WASM_OP_REF_CAST = 0x16, /* ref.cast */
WASM_OP_REF_CAST_NULLABLE = 0x17, /* ref.cast_nullable */
WASM_OP_BR_ON_CAST = 0x18, /* br_on_cast */
WASM_OP_BR_ON_CAST_FAIL = 0x19, /* br_on_cast_fail */
WASM_OP_ANY_CONVERT_EXTERN = 0x1A, /* any.convert_extern */
WASM_OP_EXTERN_CONVERT_ANY = 0x1B, /* extern.covert_any */
WASM_OP_REF_I31 = 0x1C, /* ref.i31 */
WASM_OP_I31_GET_S = 0x1D, /* i31.get_s */
WASM_OP_I31_GET_U = 0x1E, /* i31.get_u */
/* stringref related opcoded */
WASM_OP_STRING_NEW_UTF8 = 0x80, /* string.new_utf8 */
WASM_OP_STRING_NEW_WTF16 = 0x81, /* string.new_wtf16 */
WASM_OP_STRING_CONST = 0x82, /* string.const */
WASM_OP_STRING_MEASURE_UTF8 = 0x83, /* string.measure_utf8 */
WASM_OP_STRING_MEASURE_WTF8 = 0x84, /* string.measure_wtf8 */
WASM_OP_STRING_MEASURE_WTF16 = 0x85, /* string.measure_wtf16 */
WASM_OP_STRING_ENCODE_UTF8 = 0x86, /* string.encode_utf8 */
WASM_OP_STRING_ENCODE_WTF16 = 0x87, /* string.encode_wtf16 */
WASM_OP_STRING_CONCAT = 0x88, /* string.concat */
WASM_OP_STRING_EQ = 0x89, /* string.eq */
WASM_OP_STRING_IS_USV_SEQUENCE = 0x8a, /* string.is_usv_sequence */
WASM_OP_STRING_NEW_LOSSY_UTF8 = 0x8b, /* string.new_lossy_utf8 */
WASM_OP_STRING_NEW_WTF8 = 0x8c, /* string.new_wtf8 */
WASM_OP_STRING_ENCODE_LOSSY_UTF8 = 0x8d, /* string.encode_lossy_utf8 */
WASM_OP_STRING_ENCODE_WTF8 = 0x8e, /* string.encode_wtf8 */
WASM_OP_STRING_AS_WTF8 = 0x90, /* string.as_wtf8 */
WASM_OP_STRINGVIEW_WTF8_ADVANCE = 0x91, /* stringview_wtf8.advance */
WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8 =
0x92, /* stringview_wtf8.encode_utf8 */
WASM_OP_STRINGVIEW_WTF8_SLICE = 0x93, /* stringview_wtf8.slice */
WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8 =
0x94, /* stringview_wtf8.encode_lossy_utf8 */
WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8 =
0x95, /* stringview_wtf8.encode_wtf8 */
WASM_OP_STRING_AS_WTF16 = 0x98, /* string.as_wtf16 */
WASM_OP_STRINGVIEW_WTF16_LENGTH = 0x99, /* stringview_wtf16.length */
WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT =
0x9a, /* stringview_wtf16.get_codeunit */
WASM_OP_STRINGVIEW_WTF16_ENCODE = 0x9b, /* stringview_wtf16.encode */
WASM_OP_STRINGVIEW_WTF16_SLICE = 0x9c, /* stringview_wtf16.slice */
WASM_OP_STRING_AS_ITER = 0xa0, /* string.as_iter */
WASM_OP_STRINGVIEW_ITER_NEXT = 0xa1, /* stringview_iter.next */
WASM_OP_STRINGVIEW_ITER_ADVANCE = 0xa2, /* stringview_iter.advance */
WASM_OP_STRINGVIEW_ITER_REWIND = 0xa3, /* stringview_iter.rewind */
WASM_OP_STRINGVIEW_ITER_SLICE = 0xa4, /* stringview_iter.slice */
WASM_OP_STRING_NEW_UTF8_ARRAY = 0xb0, /* string.new_utf8_array */
WASM_OP_STRING_NEW_WTF16_ARRAY = 0xb1, /* string.new_wtf16_array */
WASM_OP_STRING_ENCODE_UTF8_ARRAY = 0xb2, /* string.encode_utf8_array */
WASM_OP_STRING_ENCODE_WTF16_ARRAY = 0xb3, /* string.encode_wtf16_array */
WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY =
0xb4, /* string.new_lossy_utf8_array */
WASM_OP_STRING_NEW_WTF8_ARRAY = 0xb5, /* string.new_wtf8_array */
WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY =
0xb6, /* string.encode_lossy_utf8_array */
WASM_OP_STRING_ENCODE_WTF8_ARRAY = 0xb7, /* string.encode_wtf8_array */
} WASMGCEXTOpcode;
typedef enum WASMMiscEXTOpcode {
WASM_OP_I32_TRUNC_SAT_S_F32 = 0x00,
WASM_OP_I32_TRUNC_SAT_U_F32 = 0x01,
@ -678,7 +775,7 @@ typedef enum WASMAtomicEXTOpcode {
#if WASM_ENABLE_DEBUG_INTERP != 0
#define DEF_DEBUG_BREAK_HANDLE() \
[DEBUG_OP_BREAK] = HANDLE_OPCODE(DEBUG_OP_BREAK), /* 0xd7 */
[DEBUG_OP_BREAK] = HANDLE_OPCODE(DEBUG_OP_BREAK), /* 0xdb */
#else
#define DEF_DEBUG_BREAK_HANDLE()
#endif
@ -719,8 +816,8 @@ typedef enum WASMAtomicEXTOpcode {
HANDLE_OPCODE(WASM_OP_CALL_INDIRECT), /* 0x11 */ \
HANDLE_OPCODE(WASM_OP_RETURN_CALL), /* 0x12 */ \
HANDLE_OPCODE(WASM_OP_RETURN_CALL_INDIRECT), /* 0x13 */ \
HANDLE_OPCODE(WASM_OP_UNUSED_0x14), /* 0x14 */ \
HANDLE_OPCODE(WASM_OP_UNUSED_0x15), /* 0x15 */ \
HANDLE_OPCODE(WASM_OP_CALL_REF), /* 0x14 */ \
HANDLE_OPCODE(WASM_OP_RETURN_CALL_REF), /* 0x15 */ \
HANDLE_OPCODE(WASM_OP_UNUSED_0x16), /* 0x16 */ \
HANDLE_OPCODE(WASM_OP_UNUSED_0x17), /* 0x17 */ \
HANDLE_OPCODE(WASM_OP_DELEGATE), /* 0x18 */ \
@ -910,11 +1007,16 @@ typedef enum WASMAtomicEXTOpcode {
HANDLE_OPCODE(WASM_OP_REF_NULL), /* 0xd0 */ \
HANDLE_OPCODE(WASM_OP_REF_IS_NULL), /* 0xd1 */ \
HANDLE_OPCODE(WASM_OP_REF_FUNC), /* 0xd2 */ \
HANDLE_OPCODE(EXT_OP_BLOCK), /* 0xd3 */ \
HANDLE_OPCODE(EXT_OP_LOOP), /* 0xd4 */ \
HANDLE_OPCODE(EXT_OP_IF), /* 0xd5 */ \
HANDLE_OPCODE(EXT_OP_BR_TABLE_CACHE), /* 0xd6 */ \
HANDLE_OPCODE(EXT_OP_TRY), /* 0xd7 */ \
HANDLE_OPCODE(WASM_OP_REF_EQ), /* 0xd3 */ \
HANDLE_OPCODE(WASM_OP_REF_AS_NON_NULL), /* 0xd4 */ \
HANDLE_OPCODE(WASM_OP_BR_ON_NULL), /* 0xd5 */ \
HANDLE_OPCODE(WASM_OP_BR_ON_NON_NULL), /* 0xd6 */ \
HANDLE_OPCODE(EXT_OP_BLOCK), /* 0xd7 */ \
HANDLE_OPCODE(EXT_OP_LOOP), /* 0xd8 */ \
HANDLE_OPCODE(EXT_OP_IF), /* 0xd9 */ \
HANDLE_OPCODE(EXT_OP_BR_TABLE_CACHE), /* 0xda */ \
HANDLE_OPCODE(EXT_OP_TRY), /* 0xdb */ \
SET_GOTO_TABLE_ELEM(WASM_OP_GC_PREFIX), /* 0xfb */ \
SET_GOTO_TABLE_ELEM(WASM_OP_MISC_PREFIX), /* 0xfc */ \
SET_GOTO_TABLE_SIMD_PREFIX_ELEM() /* 0xfd */ \
SET_GOTO_TABLE_ELEM(WASM_OP_ATOMIC_PREFIX), /* 0xfe */ \

1087
core/iwasm/interpreter/wasm_runtime.c
View File

File diff suppressed because it is too large Load Diff

89
core/iwasm/interpreter/wasm_runtime.h
View File

@ -71,6 +71,22 @@ typedef enum WASMExceptionID {
EXCE_OUT_OF_BOUNDS_TABLE_ACCESS,
EXCE_OPERAND_STACK_OVERFLOW,
EXCE_FAILED_TO_COMPILE_FAST_JIT_FUNC,
/* GC related exceptions */
EXCE_NULL_FUNC_OBJ,
EXCE_NULL_STRUCT_OBJ,
EXCE_NULL_ARRAY_OBJ,
EXCE_NULL_I31_OBJ,
EXCE_NULL_REFERENCE,
EXCE_FAILED_TO_CREATE_RTT_TYPE,
EXCE_FAILED_TO_CREATE_STRUCT_OBJ,
EXCE_FAILED_TO_CREATE_ARRAY_OBJ,
EXCE_FAILED_TO_CREATE_EXTERNREF_OBJ,
EXCE_CAST_FAILURE,
EXCE_ARRAY_IDX_OOB,
EXCE_FAILED_TO_CREATE_STRING,
EXCE_FAILED_TO_CREATE_STRINGREF,
EXCE_FAILED_TO_CREATE_STRINGVIEW,
EXCE_FAILED_TO_ENCODE_STRING,
EXCE_ALREADY_THROWN,
EXCE_NUM,
} WASMExceptionID;
@ -129,13 +145,29 @@ struct WASMMemoryInstance {
#endif
};
/* WASMTableInstance is used to represent table instance in
* runtime, to compute the table element address with index
* we need to know the element type and the element ref type.
* For pointer type, it's 32-bit or 64-bit, align up to 8 bytes
* to simplify the computation.
* And each struct member should be 4-byte or 8-byte aligned.
*/
struct WASMTableInstance {
/* The element type */
uint8 elem_type;
uint8 __padding__[7];
union {
#if WASM_ENABLE_GC != 0
WASMRefType *elem_ref_type;
#endif
uint64 __padding__;
} elem_ref_type;
/* Current size */
uint32 cur_size;
/* Maximum size */
uint32 max_size;
/* Table elements */
uint32 elems[1];
table_elem_type_t elems[1];
};
struct WASMGlobalInstance {
@ -147,6 +179,9 @@ struct WASMGlobalInstance {
uint32 data_offset;
/* initial value */
WASMValue initial_value;
#if WASM_ENABLE_GC != 0
WASMRefType *ref_type;
#endif
#if WASM_ENABLE_MULTI_MODULE != 0
/* just for import, keep the reference here */
WASMModuleInstance *import_module_inst;
@ -271,6 +306,13 @@ typedef struct WASMModuleInstanceExtraCommon {
#if WASM_ENABLE_REF_TYPES != 0
bh_bitmap *elem_dropped;
#endif
#if WASM_ENABLE_GC != 0
/* The gc heap memory pool */
uint8 *gc_heap_pool;
/* The gc heap created */
void *gc_heap_handle;
#endif
} WASMModuleInstanceExtraCommon;
/* Extra info of WASM module instance for interpreter/jit mode */
@ -598,7 +640,7 @@ void
wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module,
WASMModuleInstMemConsumption *mem_conspn);
#if WASM_ENABLE_REF_TYPES != 0
#if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
static inline bool
wasm_elem_is_active(uint32 mode)
{
@ -619,8 +661,18 @@ wasm_elem_is_declarative(uint32 mode)
bool
wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx,
uint32 inc_entries, uint32 init_val);
#endif /* WASM_ENABLE_REF_TYPES != 0 */
uint32 inc_entries, table_elem_type_t init_val);
#endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
#if WASM_ENABLE_GC != 0
void *
wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx,
bool throw_exce, char *error_buf, uint32 error_buf_size);
bool
wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap);
#endif
static inline WASMTableInstance *
wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx)
@ -723,19 +775,42 @@ llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx,
void
llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx,
uint32 length, uint32 val, uint32 data_offset);
uint32 length, uintptr_t val, uint32 data_offset);
uint32
llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx,
uint32 inc_entries, uint32 init_val);
uint32 inc_entries, uintptr_t init_val);
#endif
#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0
#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 \
|| WASM_ENABLE_AOT_STACK_FRAME != 0
bool
llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index);
void
llvm_jit_free_frame(WASMExecEnv *exec_env);
void
llvm_jit_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame);
#endif
#if WASM_ENABLE_GC != 0
void *
llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx,
bool throw_exce, char *error_buf,
uint32 error_buf_size);
bool
llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst,
WASMObjectRef gc_obj, uint32 type_index);
WASMRttTypeRef
llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index);
bool
llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index,
uint32 data_seg_offset, WASMArrayObjectRef array_obj,
uint32 elem_size, uint32 array_len);
#endif
#endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */

4
core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c
View File

@ -1136,6 +1136,10 @@ static WASMNativeGlobalDef native_global_defs[] = {
{ "test", "global-f32", VALUE_TYPE_F32, false, .value.f32 = 0 },
{ "test", "global-mut-i32", VALUE_TYPE_I32, true, .value.i32 = 0 },
{ "test", "global-mut-i64", VALUE_TYPE_I64, true, .value.i64 = 0 },
#if WASM_ENABLE_GC != 0
{ "G", "g", VALUE_TYPE_I32, false, .value.i32 = 4 },
{ "M", "g", REF_TYPE_HT_NON_NULLABLE, false, .value.gc_obj = 0 },
#endif
#endif
{ "global", "NaN", VALUE_TYPE_F64, .value.u64 = 0x7FF8000000000000LL },
{ "global", "Infinity", VALUE_TYPE_F64, .value.u64 = 0x7FF0000000000000LL }

364
core/shared/mem-alloc/ems/ems_alloc.c
View File

@ -5,6 +5,27 @@
#include "ems_gc_internal.h"
#if WASM_ENABLE_GC != 0
#define LOCK_HEAP(heap) \
do { \
if (!heap->is_doing_reclaim) \
/* If the heap is doing reclaim, it must have been locked, \
we should not lock the heap again. */ \
os_mutex_lock(&heap->lock); \
} while (0)
#define UNLOCK_HEAP(heap) \
do { \
if (!heap->is_doing_reclaim) \
/* If the heap is doing reclaim, it must have been locked, \
and will be unlocked after reclaim, we should not \
unlock the heap again. */ \
os_mutex_unlock(&heap->lock); \
} while (0)
#else
#define LOCK_HEAP(heap) os_mutex_lock(&heap->lock)
#define UNLOCK_HEAP(heap) os_mutex_unlock(&heap->lock)
#endif
static inline bool
hmu_is_in_heap(void *hmu, gc_uint8 *heap_base_addr, gc_uint8 *heap_end_addr)
{
@ -332,6 +353,11 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size)
bh_assert(gci_is_heap_valid(heap));
bh_assert(size > 0 && !(size & 7));
#if WASM_ENABLE_GC != 0
/* In doing reclaim, gc must not alloc memory again. */
bh_assert(!heap->is_doing_reclaim);
#endif
base_addr = heap->base_addr;
end_addr = base_addr + heap->current_size;
@ -454,6 +480,34 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size)
return NULL;
}
#if WASM_ENABLE_GC != 0
static int
do_gc_heap(gc_heap_t *heap)
{
int ret = GC_SUCCESS;
#if WASM_ENABLE_GC_PERF_PROFILING != 0
uint64 start = 0, end = 0, time = 0;
start = os_time_get_boot_microsecond();
#endif
if (heap->is_reclaim_enabled) {
UNLOCK_HEAP(heap);
ret = gci_gc_heap(heap);
LOCK_HEAP(heap);
}
#if WASM_ENABLE_GC_PERF_PROFILING != 0
end = os_time_get_boot_microsecond();
time = end - start;
heap->total_gc_time += time;
if (time > heap->max_gc_time) {
heap->max_gc_time = time;
}
heap->total_gc_count += 1;
#endif
return ret;
}
#endif
/**
* Find a proper HMU with given size
*
@ -475,12 +529,29 @@ alloc_hmu_ex(gc_heap_t *heap, gc_size_t size)
bh_assert(gci_is_heap_valid(heap));
bh_assert(size > 0 && !(size & 7));
#if WASM_ENABLE_GC != 0
#if GC_IN_EVERY_ALLOCATION != 0
if (GC_SUCCESS != do_gc_heap(heap))
return NULL;
#else
if (heap->total_free_size < heap->gc_threshold) {
if (GC_SUCCESS != do_gc_heap(heap))
return NULL;
}
else {
hmu_t *ret = NULL;
if ((ret = alloc_hmu(heap, size))) {
return ret;
}
if (GC_SUCCESS != do_gc_heap(heap))
return NULL;
}
#endif
#endif
return alloc_hmu(heap, size);
}
static unsigned long g_total_malloc = 0;
static unsigned long g_total_free = 0;
#if BH_ENABLE_GC_VERIFY == 0
gc_object_t
gc_alloc_vo(void *vheap, gc_size_t size)
@ -509,7 +580,7 @@ gc_alloc_vo_internal(void *vheap, gc_size_t size, const char *file, int line)
}
#endif
os_mutex_lock(&heap->lock);
LOCK_HEAP(heap);
hmu = alloc_hmu_ex(heap, tot_size);
if (!hmu)
@ -520,7 +591,9 @@ gc_alloc_vo_internal(void *vheap, gc_size_t size, const char *file, int line)
the required size, reset it here */
tot_size = hmu_get_size(hmu);
g_total_malloc += tot_size;
#if GC_STAT_DATA != 0
heap->total_size_allocated += tot_size;
#endif
hmu_set_ut(hmu, HMU_VO);
hmu_unfree_vo(hmu);
@ -535,7 +608,7 @@ gc_alloc_vo_internal(void *vheap, gc_size_t size, const char *file, int line)
memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned);
finish:
os_mutex_unlock(&heap->lock);
UNLOCK_HEAP(heap);
return ret;
}
@ -582,7 +655,7 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file,
base_addr = heap->base_addr;
end_addr = base_addr + heap->current_size;
os_mutex_lock(&heap->lock);
LOCK_HEAP(heap);
if (hmu_old) {
hmu_next = (hmu_t *)((char *)hmu_old + tot_size_old);
@ -592,7 +665,7 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file,
if (ut == HMU_FC && tot_size <= tot_size_old + tot_size_next) {
/* current node and next node meets requirement */
if (!unlink_hmu(heap, hmu_next)) {
os_mutex_unlock(&heap->lock);
UNLOCK_HEAP(heap);
return NULL;
}
hmu_set_size(hmu_old, tot_size);
@ -605,12 +678,12 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file,
hmu_next = (hmu_t *)((char *)hmu_old + tot_size);
tot_size_next = tot_size_old + tot_size_next - tot_size;
if (!gci_add_fc(heap, hmu_next, tot_size_next)) {
os_mutex_unlock(&heap->lock);
UNLOCK_HEAP(heap);
return NULL;
}
hmu_mark_pinuse(hmu_next);
}
os_mutex_unlock(&heap->lock);
UNLOCK_HEAP(heap);
return obj_old;
}
}
@ -624,7 +697,10 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file,
/* the total size allocated may be larger than
the required size, reset it here */
tot_size = hmu_get_size(hmu);
g_total_malloc += tot_size;
#if GC_STAT_DATA != 0
heap->total_size_allocated += tot_size;
#endif
hmu_set_ut(hmu, HMU_VO);
hmu_unfree_vo(hmu);
@ -647,7 +723,7 @@ finish:
}
}
os_mutex_unlock(&heap->lock);
UNLOCK_HEAP(heap);
if (ret && obj_old)
gc_free_vo(vheap, obj_old);
@ -655,6 +731,93 @@ finish:
return ret;
}
#if GC_MANUALLY != 0
void
gc_free_wo(void *vheap, void *ptr)
{
gc_heap_t *heap = (gc_heap_t *)vheap;
gc_object_t *obj = (gc_object_t *)ptr;
hmu_t *hmu = obj_to_hmu(obj);
bh_assert(gci_is_heap_valid(heap));
bh_assert(obj);
bh_assert((gc_uint8 *)hmu >= heap->base_addr
&& (gc_uint8 *)hmu < heap->base_addr + heap->current_size);
bh_assert(hmu_get_ut(hmu) == HMU_WO);
hmu_unmark_wo(hmu);
(void)heap;
}
#endif
/* see ems_gc.h for description*/
#if BH_ENABLE_GC_VERIFY == 0
gc_object_t
gc_alloc_wo(void *vheap, gc_size_t size)
#else
gc_object_t
gc_alloc_wo_internal(void *vheap, gc_size_t size, const char *file, int line)
#endif
{
gc_heap_t *heap = (gc_heap_t *)vheap;
hmu_t *hmu = NULL;
gc_object_t ret = (gc_object_t)NULL;
gc_size_t tot_size = 0, tot_size_unaligned;
/* hmu header + prefix + obj + suffix */
tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE;
/* aligned size*/
tot_size = GC_ALIGN_8(tot_size_unaligned);
if (tot_size < size)
/* integer overflow */
return NULL;
#if BH_ENABLE_GC_CORRUPTION_CHECK != 0
if (heap->is_heap_corrupted) {
os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n");
return NULL;
}
#endif
LOCK_HEAP(heap);
hmu = alloc_hmu_ex(heap, tot_size);
if (!hmu)
goto finish;
/* Do we need to memset the memory to 0? */
/* memset((char *)hmu + sizeof(*hmu), 0, tot_size - sizeof(*hmu)); */
bh_assert(hmu_get_size(hmu) >= tot_size);
/* the total size allocated may be larger than
the required size, reset it here */
tot_size = hmu_get_size(hmu);
#if GC_STAT_DATA != 0
heap->total_size_allocated += tot_size;
#endif
hmu_set_ut(hmu, HMU_WO);
#if GC_MANUALLY != 0
hmu_mark_wo(hmu);
#else
hmu_unmark_wo(hmu);
#endif
#if BH_ENABLE_GC_VERIFY != 0
hmu_init_prefix_and_suffix(hmu, tot_size, file, line);
#endif
ret = hmu_to_obj(hmu);
if (tot_size > tot_size_unaligned)
/* clear buffer appended by GC_ALIGN_8() */
memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned);
finish:
UNLOCK_HEAP(heap);
return ret;
}
/**
* Do some checking to see if given pointer is a possible valid heap
* @return GC_TRUE if all checking passed, GC_FALSE otherwise
@ -703,7 +866,7 @@ gc_free_vo_internal(void *vheap, gc_object_t obj, const char *file, int line)
base_addr = heap->base_addr;
end_addr = base_addr + heap->current_size;
os_mutex_lock(&heap->lock);
LOCK_HEAP(heap);
if (hmu_is_in_heap(hmu, base_addr, end_addr)) {
#if BH_ENABLE_GC_VERIFY != 0
@ -719,10 +882,12 @@ gc_free_vo_internal(void *vheap, gc_object_t obj, const char *file, int line)
size = hmu_get_size(hmu);
g_total_free += size;
heap->total_free_size += size;
#if GC_STAT_DATA != 0
heap->total_size_freed += size;
#endif
if (!hmu_get_pinuse(hmu)) {
prev = (hmu_t *)((char *)hmu - *((int *)hmu - 1));
@ -767,7 +932,7 @@ gc_free_vo_internal(void *vheap, gc_object_t obj, const char *file, int line)
}
out:
os_mutex_unlock(&heap->lock);
UNLOCK_HEAP(heap);
return ret;
}
@ -778,8 +943,12 @@ gc_dump_heap_stats(gc_heap_t *heap)
os_printf("total free: %" PRIu32 ", current: %" PRIu32
", highmark: %" PRIu32 "\n",
heap->total_free_size, heap->current_size, heap->highmark_size);
os_printf("g_total_malloc=%lu, g_total_free=%lu, occupied=%lu\n",
g_total_malloc, g_total_free, g_total_malloc - g_total_free);
#if GC_STAT_DATA != 0
os_printf("total size allocated: %" PRIu64 ", total size freed: %" PRIu64
", total occupied: %" PRIu64 "\n",
heap->total_size_allocated, heap->total_size_freed,
heap->total_size_allocated - heap->total_size_freed);
#endif
}
uint32
@ -804,12 +973,12 @@ gci_dump(gc_heap_t *heap)
ut = hmu_get_ut(cur);
size = hmu_get_size(cur);
p = hmu_get_pinuse(cur);
mark = hmu_is_jo_marked(cur);
mark = hmu_is_wo_marked(cur);
if (ut == HMU_VO)
inuse = 'V';
else if (ut == HMU_JO)
inuse = hmu_is_jo_marked(cur) ? 'J' : 'j';
else if (ut == HMU_WO)
inuse = hmu_is_wo_marked(cur) ? 'W' : 'w';
else if (ut == HMU_FC)
inuse = 'F';
@ -845,3 +1014,156 @@ gci_dump(gc_heap_t *heap)
bh_assert(cur == end);
#endif
}
#if WASM_ENABLE_GC != 0
extra_info_node_t *
gc_search_extra_info_node(gc_handle_t handle, gc_object_t obj,
gc_size_t *p_index)
{
gc_heap_t *vheap = (gc_heap_t *)handle;
int32 low = 0, high = vheap->extra_info_node_cnt - 1;
int32 mid;
extra_info_node_t *node;
if (!vheap->extra_info_nodes)
return NULL;
while (low <= high) {
mid = (low + high) / 2;
node = vheap->extra_info_nodes[mid];
if (obj == node->obj) {
if (p_index) {
*p_index = mid;
}
return node;
}
else if (obj < node->obj) {
high = mid - 1;
}
else {
low = mid + 1;
}
}
if (p_index) {
*p_index = low;
}
return NULL;
}
static bool
insert_extra_info_node(gc_heap_t *vheap, extra_info_node_t *node)
{
gc_size_t index;
extra_info_node_t *orig_node;
if (!vheap->extra_info_nodes) {
vheap->extra_info_nodes = vheap->extra_info_normal_nodes;
vheap->extra_info_node_capacity = sizeof(vheap->extra_info_normal_nodes)
/ sizeof(extra_info_node_t *);
vheap->extra_info_nodes[0] = node;
vheap->extra_info_node_cnt = 1;
return true;
}
/* extend array */
if (vheap->extra_info_node_cnt == vheap->extra_info_node_capacity) {
extra_info_node_t **new_nodes = NULL;
gc_size_t new_capacity = vheap->extra_info_node_capacity * 3 / 2;
gc_size_t total_size = sizeof(extra_info_node_t *) * new_capacity;
new_nodes = (extra_info_node_t **)BH_MALLOC(total_size);
if (!new_nodes) {
LOG_ERROR("alloc extra info nodes failed");
return false;
}
bh_memcpy_s(new_nodes, total_size, vheap->extra_info_nodes,
sizeof(extra_info_node_t *) * vheap->extra_info_node_cnt);
if (vheap->extra_info_nodes != vheap->extra_info_normal_nodes) {
BH_FREE(vheap->extra_info_nodes);
}
vheap->extra_info_nodes = new_nodes;
vheap->extra_info_node_capacity = new_capacity;
}
orig_node = gc_search_extra_info_node(vheap, node->obj, &index);
if (orig_node) {
/* replace the old node */
vheap->extra_info_nodes[index] = node;
BH_FREE(orig_node);
}
else {
bh_memmove_s(vheap->extra_info_nodes + index + 1,
(vheap->extra_info_node_capacity - index - 1)
* sizeof(extra_info_node_t *),
vheap->extra_info_nodes + index,
(vheap->extra_info_node_cnt - index)
* sizeof(extra_info_node_t *));
vheap->extra_info_nodes[index] = node;
vheap->extra_info_node_cnt += 1;
}
return true;
}
bool
gc_set_finalizer(gc_handle_t handle, gc_object_t obj, gc_finalizer_t cb,
void *data)
{
extra_info_node_t *node = NULL;
gc_heap_t *vheap = (gc_heap_t *)handle;
node = (extra_info_node_t *)BH_MALLOC(sizeof(extra_info_node_t));
if (!node) {
LOG_ERROR("alloc a new extra info node failed");
return GC_FALSE;
}
memset(node, 0, sizeof(extra_info_node_t));
node->finalizer = cb;
node->obj = obj;
node->data = data;
LOCK_HEAP(vheap);
if (!insert_extra_info_node(vheap, node)) {
BH_FREE(node);
UNLOCK_HEAP(vheap);
return GC_FALSE;
}
UNLOCK_HEAP(vheap);
gct_vm_set_extra_info_flag(obj, true);
return GC_TRUE;
}
void
gc_unset_finalizer(gc_handle_t handle, gc_object_t obj)
{
gc_size_t index;
gc_heap_t *vheap = (gc_heap_t *)handle;
extra_info_node_t *node;
LOCK_HEAP(vheap);
node = gc_search_extra_info_node(vheap, obj, &index);
if (!node) {
UNLOCK_HEAP(vheap);
return;
}
BH_FREE(node);
bh_memmove_s(
vheap->extra_info_nodes + index,
(vheap->extra_info_node_capacity - index) * sizeof(extra_info_node_t *),
vheap->extra_info_nodes + index + 1,
(vheap->extra_info_node_cnt - index - 1) * sizeof(extra_info_node_t *));
vheap->extra_info_node_cnt -= 1;
UNLOCK_HEAP(vheap);
gct_vm_set_extra_info_flag(obj, false);
}
#endif

493
core/shared/mem-alloc/ems/ems_gc.c Normal file
View File

@ -0,0 +1,493 @@
/*
* Copyright (C) 2022 Tencent Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "ems_gc.h"
#include "ems_gc_internal.h"
#define GB (1 << 30UL)
#define MARK_NODE_OBJ_CNT 256
#if WASM_ENABLE_GC != 0
/* mark node is used for gc marker*/
typedef struct mark_node_struct {
/* number of to-expand objects can be saved in this node */
gc_size_t cnt;
/* the first unused index */
uint32 idx;
/* next node on the node list */
struct mark_node_struct *next;
/* the actual to-expand objects list */
gc_object_t set[MARK_NODE_OBJ_CNT];
} mark_node_t;
/**
* Alloc a mark node from the native heap
*
* @return a valid mark node if success, NULL otherwise
*/
static mark_node_t *
alloc_mark_node(void)
{
mark_node_t *ret = (mark_node_t *)BH_MALLOC(sizeof(mark_node_t));
if (!ret) {
LOG_ERROR("alloc a new mark node failed");
return NULL;
}
ret->cnt = sizeof(ret->set) / sizeof(ret->set[0]);
ret->idx = 0;
ret->next = NULL;
return ret;
}
/* Free a mark node to the native heap
*
* @param node the mark node to free, should not be NULL
*/
static void
free_mark_node(mark_node_t *node)
{
bh_assert(node);
BH_FREE((gc_object_t)node);
}
/**
* Sweep phase of mark_sweep algorithm
* @param heap the heap to sweep, should be a valid instance heap
* which has already been marked
*/
static void
sweep_instance_heap(gc_heap_t *heap)
{
hmu_t *cur = NULL, *end = NULL, *last = NULL;
hmu_type_t ut;
gc_size_t size;
int i, lsize;
gc_size_t tot_free = 0;
bh_assert(gci_is_heap_valid(heap));
cur = (hmu_t *)heap->base_addr;
last = NULL;
end = (hmu_t *)((char *)heap->base_addr + heap->current_size);
/* reset KFC */
lsize =
(int)(sizeof(heap->kfc_normal_list) / sizeof(heap->kfc_normal_list[0]));
for (i = 0; i < lsize; i++) {
heap->kfc_normal_list[i].next = NULL;
}
heap->kfc_tree_root->right = NULL;
heap->root_set = NULL;
while (cur < end) {
ut = hmu_get_ut(cur);
size = hmu_get_size(cur);
bh_assert(size > 0);
if (ut == HMU_FC || ut == HMU_FM
|| (ut == HMU_VO && hmu_is_vo_freed(cur))
|| (ut == HMU_WO && !hmu_is_wo_marked(cur))) {
/* merge previous free areas with current one */
if (!last)
last = cur;
if (ut == HMU_WO) {
/* Invoke registered finalizer */
gc_object_t cur_obj = hmu_to_obj(cur);
if (gct_vm_get_extra_info_flag(cur_obj)) {
extra_info_node_t *node = gc_search_extra_info_node(
(gc_handle_t)heap, cur_obj, NULL);
bh_assert(node);
node->finalizer(node->obj, node->data);
gc_unset_finalizer((gc_handle_t)heap, cur_obj);
}
}
}
else {
/* current block is still live */
if (last) {
tot_free += (char *)cur - (char *)last;
gci_add_fc(heap, last, (char *)cur - (char *)last);
hmu_mark_pinuse(last);
last = NULL;
}
if (ut == HMU_WO) {
/* unmark it */
hmu_unmark_wo(cur);
}
}
cur = (hmu_t *)((char *)cur + size);
}
bh_assert(cur == end);
if (last) {
tot_free += (char *)cur - (char *)last;
gci_add_fc(heap, last, (char *)cur - (char *)last);
hmu_mark_pinuse(last);
}
heap->total_free_size = tot_free;
#if GC_STAT_DATA != 0
heap->total_gc_count++;
if ((heap->current_size - tot_free) > heap->highmark_size)
heap->highmark_size = heap->current_size - tot_free;
#endif
gc_update_threshold(heap);
}
/**
* Add a to-expand node to the to-expand list
*
* @param heap should be a valid instance heap
* @param obj should be a valid wo inside @heap
*
* @return GC_ERROR if there is no more resource for marking,
* GC_SUCCESS if success
*/
static int
add_wo_to_expand(gc_heap_t *heap, gc_object_t obj)
{
mark_node_t *mark_node = NULL, *new_node = NULL;
hmu_t *hmu = NULL;
bh_assert(obj);
hmu = obj_to_hmu(obj);
bh_assert(gci_is_heap_valid(heap));
bh_assert((gc_uint8 *)hmu >= heap->base_addr
&& (gc_uint8 *)hmu < heap->base_addr + heap->current_size);
bh_assert(hmu_get_ut(hmu) == HMU_WO);
if (hmu_is_wo_marked(hmu))
return GC_SUCCESS; /* already marked*/
mark_node = (mark_node_t *)heap->root_set;
if (!mark_node || mark_node->idx == mark_node->cnt) {
new_node = alloc_mark_node();
if (!new_node) {
LOG_ERROR("can not add obj to mark node because of mark node "
"allocation failed");
return GC_ERROR;
}
new_node->next = mark_node;
heap->root_set = new_node;
mark_node = new_node;
}
mark_node->set[mark_node->idx++] = obj;
hmu_mark_wo(hmu);
return GC_SUCCESS;
}
/* Check ems_gc.h for description*/
int
gc_add_root(void *heap_p, gc_object_t obj)
{
gc_heap_t *heap = (gc_heap_t *)heap_p;
hmu_t *hmu = NULL;
if (!obj) {
LOG_ERROR("gc_add_root with NULL obj");
return GC_ERROR;
}
hmu = obj_to_hmu(obj);
if (!gci_is_heap_valid(heap)) {
LOG_ERROR("vm_get_gc_handle_for_current_instance returns invalid heap");
return GC_ERROR;
}
if (!((gc_uint8 *)hmu >= heap->base_addr
&& (gc_uint8 *)hmu < heap->base_addr + heap->current_size)) {
LOG_ERROR("Obj is not a object in current instance heap");
return GC_ERROR;
}
if (hmu_get_ut(hmu) != HMU_WO) {
LOG_ERROR("Given object is not wo");
return GC_ERROR;
}
if (add_wo_to_expand(heap, obj) != GC_SUCCESS) {
heap->is_fast_marking_failed = 1;
return GC_ERROR;
}
return GC_SUCCESS;
}
/**
* Unmark all marked objects to do rollback
*
* @param heap the heap to do rollback, should be a valid instance heap
*/
static void
rollback_mark(gc_heap_t *heap)
{
mark_node_t *mark_node = NULL, *next_mark_node = NULL;
hmu_t *cur = NULL, *end = NULL;
hmu_type_t ut;
gc_size_t size;
bh_assert(gci_is_heap_valid(heap));
/* roll back*/
mark_node = (mark_node_t *)heap->root_set;
while (mark_node) {
next_mark_node = mark_node->next;
free_mark_node(mark_node);
mark_node = next_mark_node;
}
heap->root_set = NULL;
/* then traverse the heap to unmark all marked wos*/
cur = (hmu_t *)heap->base_addr;
end = (hmu_t *)((char *)heap->base_addr + heap->current_size);
while (cur < end) {
ut = hmu_get_ut(cur);
size = hmu_get_size(cur);
if (ut == HMU_WO && hmu_is_wo_marked(cur)) {
hmu_unmark_wo(cur);
}
cur = (hmu_t *)((char *)cur + size);
}
bh_assert(cur == end);
}
/**
* Reclaim GC instance heap
*
* @param heap the heap to reclaim, should be a valid instance heap
*
* @return GC_SUCCESS if success, GC_ERROR otherwise
*/
static int
reclaim_instance_heap(gc_heap_t *heap)
{
mark_node_t *mark_node = NULL;
int idx = 0, j = 0;
bool ret, is_compact_mode = false;
gc_object_t obj = NULL, ref = NULL;
hmu_t *hmu = NULL;
gc_uint32 ref_num = 0, ref_start_offset = 0, size = 0, offset = 0;
gc_uint16 *ref_list = NULL;
bh_assert(gci_is_heap_valid(heap));
heap->root_set = NULL;
#if WASM_ENABLE_THREAD_MGR == 0
if (!heap->exec_env)
return GC_SUCCESS;
ret = gct_vm_begin_rootset_enumeration(heap->exec_env, heap);
#else
if (!heap->cluster)
return GC_SUCCESS;
ret = gct_vm_begin_rootset_enumeration(heap->cluster, heap);
#endif
if (!ret)
return GC_ERROR;
#if BH_ENABLE_GC_VERIFY != 0
/* no matter whether the enumeration is successful or not, the data
collected should be checked at first */
mark_node = (mark_node_t *)heap->root_set;
while (mark_node) {
/* all nodes except first should be full filled */
bh_assert(mark_node == (mark_node_t *)heap->root_set
|| mark_node->idx == mark_node->cnt);
/* all nodes should be non-empty */
bh_assert(mark_node->idx > 0);
for (idx = 0; idx < (int)mark_node->idx; idx++) {
obj = mark_node->set[idx];
hmu = obj_to_hmu(obj);
bh_assert(hmu_is_wo_marked(hmu));
bh_assert((gc_uint8 *)hmu >= heap->base_addr
&& (gc_uint8 *)hmu
< heap->base_addr + heap->current_size);
}
mark_node = mark_node->next;
}
#endif
/* TODO: when fast marking failed, we can still do slow
marking, currently just simply roll it back. */
if (heap->is_fast_marking_failed) {
LOG_ERROR("enumerate rootset failed");
LOG_ERROR("all marked wos will be unmarked to keep heap consistency");
rollback_mark(heap);
heap->is_fast_marking_failed = 0;
return GC_ERROR;
}
/* the algorithm we use to mark all objects */
/* 1. mark rootset and organize them into a mark_node list (last marked
* roots at list header, i.e. stack top) */
/* 2. in every iteration, we use the top node to expand*/
/* 3. execute step 2 till no expanding */
/* this is a BFS & DFS mixed algorithm, but more like DFS */
mark_node = (mark_node_t *)heap->root_set;
while (mark_node) {
heap->root_set = mark_node->next;
/* note that mark_node->idx may change in each loop */
for (idx = 0; idx < (int)mark_node->idx; idx++) {
obj = mark_node->set[idx];
hmu = obj_to_hmu(obj);
size = hmu_get_size(hmu);
if (!gct_vm_get_wasm_object_ref_list(obj, &is_compact_mode,
&ref_num, &ref_list,
&ref_start_offset)) {
LOG_ERROR("mark process failed because failed "
"vm_get_wasm_object_ref_list");
break;
}
if (ref_num >= 2U * GB) {
LOG_ERROR("Invalid ref_num returned");
break;
}
if (is_compact_mode) {
for (j = 0; j < (int)ref_num; j++) {
offset = ref_start_offset + j * sizeof(void *);
bh_assert(offset + sizeof(void *) < size);
ref = *(gc_object_t *)(((gc_uint8 *)obj) + offset);
if (ref == NULL_REF || ((uintptr_t)ref & 1))
continue; /* null object or i31 object */
if (add_wo_to_expand(heap, ref) == GC_ERROR) {
LOG_ERROR("add_wo_to_expand failed");
break;
}
}
if (j < (int)ref_num)
break;
}
else {
for (j = 0; j < (int)ref_num; j++) {
offset = ref_list[j];
bh_assert(offset + sizeof(void *) < size);
ref = *(gc_object_t *)(((gc_uint8 *)obj) + offset);
if (ref == NULL_REF || ((uintptr_t)ref & 1))
continue; /* null object or i31 object */
if (add_wo_to_expand(heap, ref) == GC_ERROR) {
LOG_ERROR("mark process failed");
break;
}
}
if (j < (int)ref_num)
break;
}
}
if (idx < (int)mark_node->idx)
break; /* not yet done */
/* obj's in mark_node are all expanded */
free_mark_node(mark_node);
mark_node = heap->root_set;
}
if (mark_node) {
LOG_ERROR("mark process is not successfully finished");
free_mark_node(mark_node);
/* roll back is required */
rollback_mark(heap);
return GC_ERROR;
}
/* now sweep */
sweep_instance_heap(heap);
(void)size;
return GC_SUCCESS;
}
/**
* Do GC on given heap
*
* @param the heap to do GC, should be a valid heap
*
* @return GC_SUCCESS if success, GC_ERROR otherwise
*/
int
gci_gc_heap(void *h)
{
int ret = GC_ERROR;
gc_heap_t *heap = (gc_heap_t *)h;
bh_assert(gci_is_heap_valid(heap));
LOG_VERBOSE("#reclaim instance heap %p", heap);
gct_vm_gc_prepare();
gct_vm_mutex_lock(&heap->lock);
heap->is_doing_reclaim = 1;
ret = reclaim_instance_heap(heap);
heap->is_doing_reclaim = 0;
gct_vm_mutex_unlock(&heap->lock);
gct_vm_gc_finished();
LOG_VERBOSE("#reclaim instance heap %p done", heap);
#if BH_ENABLE_GC_VERIFY != 0
gci_verify_heap(heap);
#endif
#if GC_STAT_SHOW != 0
gc_show_stat(heap);
gc_show_fragment(heap);
#endif
return ret;
}
int
gc_is_dead_object(void *obj)
{
return !hmu_is_wo_marked(obj_to_hmu(obj));
}
#else
int
gci_gc_heap(void *h)
{
(void)h;
return GC_ERROR;
}
#endif /* end of WASM_ENABLE_GC != 0 */

179
core/shared/mem-alloc/ems/ems_gc.h
View File

@ -19,9 +19,27 @@
extern "C" {
#endif
#ifndef GC_STAT_DATA
#define GC_STAT_DATA 0
#endif
#ifndef GC_STAT_SHOW
#define GC_STAT_SHOW 0
#endif
#ifndef GC_IN_EVERY_ALLOCATION
#define GC_IN_EVERY_ALLOCATION 0
#endif
#ifndef GC_MANUALLY
#define GC_MANUALLY 0
#endif
#define GC_HEAD_PADDING 4
#ifndef NULL_REF
#define NULL_REF ((gc_object_t)NULL)
#endif
#define GC_SUCCESS (0)
#define GC_ERROR (-1)
@ -33,6 +51,7 @@ extern "C" {
typedef void *gc_handle_t;
typedef void *gc_object_t;
typedef uint64 gc_uint64;
typedef int64 gc_int64;
typedef uint32 gc_uint32;
typedef int32 gc_int32;
@ -46,8 +65,24 @@ typedef enum {
GC_STAT_TOTAL = 0,
GC_STAT_FREE,
GC_STAT_HIGHMARK,
GC_STAT_COUNT,
GC_STAT_TIME,
GC_STAT_MAX
} GC_STAT_INDEX;
typedef void (*gc_finalizer_t)(void *obj, void *data);
#ifndef EXTRA_INFO_NORMAL_NODE_CNT
#define EXTRA_INFO_NORMAL_NODE_CNT 32
#endif
/* extra information attached to specific object */
typedef struct extra_info_node {
gc_object_t obj;
gc_finalizer_t finalizer;
void *data;
} extra_info_node_t;
/**
* GC initialization from a buffer, which is separated into
* two parts: the beginning of the buffer is used to create
@ -87,6 +122,28 @@ gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size,
int
gc_destroy_with_pool(gc_handle_t handle);
#if WASM_ENABLE_GC != 0
/**
* Enable or disable GC reclaim for a heap
*
* @param handle handle of the heap
* @param exec_env the exec_env of current module instance
*/
#if WASM_ENABLE_THREAD_MGR == 0
void
gc_enable_gc_reclaim(gc_handle_t handle, void *exec_env);
#else
/**
* Enable or disable GC reclaim for a heap
*
* @param handle handle of the heap
* @param cluster the tread cluster of current module instance
*/
void
gc_enable_gc_reclaim(gc_handle_t handle, void *cluster);
#endif
#endif
/**
* Return heap struct size
*/
@ -136,6 +193,14 @@ gc_realloc_vo(void *heap, void *ptr, gc_size_t size);
int
gc_free_vo(void *heap, gc_object_t obj);
#if WASM_ENABLE_GC != 0
gc_object_t
gc_alloc_wo(void *heap, gc_size_t size);
void
gc_free_wo(void *vheap, void *ptr);
#endif
#else /* else of BH_ENABLE_GC_VERIFY */
gc_object_t
@ -148,6 +213,14 @@ gc_realloc_vo_internal(void *heap, void *ptr, gc_size_t size, const char *file,
int
gc_free_vo_internal(void *heap, gc_object_t obj, const char *file, int line);
#if WASM_ENABLE_GC != 0
gc_object_t
gc_alloc_wo_internal(void *heap, gc_size_t size, const char *file, int line);
void
gc_free_wo_internal(void *vheap, void *ptr, const char *file, int line);
#endif
/* clang-format off */
#define gc_alloc_vo(heap, size) \
gc_alloc_vo_internal(heap, size, __FILE__, __LINE__)
@ -157,10 +230,116 @@ gc_free_vo_internal(void *heap, gc_object_t obj, const char *file, int line);
#define gc_free_vo(heap, obj) \
gc_free_vo_internal(heap, obj, __FILE__, __LINE__)
#if WASM_ENABLE_GC != 0
#define gc_alloc_wo(heap, size) \
gc_alloc_wo_internal(heap, size, __FILE__, __LINE__)
#define gc_free_wo(heap, obj) \
gc_free_wo_internal(heap, obj, __FILE__, __LINE__)
#endif
/* clang-format on */
#endif /* end of BH_ENABLE_GC_VERIFY */
#if WASM_ENABLE_GC != 0
/**
* Add gc object ref to the rootset of a gc heap.
*
* @param heap the heap to add the gc object to its rootset
* @param obj pointer to a valid WASM object managed by the gc heap.
*
* @return GC_SUCCESS if success, GC_ERROR otherwise
*/
int
gc_add_root(void *heap, gc_object_t obj);
int
gci_gc_heap(void *heap);
extra_info_node_t *
gc_search_extra_info_node(gc_handle_t handle, gc_object_t obj,
gc_size_t *p_index);
/**
* Set finalizer to the given object, if another finalizer is set to the same
* object, the previous one will be cancelled
*
* @param handle handle of the heap
* @param obj object to set finalizer
* @param cb finalizer function to be called before this object is freed
* @param data custom data to be passed to finalizer function
*
* @return true if success, false otherwise
*/
bool
gc_set_finalizer(gc_handle_t handle, gc_object_t obj, gc_finalizer_t cb,
void *data);
/**
* Unset finalizer to the given object
*
* @param handle handle of the heap
* @param obj object to unset finalizer
*/
void
gc_unset_finalizer(gc_handle_t handle, gc_object_t obj);
#if WASM_ENABLE_THREAD_MGR == 0
bool
wasm_runtime_traverse_gc_rootset(void *exec_env, void *heap);
#else
bool
wasm_runtime_traverse_gc_rootset(void *cluster, void *heap);
#endif
bool
wasm_runtime_get_wasm_object_ref_list(gc_object_t obj, bool *p_is_compact_mode,
gc_uint32 *p_ref_num,
gc_uint16 **p_ref_list,
gc_uint32 *p_ref_start_offset);
bool
wasm_runtime_get_wasm_object_extra_info_flag(gc_object_t obj);
void
wasm_runtime_set_wasm_object_extra_info_flag(gc_object_t obj, bool set);
void
wasm_runtime_gc_prepare();
void
wasm_runtime_gc_finalize();
#endif /* end of WASM_ENABLE_GC != 0 */
#define GC_HEAP_STAT_SIZE (128 / 4)
typedef struct {
int usage;
int usage_block;
int vo_usage;
int wo_usage;
int free;
int free_block;
int vo_free;
int wo_free;
int usage_sizes[GC_HEAP_STAT_SIZE];
int free_sizes[GC_HEAP_STAT_SIZE];
} gc_stat_t;
void
gc_show_stat(gc_handle_t handle);
#if WASM_ENABLE_GC != 0
void
gc_show_fragment(gc_handle_t handle);
#if WASM_ENABLE_GC_PERF_PROFILING != 0
void
gc_dump_perf_profiling(gc_handle_t *handle);
#endif
#endif
#ifdef __cplusplus
}
#endif

89
core/shared/mem-alloc/ems/ems_gc_internal.h
View File

@ -17,8 +17,8 @@ extern "C" {
typedef enum hmu_type_enum {
HMU_TYPE_MIN = 0,
HMU_TYPE_MAX = 3,
HMU_JO = 3,
HMU_VO = 2,
HMU_WO = 3, /* WASM Object */
HMU_VO = 2, /* VM Object */
HMU_FC = 1,
HMU_FM = 0
} hmu_type_t;
@ -135,13 +135,13 @@ hmu_verify(void *vheap, hmu_t *hmu);
#define hmu_unmark_pinuse(hmu) CLRBIT((hmu)->header, HMU_P_OFFSET)
#define hmu_get_pinuse(hmu) GETBIT((hmu)->header, HMU_P_OFFSET)
#define HMU_JO_VT_SIZE 27
#define HMU_JO_VT_OFFSET 0
#define HMU_JO_MB_OFFSET 28
#define HMU_WO_VT_SIZE 27
#define HMU_WO_VT_OFFSET 0
#define HMU_WO_MB_OFFSET 28
#define hmu_mark_jo(hmu) SETBIT((hmu)->header, HMU_JO_MB_OFFSET)
#define hmu_unmark_jo(hmu) CLRBIT((hmu)->header, HMU_JO_MB_OFFSET)
#define hmu_is_jo_marked(hmu) GETBIT((hmu)->header, HMU_JO_MB_OFFSET)
#define hmu_mark_wo(hmu) SETBIT((hmu)->header, HMU_WO_MB_OFFSET)
#define hmu_unmark_wo(hmu) CLRBIT((hmu)->header, HMU_WO_MB_OFFSET)
#define hmu_is_wo_marked(hmu) GETBIT((hmu)->header, HMU_WO_MB_OFFSET)
/**
* The hmu size is divisible by 8, its lowest 3 bits are 0, so we only
@ -271,6 +271,33 @@ typedef struct gc_heap_struct {
size[left] <= size[cur] < size[right] */
hmu_tree_node_t *kfc_tree_root;
#if WASM_ENABLE_GC != 0
/* for rootset enumeration of private heap*/
void *root_set;
#if WASM_ENABLE_THREAD_MGR == 0
/* exec_env of current wasm module instance */
void *exec_env;
#else
/* thread cluster of current module instances */
void *cluster;
#endif
/* whether the fast mode of marking process that requires
additional memory fails. When the fast mode fails, the
marking process can still be done in the slow mode, which
doesn't need additional memory (by walking through all
blocks and marking sucessors of marked nodes until no new
node is marked). TODO: slow mode is not implemented. */
unsigned is_fast_marking_failed : 1;
/* whether the heap is doing reclaim */
unsigned is_doing_reclaim : 1;
/* Whether the heap can do reclaim */
unsigned is_reclaim_enabled : 1;
#endif
#if BH_ENABLE_GC_CORRUPTION_CHECK != 0
/* whether heap is corrupted, e.g. the hmu nodes are modified
by user */
@ -280,8 +307,54 @@ typedef struct gc_heap_struct {
gc_size_t init_size;
gc_size_t highmark_size;
gc_size_t total_free_size;
#if WASM_ENABLE_GC != 0
gc_size_t gc_threshold;
gc_size_t gc_threshold_factor;
gc_size_t total_gc_count;
gc_size_t total_gc_time;
gc_size_t max_gc_time;
/* Usually there won't be too many extra info node, so we try to use a fixed
* array to store them, if the fixed array don't have enough space to store
* the nodes, a new space will be allocated from heap */
extra_info_node_t *extra_info_normal_nodes[EXTRA_INFO_NORMAL_NODE_CNT];
/* Used to store extra information such as finalizer for specified nodes, we
* introduce a seperate space to store these information so only nodes who
* really require extra information will occupy additional memory spaces. */
extra_info_node_t **extra_info_nodes;
gc_size_t extra_info_node_cnt;
gc_size_t extra_info_node_capacity;
#endif
#if GC_STAT_DATA != 0
gc_uint64 total_size_allocated;
gc_uint64 total_size_freed;
#endif
} gc_heap_t;
#if WASM_ENABLE_GC != 0
#define GC_DEFAULT_THRESHOLD_FACTOR 300
static inline void
gc_update_threshold(gc_heap_t *heap)
{
heap->gc_threshold =
heap->total_free_size * heap->gc_threshold_factor / 1000;
}
#define gct_vm_mutex_init os_mutex_init
#define gct_vm_mutex_destroy os_mutex_destroy
#define gct_vm_mutex_lock os_mutex_lock
#define gct_vm_mutex_unlock os_mutex_unlock
#define gct_vm_gc_prepare wasm_runtime_gc_prepare
#define gct_vm_gc_finished wasm_runtime_gc_finalize
#define gct_vm_begin_rootset_enumeration wasm_runtime_traverse_gc_rootset
#define gct_vm_get_wasm_object_ref_list wasm_runtime_get_wasm_object_ref_list
#define gct_vm_get_extra_info_flag wasm_runtime_get_wasm_object_extra_info_flag
#define gct_vm_set_extra_info_flag wasm_runtime_set_wasm_object_extra_info_flag
#endif /* end of WAMS_ENABLE_GC != 0 */
/**
* MISC internal used APIs
*/

6
core/shared/mem-alloc/ems/ems_hmu.c
View File

@ -24,7 +24,7 @@ hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size,
gc_uint32 i = 0;
bh_assert(hmu);
bh_assert(hmu_get_ut(hmu) == HMU_JO || hmu_get_ut(hmu) == HMU_VO);
bh_assert(hmu_get_ut(hmu) == HMU_WO || hmu_get_ut(hmu) == HMU_VO);
bh_assert(tot_size >= OBJ_EXTRA_SIZE);
bh_assert(!(tot_size & 7));
bh_assert(hmu_get_ut(hmu) != HMU_VO || hmu_get_size(hmu) >= tot_size);
@ -48,7 +48,9 @@ hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size,
void
hmu_verify(void *vheap, hmu_t *hmu)
{
#if BH_ENABLE_GC_CORRUPTION_CHECK != 0
gc_heap_t *heap = (gc_heap_t *)vheap;
#endif
gc_object_prefix_t *prefix = NULL;
gc_object_suffix_t *suffix = NULL;
gc_uint32 i = 0;
@ -64,7 +66,7 @@ hmu_verify(void *vheap, hmu_t *hmu)
size = prefix->size;
suffix = (gc_object_suffix_t *)((gc_uint8 *)hmu + size - OBJ_SUFFIX_SIZE);
if (ut == HMU_VO || ut == HMU_JO) {
if (ut == HMU_VO || ut == HMU_WO) {
/* check padding*/
for (i = 0; i < GC_OBJECT_PREFIX_PADDING_CNT; i++) {
if (prefix->padding[i] != GC_OBJECT_PADDING_VALUE) {

215
core/shared/mem-alloc/ems/ems_kfc.c
View File

@ -12,6 +12,7 @@ gc_init_internal(gc_heap_t *heap, char *base_addr, gc_size_t heap_max_size)
int ret;
memset(heap, 0, sizeof *heap);
memset(base_addr, 0, heap_max_size);
ret = os_mutex_init(&heap->lock);
if (ret != BHT_OK) {
@ -26,6 +27,10 @@ gc_init_internal(gc_heap_t *heap, char *base_addr, gc_size_t heap_max_size)
heap->total_free_size = heap->current_size;
heap->highmark_size = 0;
#if WASM_ENABLE_GC != 0
heap->gc_threshold_factor = GC_DEFAULT_THRESHOLD_FACTOR;
gc_update_threshold(heap);
#endif
root = heap->kfc_tree_root = (hmu_tree_node_t *)heap->kfc_tree_root_buf;
memset(root, 0, sizeof *root);
@ -129,6 +134,28 @@ gc_destroy_with_pool(gc_handle_t handle)
gc_heap_t *heap = (gc_heap_t *)handle;
int ret = GC_SUCCESS;
#if WASM_ENABLE_GC != 0
gc_size_t i = 0;
if (heap->extra_info_node_cnt > 0) {
for (i = 0; i < heap->extra_info_node_cnt; i++) {
extra_info_node_t *node = heap->extra_info_nodes[i];
#if BH_ENABLE_GC_VERIFY != 0
os_printf("Memory leak detected: gc object [%p] not claimed\n",
node->obj);
#endif
bh_assert(heap->is_reclaim_enabled);
node->finalizer(node->obj, node->data);
BH_FREE(heap->extra_info_nodes[i]);
}
if (heap->extra_info_nodes != heap->extra_info_normal_nodes) {
BH_FREE(heap->extra_info_nodes);
}
}
#endif
#if BH_ENABLE_GC_VERIFY != 0
hmu_t *cur = (hmu_t *)heap->base_addr;
hmu_t *end = (hmu_t *)((char *)heap->base_addr + heap->current_size);
@ -145,10 +172,33 @@ gc_destroy_with_pool(gc_handle_t handle)
#endif
os_mutex_destroy(&heap->lock);
memset(heap->base_addr, 0, heap->current_size);
memset(heap, 0, sizeof(gc_heap_t));
return ret;
}
#if WASM_ENABLE_GC != 0
#if WASM_ENABLE_THREAD_MGR == 0
void
gc_enable_gc_reclaim(gc_handle_t handle, void *exec_env)
{
gc_heap_t *heap = (gc_heap_t *)handle;
heap->is_reclaim_enabled = 1;
heap->exec_env = exec_env;
}
#else
void
gc_enable_gc_reclaim(gc_handle_t handle, void *cluster)
{
gc_heap_t *heap = (gc_heap_t *)handle;
heap->is_reclaim_enabled = 1;
heap->cluster = cluster;
}
#endif
#endif
uint32
gc_get_heap_struct_size()
{
@ -287,12 +337,103 @@ gci_verify_heap(gc_heap_t *heap)
}
#endif
void
gc_heap_stat(void *heap_ptr, gc_stat_t *stat)
{
hmu_t *cur = NULL, *end = NULL;
hmu_type_t ut;
gc_size_t size;
gc_heap_t *heap = (gc_heap_t *)heap_ptr;
memset(stat, 0, sizeof(gc_stat_t));
cur = (hmu_t *)heap->base_addr;
end = (hmu_t *)((char *)heap->base_addr + heap->current_size);
while (cur < end) {
ut = hmu_get_ut(cur);
size = hmu_get_size(cur);
bh_assert(size > 0);
if (ut == HMU_FC || ut == HMU_FM
|| (ut == HMU_VO && hmu_is_vo_freed(cur))
|| (ut == HMU_WO && !hmu_is_wo_marked(cur))) {
if (ut == HMU_VO)
stat->vo_free += size;
if (ut == HMU_WO)
stat->wo_free += size;
stat->free += size;
stat->free_block++;
if (size / sizeof(int) < GC_HEAP_STAT_SIZE - 1)
stat->free_sizes[size / sizeof(int)] += 1;
else
stat->free_sizes[GC_HEAP_STAT_SIZE - 1] += 1;
}
else {
if (ut == HMU_VO)
stat->vo_usage += size;
if (ut == HMU_WO)
stat->wo_usage += size;
stat->usage += size;
stat->usage_block++;
if (size / sizeof(int) < GC_HEAP_STAT_SIZE - 1)
stat->usage_sizes[size / sizeof(int)] += 1;
else
stat->usage_sizes[GC_HEAP_STAT_SIZE - 1] += 1;
}
cur = (hmu_t *)((char *)cur + size);
}
}
void
gc_print_stat(void *heap_ptr, int verbose)
{
gc_stat_t stat;
int i;
bh_assert(heap_ptr != NULL);
gc_heap_t *heap = (gc_heap_t *)(heap_ptr);
gc_heap_stat(heap, &stat);
os_printf("# stat %s %p use %d free %d \n", "instance", heap, stat.usage,
stat.free);
os_printf("# stat %s %p wo_usage %d vo_usage %d \n", "instance", heap,
stat.wo_usage, stat.vo_usage);
os_printf("# stat %s %p wo_free %d vo_free %d \n", "instance", heap,
stat.wo_free, stat.vo_free);
#if WASM_ENABLE_GC == 0
os_printf("# stat free size %" PRIu32 " high %" PRIu32 "\n",
heap->total_free_size, heap->highmark_size);
#else
os_printf("# stat gc %" PRIu32 " free size %" PRIu32 " high %" PRIu32 "\n",
heap->total_gc_count, heap->total_free_size, heap->highmark_size);
#endif
if (verbose) {
os_printf("usage sizes: \n");
for (i = 0; i < GC_HEAP_STAT_SIZE; i++)
if (stat.usage_sizes[i])
os_printf(" %d: %d; ", i * 4, stat.usage_sizes[i]);
os_printf(" \n");
os_printf("free sizes: \n");
for (i = 0; i < GC_HEAP_STAT_SIZE; i++)
if (stat.free_sizes[i])
os_printf(" %d: %d; ", i * 4, stat.free_sizes[i]);
}
}
void *
gc_heap_stats(void *heap_arg, uint32 *stats, int size)
{
int i;
gc_heap_t *heap = (gc_heap_t *)heap_arg;
if (!gci_is_heap_valid(heap)) {
for (i = 0; i < size; i++)
stats[i] = 0;
return NULL;
}
for (i = 0; i < size; i++) {
switch (i) {
case GC_STAT_TOTAL:
@ -304,9 +445,83 @@ gc_heap_stats(void *heap_arg, uint32 *stats, int size)
case GC_STAT_HIGHMARK:
stats[i] = heap->highmark_size;
break;
#if WASM_ENABLE_GC != 0
case GC_STAT_COUNT:
stats[i] = heap->total_gc_count;
break;
case GC_STAT_TIME:
stats[i] = heap->total_gc_time;
break;
#endif
default:
break;
}
}
return heap;
}
void
gc_traverse_tree(hmu_tree_node_t *node, gc_size_t *stats, int *n)
{
if (!node)
return;
if (*n > 0)
gc_traverse_tree(node->right, stats, n);
if (*n > 0) {
(*n)--;
stats[*n] = node->size;
}
if (*n > 0)
gc_traverse_tree(node->left, stats, n);
}
void
gc_show_stat(void *heap)
{
uint32 stats[GC_STAT_MAX];
heap = gc_heap_stats(heap, stats, GC_STAT_MAX);
os_printf("\n[GC stats %p] %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32
" %" PRIu32 "\n",
heap, stats[0], stats[1], stats[2], stats[3], stats[4]);
}
#if WASM_ENABLE_GC != 0
void
gc_show_fragment(void *heap_arg)
{
uint32 stats[3];
int n = 3;
gc_heap_t *heap = (gc_heap_t *)heap_arg;
memset(stats, 0, n * sizeof(int));
gct_vm_mutex_lock(&heap->lock);
gc_traverse_tree(heap->kfc_tree_root, (gc_size_t *)stats, &n);
gct_vm_mutex_unlock(&heap->lock);
os_printf("\n[GC %p top sizes] %" PRIu32 " %" PRIu32 " %" PRIu32 "\n", heap,
stats[0], stats[1], stats[2]);
}
#if WASM_ENABLE_GC_PERF_PROFILING != 0
void
gc_dump_perf_profiling(gc_handle_t *handle)
{
gc_heap_t *gc_heap_handle = (void *)handle;
if (gc_heap_handle) {
os_printf("\nGC performance summary\n");
os_printf(" Total GC time (ms): %u\n",
gc_heap_handle->total_gc_time);
os_printf(" Max GC time (ms): %u\n", gc_heap_handle->max_gc_time);
}
else {
os_printf("Failed to dump GC performance\n");
}
}
#endif
#endif

62
core/shared/mem-alloc/mem_alloc.c
View File

@ -4,6 +4,7 @@
*/
#include "mem_alloc.h"
#include <stdbool.h>
#if DEFAULT_MEM_ALLOCATOR == MEM_ALLOCATOR_EMS
@ -56,6 +57,43 @@ mem_allocator_free(mem_allocator_t allocator, void *ptr)
gc_free_vo((gc_handle_t)allocator, ptr);
}
#if WASM_ENABLE_GC != 0
void *
mem_allocator_malloc_with_gc(mem_allocator_t allocator, uint32_t size)
{
return gc_alloc_wo((gc_handle_t)allocator, size);
}
#if WASM_GC_MANUALLY != 0
void
mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr)
{
if (ptr)
gc_free_wo((gc_handle_t)allocator, ptr);
}
#endif
#if WASM_ENABLE_THREAD_MGR == 0
void
mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env)
{
return gc_enable_gc_reclaim((gc_handle_t)allocator, exec_env);
}
#else
void
mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster)
{
return gc_enable_gc_reclaim((gc_handle_t)allocator, cluster);
}
#endif
int
mem_allocator_add_root(mem_allocator_t allocator, WASMObjectRef obj)
{
return gc_add_root((gc_handle_t)allocator, (gc_object_t)obj);
}
#endif
int
mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new,
uint32 pool_buf_size)
@ -76,6 +114,30 @@ mem_allocator_get_alloc_info(mem_allocator_t allocator, void *mem_alloc_info)
return true;
}
#if WASM_ENABLE_GC != 0
bool
mem_allocator_set_gc_finalizer(mem_allocator_t allocator, void *obj,
gc_finalizer_t cb, void *data)
{
return gc_set_finalizer((gc_handle_t)allocator, (gc_object_t)obj, cb, data);
}
void
mem_allocator_unset_gc_finalizer(mem_allocator_t allocator, void *obj)
{
gc_unset_finalizer((gc_handle_t)allocator, (gc_object_t)obj);
}
#if WASM_ENABLE_GC_PERF_PROFILING != 0
void
mem_allocator_dump_perf_profiling(mem_allocator_t allocator)
{
gc_dump_perf_profiling((gc_handle_t)allocator);
}
#endif
#endif
#else /* else of DEFAULT_MEM_ALLOCATOR */
#include "tlsf/tlsf.h"

8
core/shared/mem-alloc/mem_alloc.cmake
View File

@ -11,7 +11,13 @@ if (WAMR_BUILD_GC_VERIFY EQUAL 1)
endif ()
if (NOT DEFINED WAMR_BUILD_GC_CORRUPTION_CHECK)
set (WAMR_BUILD_GC_CORRUPTION_CHECK 1)
# Disable memory allocator heap corruption check
# when GC is enabled
if (WAMR_BUILD_GC EQUAL 1)
set (WAMR_BUILD_GC_CORRUPTION_CHECK 0)
else ()
set (WAMR_BUILD_GC_CORRUPTION_CHECK 1)
endif ()
endif ()
if (WAMR_BUILD_GC_CORRUPTION_CHECK EQUAL 0)

38
core/shared/mem-alloc/mem_alloc.h
View File

@ -7,6 +7,9 @@
#define __MEM_ALLOC_H
#include "bh_platform.h"
#if WASM_ENABLE_GC != 0
#include "../../common/gc/gc_object.h"
#endif
#ifdef __cplusplus
extern "C" {
@ -14,6 +17,8 @@ extern "C" {
typedef void *mem_allocator_t;
typedef void (*gc_finalizer_t)(void *obj, void *data);
mem_allocator_t
mem_allocator_create(void *mem, uint32_t size);
@ -45,6 +50,39 @@ mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new,
bool
mem_allocator_is_heap_corrupted(mem_allocator_t allocator);
#if WASM_ENABLE_GC != 0
void *
mem_allocator_malloc_with_gc(mem_allocator_t allocator, uint32_t size);
#if WASM_GC_MANUALLY != 0
void
mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr);
#endif
#if WASM_ENABLE_THREAD_MGR == 0
void
mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env);
#else
void
mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster);
#endif
int
mem_allocator_add_root(mem_allocator_t allocator, WASMObjectRef obj);
bool
mem_allocator_set_gc_finalizer(mem_allocator_t allocator, void *obj,
gc_finalizer_t cb, void *data);
void
mem_allocator_unset_gc_finalizer(mem_allocator_t allocator, void *obj);
#if WASM_ENABLE_GC_PERF_PROFILING != 0
void
mem_allocator_dump_perf_profiling(mem_allocator_t allocator);
#endif
#endif /* end of WASM_ENABLE_GC != 0 */
bool
mem_allocator_get_alloc_info(mem_allocator_t allocator, void *mem_alloc_info);