christoph/wamr
1
Fork 0
Code Releases Activity

Implement GC (Garbage Collection) feature for interpreter, AOT and LLVM-JIT (#3125)

Browse Source 
Implement the GC (Garbage Collection) feature for interpreter mode,
AOT mode and LLVM-JIT mode, and support most features of the latest
spec proposal, and also enable the stringref feature.

Use `cmake -DWAMR_BUILD_GC=1/0` to enable/disable the feature,
and `wamrc --enable-gc` to generate the AOT file with GC supported.

And update the AOT file version from 2 to 3 since there are many AOT
ABI breaks, including the changes of AOT file format, the changes of
AOT module/memory instance layouts, the AOT runtime APIs for the
AOT code to invoke and so on.
This commit is contained in:
Wenyong Huang
2024-02-06 20:47:11 +08:00
committed by GitHub
parent 5931aaacbe
commit 16a4d71b34
98 changed files with 33469 additions and 3159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

451
core/iwasm/compilation/aot_compiler.h
View File

@ -8,6 +8,8 @@
#include "aot.h"
#include "aot_llvm.h"
#include "../interpreter/wasm_interp.h"
#include "../aot/aot_runtime.h"
#ifdef __cplusplus
extern "C" {
@ -78,8 +80,33 @@ typedef enum FloatArithmetic {
FLOAT_MAX,
} FloatArithmetic;
/**
* Check whether a value type is a GC reference type,
* don't use wasm_is_type_reftype since it requires
* GC feature and may result in compilation error when
* GC feature isn't compiled
*/
static inline bool
check_type_compatible(uint8 src_type, uint8 dst_type)
aot_is_type_gc_reftype(uint8 type)
{
return ((type >= (uint8)REF_TYPE_ARRAYREF
&& type <= (uint8)REF_TYPE_NULLFUNCREF)
|| (type >= (uint8)REF_TYPE_HT_NULLABLE
&& type <= (uint8)REF_TYPE_HT_NON_NULLABLE)
#if WASM_ENABLE_STRINGREF != 0
|| (type >= (uint8)REF_TYPE_STRINGVIEWWTF8
&& type <= (uint8)REF_TYPE_STRINGREF)
|| (type >= (uint8)REF_TYPE_STRINGVIEWITER
&& type <= (uint8)REF_TYPE_STRINGVIEWWTF16)
#endif
)
? true
: false;
}
static inline bool
check_type_compatible(const AOTCompContext *comp_ctx, uint8 src_type,
uint8 dst_type)
{
if (src_type == dst_type) {
return true;
@ -92,20 +119,332 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
/* i32 <==> func.ref, i32 <==> extern.ref */
if (src_type == VALUE_TYPE_I32
&& (dst_type == VALUE_TYPE_EXTERNREF
|| dst_type == VALUE_TYPE_FUNCREF)) {
&& (comp_ctx->enable_ref_types
&& (dst_type == VALUE_TYPE_EXTERNREF
|| dst_type == VALUE_TYPE_FUNCREF))) {
return true;
}
if (dst_type == VALUE_TYPE_I32
&& (src_type == VALUE_TYPE_FUNCREF
|| src_type == VALUE_TYPE_EXTERNREF)) {
&& (comp_ctx->enable_ref_types
&& (src_type == VALUE_TYPE_FUNCREF
|| src_type == VALUE_TYPE_EXTERNREF))) {
return true;
}
return false;
}
/**
* Operations for AOTCompFrame
*/
/**
* Get the offset from frame pointer to the n-th local variable slot.
*
* @param n the index to the local variable array
*
* @return the offset from frame pointer to the local variable slot
*/
static inline uint32
offset_of_local(AOTCompContext *comp_ctx, unsigned n)
{
if (!comp_ctx->is_jit_mode)
/* In AOTFrame, there are 7 pointers before field lp */
return comp_ctx->pointer_size
* (offsetof(AOTFrame, lp) / sizeof(uintptr_t))
+ sizeof(uint32) * n;
else
return offsetof(WASMInterpFrame, lp) + sizeof(uint32) * n;
}
uint32
offset_of_local_in_outs_area(AOTCompContext *comp_ctx, unsigned n);
/**
* Get the offset from frame pointer to the n-th local variable's
* reference flag slot.
*
* @param n the index to the local variable array
*
* @return the offset from frame pointer to the local variable slot
*/
static inline unsigned
offset_of_ref(AOTCompContext *comp_ctx, unsigned n)
{
AOTCompFrame *frame = comp_ctx->aot_frame;
uint32 all_cell_num = frame->max_local_cell_num + frame->max_stack_cell_num;
return offset_of_local(comp_ctx, all_cell_num) + n;
}
/**
* Generate instructions to commit computation result to the frame.
* The general principle is to only commit values that will be used
* through the frame.
*
* @param frame the frame information
*/
bool
aot_gen_commit_values(AOTCompFrame *frame);
/**
* Generate instructions to commit SP and IP pointers to the frame.
*
* @param frame the frame information
*/
bool
aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip);
bool
aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value,
uint8 value_type, LLVMValueRef cur_frame, uint32 offset);
static inline void
push_32bit(AOTCompFrame *frame, AOTValue *aot_value)
{
frame->sp->value = aot_value->value;
frame->sp->type = aot_value->type;
frame->sp->dirty = 1;
frame->sp++;
}
static inline void
push_64bit(AOTCompFrame *frame, AOTValue *aot_value)
{
push_32bit(frame, aot_value);
push_32bit(frame, aot_value);
}
static inline void
push_i32(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_I32
|| aot_value->type == VALUE_TYPE_I1);
push_32bit(frame, aot_value);
}
static inline void
push_i64(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_I64);
push_64bit(frame, aot_value);
}
static inline void
push_f32(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_F32);
push_32bit(frame, aot_value);
}
static inline void
push_f64(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_F64);
push_64bit(frame, aot_value);
}
static inline void
push_v128(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(aot_value->type == VALUE_TYPE_V128);
push_64bit(frame, aot_value);
push_64bit(frame, aot_value);
}
static inline void
push_ref(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(frame->comp_ctx->enable_ref_types);
push_32bit(frame, aot_value);
}
#if WASM_ENABLE_GC != 0
static inline void
push_gc_ref(AOTCompFrame *frame, AOTValue *aot_value)
{
bh_assert(frame->comp_ctx->enable_gc);
bh_assert(aot_value->type == VALUE_TYPE_GC_REF);
if (frame->comp_ctx->pointer_size == sizeof(uint64)) {
push_64bit(frame, aot_value);
(frame->sp - 1)->ref = (frame->sp - 2)->ref = 1;
}
else {
push_32bit(frame, aot_value);
(frame->sp - 1)->ref = 1;
}
}
#endif
/* Clear value slots except ref and committed_ref */
static inline void
clear_frame_value_slots(AOTValueSlot *slots, uint32 n)
{
uint32 i;
for (i = 0; i < n; i++) {
slots[i].value = 0;
slots[i].type = 0;
slots[i].dirty = 0;
}
}
static inline void
pop_i32(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_I32
|| (frame->sp - 1)->type == VALUE_TYPE_I1);
frame->sp--;
clear_frame_value_slots(frame->sp, 1);
}
static inline void
pop_i64(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 2);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_I64
&& (frame->sp - 2)->type == VALUE_TYPE_I64);
frame->sp -= 2;
clear_frame_value_slots(frame->sp, 2);
}
static inline void
pop_f32(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_F32);
frame->sp--;
clear_frame_value_slots(frame->sp, 1);
}
static inline void
pop_f64(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 2);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_F64
&& (frame->sp - 2)->type == VALUE_TYPE_F64);
frame->sp -= 2;
clear_frame_value_slots(frame->sp, 2);
}
static inline void
pop_v128(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 4);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_V128
&& (frame->sp - 2)->type == VALUE_TYPE_V128
&& (frame->sp - 3)->type == VALUE_TYPE_V128
&& (frame->sp - 4)->type == VALUE_TYPE_V128);
frame->sp -= 4;
clear_frame_value_slots(frame->sp, 4);
}
static inline void
pop_ref(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_FUNCREF
|| (frame->sp - 1)->type == VALUE_TYPE_EXTERNREF);
frame->sp -= 1;
clear_frame_value_slots(frame->sp, 1);
}
#if WASM_ENABLE_GC != 0
static inline void
pop_gc_ref(AOTCompFrame *frame)
{
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_GC_REF);
frame->sp -= 1;
clear_frame_value_slots(frame->sp, 1);
frame->sp->ref = 0;
if (frame->comp_ctx->pointer_size == sizeof(uint64)) {
bh_assert(frame->sp - frame->lp >= 1);
bh_assert((frame->sp - 1)->type == VALUE_TYPE_GC_REF);
frame->sp -= 1;
clear_frame_value_slots(frame->sp, 1);
frame->sp->ref = 0;
}
}
#endif
static inline void
set_local_i32(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_I32;
frame->lp[n].dirty = 1;
}
static inline void
set_local_i64(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_I64;
frame->lp[n].dirty = 1;
frame->lp[n + 1].value = value;
frame->lp[n + 1].type = VALUE_TYPE_I64;
frame->lp[n + 1].dirty = 1;
}
static inline void
set_local_f32(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_F32;
frame->lp[n].dirty = 1;
}
static inline void
set_local_f64(AOTCompFrame *frame, int n, LLVMValueRef value)
{
frame->lp[n].value = value;
frame->lp[n].type = VALUE_TYPE_F64;
frame->lp[n].dirty = 1;
frame->lp[n + 1].value = value;
frame->lp[n + 1].type = VALUE_TYPE_F64;
frame->lp[n + 1].dirty = 1;
}
static inline void
set_local_v128(AOTCompFrame *frame, int n, LLVMValueRef value)
{
uint32 i;
for (i = 0; i < 4; i++) {
frame->lp[n + i].value = value;
frame->lp[n + i].type = VALUE_TYPE_V128;
frame->lp[n + i].dirty = 1;
}
}
static inline void
set_local_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
{
bh_assert(frame->comp_ctx->enable_ref_types);
frame->lp[n].value = value;
frame->lp[n].type = ref_type;
frame->lp[n].dirty = 1;
}
#if WASM_ENABLE_GC != 0
static inline void
set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type)
{
bh_assert(frame->comp_ctx->enable_gc);
bh_assert(ref_type == VALUE_TYPE_GC_REF);
frame->lp[n].value = value;
frame->lp[n].type = ref_type;
frame->lp[n].dirty = 1;
frame->lp[n].ref = 1;
if (frame->comp_ctx->pointer_size == sizeof(uint64)) {
frame->lp[n + 1].value = value;
frame->lp[n + 1].type = ref_type;
frame->lp[n + 1].dirty = 1;
frame->lp[n + 1].ref = 1;
}
}
#endif
#define CHECK_STACK() \
do { \
if (!func_ctx->block_stack.block_list_end) { \
@ -119,37 +458,61 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
} \
} while (0)
#if WASM_ENABLE_GC != 0
#define GET_GC_REF_FROM_STACK(llvm_value) \
do { \
AOTValue *aot_value; \
CHECK_STACK(); \
aot_value = \
func_ctx->block_stack.block_list_end->value_stack.value_list_end; \
if (aot_value->type != VALUE_TYPE_GC_REF) { \
aot_set_last_error("WASM stack data type is not reference"); \
goto fail; \
} \
llvm_value = aot_value->value; \
} while (0)
#endif
#define POP(llvm_value, value_type) \
do { \
AOTValue *aot_value; \
uint8 val_type_to_pop = value_type; \
CHECK_STACK(); \
aot_value = aot_value_stack_pop( \
&func_ctx->block_stack.block_list_end->value_stack); \
if (!check_type_compatible(aot_value->type, value_type)) { \
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); \
if (comp_ctx->enable_gc && aot_is_type_gc_reftype(value_type)) \
val_type_to_pop = VALUE_TYPE_GC_REF; \
if (!check_type_compatible(comp_ctx, aot_value->type, \
val_type_to_pop)) { \
aot_set_last_error("invalid WASM stack data type."); \
wasm_runtime_free(aot_value); \
goto fail; \
} \
if (aot_value->type == value_type) \
if (aot_value->type == val_type_to_pop) \
llvm_value = aot_value->value; \
else { \
if (aot_value->type == VALUE_TYPE_I1) { \
if (!(llvm_value = \
LLVMBuildZExt(comp_ctx->builder, aot_value->value, \
I32_TYPE, "i1toi32"))) { \
aot_set_last_error("invalid WASM stack " \
"data type."); \
aot_set_last_error("invalid WASM stack data type."); \
wasm_runtime_free(aot_value); \
goto fail; \
} \
} \
else { \
bh_assert(aot_value->type == VALUE_TYPE_I32 \
|| aot_value->type == VALUE_TYPE_FUNCREF \
|| aot_value->type == VALUE_TYPE_EXTERNREF); \
bh_assert(value_type == VALUE_TYPE_I32 \
|| value_type == VALUE_TYPE_FUNCREF \
|| value_type == VALUE_TYPE_EXTERNREF); \
bh_assert( \
aot_value->type == VALUE_TYPE_I32 \
|| (comp_ctx->enable_ref_types \
&& (aot_value->type == VALUE_TYPE_FUNCREF \
|| aot_value->type == VALUE_TYPE_EXTERNREF))); \
bh_assert( \
val_type_to_pop == VALUE_TYPE_I32 \
|| (comp_ctx->enable_ref_types \
&& (val_type_to_pop == VALUE_TYPE_FUNCREF \
|| val_type_to_pop == VALUE_TYPE_EXTERNREF))); \
llvm_value = aot_value->value; \
} \
} \
@ -163,13 +526,14 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define POP_V128(v) POP(v, VALUE_TYPE_V128)
#define POP_FUNCREF(v) POP(v, VALUE_TYPE_FUNCREF)
#define POP_EXTERNREF(v) POP(v, VALUE_TYPE_EXTERNREF)
#define POP_GC_REF(v) POP(v, VALUE_TYPE_GC_REF)
#define POP_COND(llvm_value) \
do { \
AOTValue *aot_value; \
CHECK_STACK(); \
aot_value = aot_value_stack_pop( \
&func_ctx->block_stack.block_list_end->value_stack); \
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack); \
if (aot_value->type != VALUE_TYPE_I1 \
&& aot_value->type != VALUE_TYPE_I32) { \
aot_set_last_error("invalid WASM stack data type."); \
@ -190,23 +554,27 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
wasm_runtime_free(aot_value); \
} while (0)
#define PUSH(llvm_value, value_type) \
do { \
AOTValue *aot_value; \
if (!func_ctx->block_stack.block_list_end) { \
aot_set_last_error("WASM block stack underflow."); \
goto fail; \
} \
aot_value = wasm_runtime_malloc(sizeof(AOTValue)); \
if (!aot_value) { \
aot_set_last_error("allocate memory failed."); \
goto fail; \
} \
memset(aot_value, 0, sizeof(AOTValue)); \
aot_value->type = value_type; \
aot_value->value = llvm_value; \
aot_value_stack_push( \
&func_ctx->block_stack.block_list_end->value_stack, aot_value); \
#define PUSH(llvm_value, value_type) \
do { \
AOTValue *aot_value; \
if (!func_ctx->block_stack.block_list_end) { \
aot_set_last_error("WASM block stack underflow."); \
goto fail; \
} \
aot_value = wasm_runtime_malloc(sizeof(AOTValue)); \
if (!aot_value) { \
aot_set_last_error("allocate memory failed."); \
goto fail; \
} \
memset(aot_value, 0, sizeof(AOTValue)); \
if (comp_ctx->enable_gc && aot_is_type_gc_reftype(value_type)) \
aot_value->type = VALUE_TYPE_GC_REF; \
else \
aot_value->type = value_type; \
aot_value->value = llvm_value; \
aot_value_stack_push( \
comp_ctx, &func_ctx->block_stack.block_list_end->value_stack, \
aot_value); \
} while (0)
#define PUSH_I32(v) PUSH(v, VALUE_TYPE_I32)
@ -217,9 +585,10 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define PUSH_COND(v) PUSH(v, VALUE_TYPE_I1)
#define PUSH_FUNCREF(v) PUSH(v, VALUE_TYPE_FUNCREF)
#define PUSH_EXTERNREF(v) PUSH(v, VALUE_TYPE_EXTERNREF)
#define PUSH_GC_REF(v) PUSH(v, VALUE_TYPE_GC_REF)
#define TO_LLVM_TYPE(wasm_type) \
wasm_type_to_llvm_type(&comp_ctx->basic_types, wasm_type)
wasm_type_to_llvm_type(comp_ctx, &comp_ctx->basic_types, wasm_type)
#define I32_TYPE comp_ctx->basic_types.int32_type
#define I64_TYPE comp_ctx->basic_types.int64_type
@ -229,15 +598,19 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define INT1_TYPE comp_ctx->basic_types.int1_type
#define INT8_TYPE comp_ctx->basic_types.int8_type
#define INT16_TYPE comp_ctx->basic_types.int16_type
#define INTPTR_T_TYPE comp_ctx->basic_types.intptr_t_type
#define MD_TYPE comp_ctx->basic_types.meta_data_type
#define INT8_PTR_TYPE comp_ctx->basic_types.int8_ptr_type
#define INT16_PTR_TYPE comp_ctx->basic_types.int16_ptr_type
#define INT32_PTR_TYPE comp_ctx->basic_types.int32_ptr_type
#define INT64_PTR_TYPE comp_ctx->basic_types.int64_ptr_type
#define INTPTR_T_PTR_TYPE comp_ctx->basic_types.intptr_t_ptr_type
#define F32_PTR_TYPE comp_ctx->basic_types.float32_ptr_type
#define F64_PTR_TYPE comp_ctx->basic_types.float64_ptr_type
#define FUNC_REF_TYPE comp_ctx->basic_types.funcref_type
#define EXTERN_REF_TYPE comp_ctx->basic_types.externref_type
#define GC_REF_TYPE comp_ctx->basic_types.gc_ref_type
#define GC_REF_PTR_TYPE comp_ctx->basic_types.gc_ref_ptr_type
#define INT8_PTR_TYPE_GS comp_ctx->basic_types.int8_ptr_type_gs
#define INT16_PTR_TYPE_GS comp_ctx->basic_types.int16_ptr_type_gs
@ -253,7 +626,10 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define I8_CONST(v) LLVMConstInt(INT8_TYPE, v, true)
#define LLVM_CONST(name) (comp_ctx->llvm_consts.name)
#define I1_ZERO LLVM_CONST(i1_zero)
#define I1_ONE LLVM_CONST(i1_one)
#define I8_ZERO LLVM_CONST(i8_zero)
#define I8_ONE LLVM_CONST(i8_one)
#define I32_ZERO LLVM_CONST(i32_zero)
#define I64_ZERO LLVM_CONST(i64_zero)
#define F32_ZERO LLVM_CONST(f32_zero)
@ -267,6 +643,9 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define I32_SEVEN LLVM_CONST(i32_seven)
#define I32_EIGHT LLVM_CONST(i32_eight)
#define I32_NINE LLVM_CONST(i32_nine)
#define I32_TEN LLVM_CONST(i32_ten)
#define I32_ELEVEN LLVM_CONST(i32_eleven)
#define I32_TWELVE LLVM_CONST(i32_twelve)
#define I32_NEG_ONE LLVM_CONST(i32_neg_one)
#define I64_NEG_ONE LLVM_CONST(i64_neg_one)
#define I32_MIN LLVM_CONST(i32_min)
@ -276,6 +655,8 @@ check_type_compatible(uint8 src_type, uint8 dst_type)
#define I64_63 LLVM_CONST(i64_63)
#define I64_64 LLVM_CONST(i64_64)
#define REF_NULL I32_NEG_ONE
#define GC_REF_NULL LLVM_CONST(gc_ref_null)
#define I8_PTR_NULL LLVM_CONST(i8_ptr_null)
#define V128_TYPE comp_ctx->basic_types.v128_type
#define V128_PTR_TYPE comp_ctx->basic_types.v128_ptr_type