christoph/wamr
1
Fork 0
Code Releases Activity

Implement GC (Garbage Collection) feature for interpreter, AOT and LLVM-JIT (#3125)

Browse Source 
Implement the GC (Garbage Collection) feature for interpreter mode,
AOT mode and LLVM-JIT mode, and support most features of the latest
spec proposal, and also enable the stringref feature.

Use `cmake -DWAMR_BUILD_GC=1/0` to enable/disable the feature,
and `wamrc --enable-gc` to generate the AOT file with GC supported.

And update the AOT file version from 2 to 3 since there are many AOT
ABI breaks, including the changes of AOT file format, the changes of
AOT module/memory instance layouts, the AOT runtime APIs for the
AOT code to invoke and so on.
This commit is contained in:
Wenyong Huang
2024-02-06 20:47:11 +08:00
committed by GitHub
parent 5931aaacbe
commit 16a4d71b34
98 changed files with 33469 additions and 3159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

399
core/iwasm/compilation/aot.c
View File

@ -114,9 +114,9 @@ aot_create_table_init_data_list(const WASMModule *module)
/* Create each table data segment */
for (i = 0; i < module->table_seg_count; i++) {
size =
offsetof(AOTTableInitData, func_indexes)
+ sizeof(uint32) * (uint64)module->table_segments[i].function_count;
size = offsetof(AOTTableInitData, init_values)
+ sizeof(InitializerExpression)
* (uint64)module->table_segments[i].value_count;
if (size >= UINT32_MAX
|| !(data_list[i] = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
@ -124,8 +124,7 @@ aot_create_table_init_data_list(const WASMModule *module)
}
data_list[i]->offset = module->table_segments[i].base_offset;
data_list[i]->func_index_count =
module->table_segments[i].function_count;
data_list[i]->value_count = module->table_segments[i].value_count;
data_list[i]->mode = module->table_segments[i].mode;
data_list[i]->elem_type = module->table_segments[i].elem_type;
/* runtime control it */
@ -133,12 +132,16 @@ aot_create_table_init_data_list(const WASMModule *module)
bh_memcpy_s(&data_list[i]->offset, sizeof(AOTInitExpr),
&module->table_segments[i].base_offset,
sizeof(AOTInitExpr));
data_list[i]->func_index_count =
module->table_segments[i].function_count;
bh_memcpy_s(data_list[i]->func_indexes,
sizeof(uint32) * module->table_segments[i].function_count,
module->table_segments[i].func_indexes,
sizeof(uint32) * module->table_segments[i].function_count);
data_list[i]->value_count = module->table_segments[i].value_count;
#if WASM_ENABLE_GC != 0
data_list[i]->elem_ref_type = module->table_segments[i].elem_ref_type;
#endif
bh_memcpy_s(data_list[i]->init_values,
sizeof(InitializerExpression)
* module->table_segments[i].value_count,
module->table_segments[i].init_values,
sizeof(InitializerExpression)
* module->table_segments[i].value_count);
}
return data_list;
@ -148,13 +151,60 @@ fail:
return NULL;
}
static void
get_value_type_size(uint8 value_type, bool gc_enabled, uint32 *p_size_64bit,
uint32 *p_size_32bit)
{
uint32 size_64bit = 0, size_32bit = 0;
if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_F32)
size_64bit = size_32bit = sizeof(int32);
else if (value_type == VALUE_TYPE_I64 || value_type == VALUE_TYPE_F64)
size_64bit = size_32bit = sizeof(int64);
else if (value_type == VALUE_TYPE_V128)
size_64bit = size_32bit = sizeof(int64) * 2;
else if (!gc_enabled
&& (value_type == VALUE_TYPE_FUNCREF
|| value_type == VALUE_TYPE_EXTERNREF))
size_64bit = size_32bit = sizeof(int32);
else if (gc_enabled
&& ((value_type >= (uint8)REF_TYPE_ARRAYREF
&& value_type <= (uint8)REF_TYPE_NULLFUNCREF)
|| (value_type >= (uint8)REF_TYPE_HT_NULLABLE
&& value_type <= (uint8)REF_TYPE_HT_NON_NULLABLE)
#if WASM_ENABLE_STRINGREF != 0
|| (value_type >= (uint8)REF_TYPE_STRINGVIEWWTF8
&& value_type <= (uint8)REF_TYPE_STRINGREF)
|| (value_type >= (uint8)REF_TYPE_STRINGVIEWITER
&& value_type <= (uint8)REF_TYPE_STRINGVIEWWTF16)
#endif
)) {
size_64bit = sizeof(uint64);
size_32bit = sizeof(uint32);
}
else if (gc_enabled && value_type == PACKED_TYPE_I8) {
size_64bit = size_32bit = sizeof(int8);
}
else if (gc_enabled && value_type == PACKED_TYPE_I16) {
size_64bit = size_32bit = sizeof(int16);
}
else {
bh_assert(0);
}
*p_size_64bit = size_64bit;
*p_size_32bit = size_32bit;
}
static AOTImportGlobal *
aot_create_import_globals(const WASMModule *module,
uint32 *p_import_global_data_size)
aot_create_import_globals(const WASMModule *module, bool gc_enabled,
uint32 *p_import_global_data_size_64bit,
uint32 *p_import_global_data_size_32bit)
{
AOTImportGlobal *import_globals;
uint64 size;
uint32 i, data_offset = 0;
uint32 i, data_offset_64bit = 0, data_offset_32bit = 0;
uint32 value_size_64bit, value_size_32bit;
/* Allocate memory */
size = sizeof(AOTImportGlobal) * (uint64)module->import_global_count;
@ -175,23 +225,37 @@ aot_create_import_globals(const WASMModule *module,
import_globals[i].is_mutable = import_global->is_mutable;
import_globals[i].global_data_linked =
import_global->global_data_linked;
import_globals[i].size = wasm_value_type_size(import_global->type);
/* Calculate data offset */
import_globals[i].data_offset = data_offset;
data_offset += wasm_value_type_size(import_global->type);
import_globals[i].data_offset_64bit = data_offset_64bit;
import_globals[i].data_offset_32bit = data_offset_32bit;
get_value_type_size(import_global->type, gc_enabled, &value_size_64bit,
&value_size_32bit);
import_globals[i].size_64bit = value_size_64bit;
import_globals[i].size_32bit = value_size_32bit;
data_offset_64bit += value_size_64bit;
data_offset_32bit += value_size_32bit;
}
*p_import_global_data_size = data_offset;
*p_import_global_data_size_64bit = data_offset_64bit;
*p_import_global_data_size_32bit = data_offset_32bit;
return import_globals;
}
static AOTGlobal *
aot_create_globals(const WASMModule *module, uint32 global_data_start_offset,
uint32 *p_global_data_size)
aot_create_globals(const WASMModule *module, bool gc_enabled,
uint32 global_data_start_offset_64bit,
uint32 global_data_start_offset_32bit,
uint32 *p_global_data_size_64bit,
uint32 *p_global_data_size_32bit)
{
AOTGlobal *globals;
uint64 size;
uint32 i, data_offset = global_data_start_offset;
uint32 i;
uint32 data_offset_64bit = global_data_start_offset_64bit;
uint32 data_offset_32bit = global_data_start_offset_32bit;
uint32 value_size_64bit, value_size_32bit;
/* Allocate memory */
size = sizeof(AOTGlobal) * (uint64)module->global_count;
@ -207,65 +271,28 @@ aot_create_globals(const WASMModule *module, uint32 global_data_start_offset,
WASMGlobal *global = &module->globals[i];
globals[i].type = global->type;
globals[i].is_mutable = global->is_mutable;
globals[i].size = wasm_value_type_size(global->type);
memcpy(&globals[i].init_expr, &global->init_expr,
sizeof(global->init_expr));
/* Calculate data offset */
globals[i].data_offset = data_offset;
data_offset += wasm_value_type_size(global->type);
globals[i].data_offset_64bit = data_offset_64bit;
globals[i].data_offset_32bit = data_offset_32bit;
get_value_type_size(global->type, gc_enabled, &value_size_64bit,
&value_size_32bit);
globals[i].size_64bit = value_size_64bit;
globals[i].size_32bit = value_size_32bit;
data_offset_64bit += value_size_64bit;
data_offset_32bit += value_size_32bit;
}
*p_global_data_size = data_offset - global_data_start_offset;
*p_global_data_size_64bit =
data_offset_64bit - global_data_start_offset_64bit;
*p_global_data_size_32bit =
data_offset_32bit - global_data_start_offset_32bit;
return globals;
}
static void
aot_destroy_func_types(AOTFuncType **func_types, uint32 count)
{
uint32 i;
for (i = 0; i < count; i++)
if (func_types[i])
wasm_runtime_free(func_types[i]);
wasm_runtime_free(func_types);
}
static AOTFuncType **
aot_create_func_types(const WASMModule *module)
{
AOTFuncType **func_types;
uint64 size;
uint32 i;
/* Allocate memory */
size = sizeof(AOTFuncType *) * (uint64)module->type_count;
if (size >= UINT32_MAX
|| !(func_types = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
return NULL;
}
memset(func_types, 0, size);
/* Create each function type */
for (i = 0; i < module->type_count; i++) {
size = offsetof(AOTFuncType, types)
+ (uint64)module->types[i]->param_count
+ (uint64)module->types[i]->result_count;
if (size >= UINT32_MAX
|| !(func_types[i] = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
memcpy(func_types[i], module->types[i], size);
}
return func_types;
fail:
aot_destroy_func_types(func_types, module->type_count);
return NULL;
}
static AOTImportFunc *
aot_create_import_funcs(const WASMModule *module)
{
@ -295,7 +322,7 @@ aot_create_import_funcs(const WASMModule *module)
import_funcs[i].call_conv_wasm_c_api = false;
/* Resolve function type index */
for (j = 0; j < module->type_count; j++)
if (import_func->func_type == module->types[j]) {
if (import_func->func_type == (WASMFuncType *)module->types[j]) {
import_funcs[i].func_type_index = j;
break;
}
@ -310,13 +337,16 @@ aot_destroy_funcs(AOTFunc **funcs, uint32 count)
uint32 i;
for (i = 0; i < count; i++)
if (funcs[i])
if (funcs[i]) {
if (funcs[i]->local_offsets)
wasm_runtime_free(funcs[i]->local_offsets);
wasm_runtime_free(funcs[i]);
}
wasm_runtime_free(funcs);
}
static AOTFunc **
aot_create_funcs(const WASMModule *module)
aot_create_funcs(const WASMModule *module, uint32 pointer_size)
{
AOTFunc **funcs;
uint64 size;
@ -334,28 +364,70 @@ aot_create_funcs(const WASMModule *module)
/* Create each function */
for (i = 0; i < module->function_count; i++) {
WASMFunction *func = module->functions[i];
AOTFuncType *func_type = NULL;
AOTFunc *aot_func = NULL;
uint64 total_size;
uint32 offset = 0;
size = sizeof(AOTFunc);
if (!(funcs[i] = wasm_runtime_malloc((uint32)size))) {
if (!(aot_func = funcs[i] = wasm_runtime_malloc((uint32)size))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
memset(aot_func, 0, sizeof(AOTFunc));
func_type = aot_func->func_type = func->func_type;
/* Resolve function type index */
for (j = 0; j < module->type_count; j++) {
if (func_type == (WASMFuncType *)module->types[j]) {
aot_func->func_type_index = j;
break;
}
}
total_size = sizeof(uint16)
* ((uint64)func_type->param_count + func->local_count);
if ((total_size > 0)
&& (total_size >= UINT32_MAX
|| !(aot_func->local_offsets =
wasm_runtime_malloc((uint32)total_size)))) {
aot_set_last_error("allocate memory failed.");
goto fail;
}
funcs[i]->func_type = func->func_type;
/* Resolve function type index */
for (j = 0; j < module->type_count; j++)
if (func->func_type == module->types[j]) {
funcs[i]->func_type_index = j;
break;
}
/* Resolve local variable info and code info */
funcs[i]->local_count = func->local_count;
funcs[i]->local_types = func->local_types;
funcs[i]->param_cell_num = func->param_cell_num;
funcs[i]->local_cell_num = func->local_cell_num;
funcs[i]->code = func->code;
funcs[i]->code_size = func->code_size;
aot_func->local_count = func->local_count;
aot_func->local_types_wp = func->local_types;
aot_func->code = func->code;
aot_func->code_size = func->code_size;
/* Resolve local offsets */
for (j = 0; j < func_type->param_count; j++) {
aot_func->local_offsets[j] = (uint16)offset;
offset += wasm_value_type_cell_num_internal(func_type->types[j],
pointer_size);
}
aot_func->param_cell_num = offset;
for (j = 0; j < func->local_count; j++) {
aot_func->local_offsets[func_type->param_count + j] =
(uint16)offset;
offset += wasm_value_type_cell_num_internal(func->local_types[j],
pointer_size);
}
aot_func->local_cell_num = offset - aot_func->param_cell_num;
aot_func->max_stack_cell_num = func->max_stack_cell_num;
/* We use max_stack_cell_num calculated from wasm_loader, which is based
* on wamrc's target type.
* - If the wamrc is compiled as 64bit, then the number is enough for
* both 32bit and 64bit runtime target
* - If the wamrc is compiled as 32bit, then we multiply this number by
* two to avoid overflow on 64bit runtime target */
if (sizeof(uintptr_t) == 4) {
aot_func->max_stack_cell_num *= 2;
}
}
return funcs;
@ -365,12 +437,94 @@ fail:
return NULL;
}
#if WASM_ENABLE_GC != 0
static void
calculate_struct_field_sizes_offsets(AOTCompData *comp_data, bool is_target_x86,
bool gc_enabled)
{
uint32 i;
for (i = 0; i < comp_data->type_count; i++) {
if (comp_data->types[i]->type_flag == WASM_TYPE_STRUCT) {
WASMStructType *struct_type = (WASMStructType *)comp_data->types[i];
WASMStructFieldType *fields = struct_type->fields;
uint32 field_offset_64bit, field_offset_32bit;
uint32 field_size_64bit, field_size_32bit, j;
/* offsetof(WASMStructObject, field_data) in 64-bit */
field_offset_64bit = sizeof(uint64);
/* offsetof(WASMStructObject, field_data) in 32-bit */
field_offset_32bit = sizeof(uint32);
for (j = 0; j < struct_type->field_count; j++) {
get_value_type_size(fields[j].field_type, gc_enabled,
&field_size_64bit, &field_size_32bit);
fields[j].field_size_64bit = field_size_64bit;
fields[j].field_size_32bit = field_size_32bit;
if (!is_target_x86) {
if (field_size_64bit == 2)
field_offset_64bit = align_uint(field_offset_64bit, 2);
else if (field_size_64bit >= 4)
field_offset_64bit = align_uint(field_offset_64bit, 4);
if (field_size_32bit == 2)
field_offset_32bit = align_uint(field_offset_32bit, 2);
else if (field_size_32bit >= 4)
field_offset_32bit = align_uint(field_offset_32bit, 4);
}
fields[j].field_offset_64bit = field_offset_64bit;
fields[j].field_offset_32bit = field_offset_32bit;
field_offset_64bit += field_size_64bit;
field_offset_32bit += field_size_32bit;
}
}
}
}
#endif
AOTCompData *
aot_create_comp_data(WASMModule *module)
aot_create_comp_data(WASMModule *module, const char *target_arch,
bool gc_enabled)
{
AOTCompData *comp_data;
uint32 import_global_data_size = 0, global_data_size = 0, i, j;
uint32 import_global_data_size_64bit = 0, global_data_size_64bit = 0, i, j;
uint32 import_global_data_size_32bit = 0, global_data_size_32bit = 0;
uint64 size;
bool is_64bit_target = false;
#if WASM_ENABLE_GC != 0
bool is_target_x86 = false;
#endif
#if WASM_ENABLE_GC != 0
if (!target_arch) {
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \
|| defined(BUILD_TARGET_X86_32)
is_target_x86 = true;
#endif
}
else {
if (!strncmp(target_arch, "x86_64", 6)
|| !strncmp(target_arch, "i386", 4))
is_target_x86 = true;
}
#endif
if (!target_arch) {
#if UINTPTR_MAX == UINT64_MAX
is_64bit_target = true;
#endif
}
else {
/* All 64bit targets contains "64" string in their target name */
if (strstr(target_arch, "64") != NULL) {
is_64bit_target = true;
}
}
/* Allocate memory */
if (!(comp_data = wasm_runtime_malloc(sizeof(AOTCompData)))) {
@ -457,6 +611,10 @@ aot_create_comp_data(WASMModule *module)
module->import_tables[i].u.table.init_size;
comp_data->tables[i].table_max_size =
module->import_tables[i].u.table.max_size;
#if WASM_ENABLE_GC != 0
comp_data->tables[i].elem_ref_type =
module->import_tables[i].u.table.elem_ref_type;
#endif
comp_data->tables[i].possible_grow =
module->import_tables[i].u.table.possible_grow;
}
@ -470,6 +628,16 @@ aot_create_comp_data(WASMModule *module)
module->tables[j].max_size;
comp_data->tables[i].possible_grow =
module->tables[j].possible_grow;
#if WASM_ENABLE_GC != 0
comp_data->tables[j].elem_ref_type =
module->tables[j].elem_ref_type;
/* Note: if the init_expr contains extra data for struct/array
* initialization information (init_expr.u.data), the pointer is
* copied.
* The pointers should still belong to wasm module, so DO NOT
* free the pointers copied to comp_data */
comp_data->tables[j].init_expr = module->tables[j].init_expr;
#endif
}
}
}
@ -484,24 +652,33 @@ aot_create_comp_data(WASMModule *module)
/* Create import globals */
comp_data->import_global_count = module->import_global_count;
if (comp_data->import_global_count > 0
&& !(comp_data->import_globals =
aot_create_import_globals(module, &import_global_data_size)))
&& !(comp_data->import_globals = aot_create_import_globals(
module, gc_enabled, &import_global_data_size_64bit,
&import_global_data_size_32bit)))
goto fail;
/* Create globals */
comp_data->global_count = module->global_count;
if (comp_data->global_count
&& !(comp_data->globals = aot_create_globals(
module, import_global_data_size, &global_data_size)))
module, gc_enabled, import_global_data_size_64bit,
import_global_data_size_32bit, &global_data_size_64bit,
&global_data_size_32bit)))
goto fail;
comp_data->global_data_size = import_global_data_size + global_data_size;
comp_data->global_data_size_64bit =
import_global_data_size_64bit + global_data_size_64bit;
comp_data->global_data_size_32bit =
import_global_data_size_32bit + global_data_size_32bit;
/* Create function types */
comp_data->func_type_count = module->type_count;
if (comp_data->func_type_count
&& !(comp_data->func_types = aot_create_func_types(module)))
goto fail;
/* Create types, they are checked by wasm loader */
comp_data->type_count = module->type_count;
comp_data->types = module->types;
#if WASM_ENABLE_GC != 0
/* Calculate the field sizes and field offsets for 64-bit and 32-bit
targets since they may vary in 32-bit target and 64-bit target */
calculate_struct_field_sizes_offsets(comp_data, is_target_x86, gc_enabled);
#endif
/* Create import functions */
comp_data->import_func_count = module->import_function_count;
@ -511,7 +688,9 @@ aot_create_comp_data(WASMModule *module)
/* Create functions */
comp_data->func_count = module->function_count;
if (comp_data->func_count && !(comp_data->funcs = aot_create_funcs(module)))
if (comp_data->func_count
&& !(comp_data->funcs =
aot_create_funcs(module, is_64bit_target ? 8 : 4)))
goto fail;
#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
@ -534,6 +713,12 @@ aot_create_comp_data(WASMModule *module)
comp_data->free_func_index = module->free_function;
comp_data->retain_func_index = module->retain_function;
#if WASM_ENABLE_STRINGREF != 0
comp_data->string_literal_count = module->string_literal_count;
comp_data->string_literal_ptrs_wp = module->string_literal_ptrs;
comp_data->string_literal_lengths_wp = module->string_literal_lengths;
#endif
comp_data->wasm_module = module;
return comp_data;
@ -576,10 +761,6 @@ aot_destroy_comp_data(AOTCompData *comp_data)
if (comp_data->globals)
wasm_runtime_free(comp_data->globals);
if (comp_data->func_types)
aot_destroy_func_types(comp_data->func_types,
comp_data->func_type_count);
if (comp_data->import_funcs)
wasm_runtime_free(comp_data->import_funcs);