Support muti-module for AOT mode (#2482)

Support muti-module for AOT mode, currently only implement the multi-module's function import feature for AOT, the memory/table/ global import are not implemented yet. And update wamr-test-suites scripts, multi-module sample and some CIs accordingly.
2023-09-28 08:56:11 +08:00
parent fff0e2ad1c
commit 79b27c1934
20 changed files with 1018 additions and 343 deletions
--- a/core/iwasm/common/wasm_runtime_common.c
+++ b/core/iwasm/common/wasm_runtime_common.c
@ -125,6 +125,36 @@ runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst,
    return mem;
 }

+#if WASM_ENABLE_MULTI_MODULE != 0
+/*
+    TODO:
+    Let loader_malloc be a general API both for AOT and WASM.
+*/
+
+#define loader_malloc(size, error_buf, error_buf_size) \
+    runtime_malloc(size, NULL, error_buf, error_buf_size)
+static void
+set_error_buf_v(const WASMModuleCommon *module, char *error_buf,
+                uint32 error_buf_size, const char *format, ...)
+{
+    va_list args;
+    char buf[128];
+    if (error_buf != NULL) {
+        va_start(args, format);
+        vsnprintf(buf, sizeof(buf), format, args);
+        va_end(args);
+        if (module->module_type == Wasm_Module_AoT) {
+            snprintf(error_buf, error_buf_size, "AOT module load failed: %s",
+                     buf);
+        }
+        else if (module->module_type == Wasm_Module_Bytecode) {
+            snprintf(error_buf, error_buf_size, "WASM module load failed: %s",
+                     buf);
+        }
+    }
+}
+#endif
+
 #if WASM_ENABLE_FAST_JIT != 0
 static JitCompOptions jit_options = { 0 };
 #endif
@ -1198,27 +1228,34 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf,
    if (get_package_type(buf, size) == Wasm_Module_Bytecode) {
 #if WASM_ENABLE_INTERP != 0
        module_common =
-            (WASMModuleCommon *)wasm_load(buf, size, error_buf, error_buf_size);
-        return register_module_with_null_name(module_common, error_buf,
-                                              error_buf_size);
+            (WASMModuleCommon *)wasm_load(buf, size,
+#if WASM_ENABLE_MULTI_MODULE != 0
+                                          true,
+#endif
+                                          error_buf, error_buf_size);
 #endif
    }
    else if (get_package_type(buf, size) == Wasm_Module_AoT) {
 #if WASM_ENABLE_AOT != 0
        module_common = (WASMModuleCommon *)aot_load_from_aot_file(
            buf, size, error_buf, error_buf_size);
-        return register_module_with_null_name(module_common, error_buf,
-                                              error_buf_size);
 #endif
    }
-
-    if (size < 4)
-        set_error_buf(error_buf, error_buf_size,
-                      "WASM module load failed: unexpected end");
-    else
-        set_error_buf(error_buf, error_buf_size,
-                      "WASM module load failed: magic header not detected");
-    return NULL;
+    else {
+        if (size < 4)
+            set_error_buf(error_buf, error_buf_size,
+                          "WASM module load failed: unexpected end");
+        else
+            set_error_buf(error_buf, error_buf_size,
+                          "WASM module load failed: magic header not detected");
+        return NULL;
+    }
+    if (!module_common) {
+        LOG_DEBUG("WASM module load failed");
+        return NULL;
+    }
+    return register_module_with_null_name(module_common, error_buf,
+                                          error_buf_size);
 }

 WASMModuleCommon *
@ -1231,6 +1268,10 @@ wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot,
 #if WASM_ENABLE_INTERP != 0
        module_common = (WASMModuleCommon *)wasm_load_from_sections(
            section_list, error_buf, error_buf_size);
+        if (!module_common) {
+            LOG_DEBUG("WASM module load failed from sections");
+            return NULL;
+        }
        return register_module_with_null_name(module_common, error_buf,
                                              error_buf_size);
 #endif
@ -1239,6 +1280,10 @@ wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot,
 #if WASM_ENABLE_AOT != 0
        module_common = (WASMModuleCommon *)aot_load_from_sections(
            section_list, error_buf, error_buf_size);
+        if (!module_common) {
+            LOG_DEBUG("WASM module load failed from sections");
+            return NULL;
+        }
        return register_module_with_null_name(module_common, error_buf,
                                              error_buf_size);
 #endif
@ -5663,6 +5708,314 @@ wasm_runtime_is_import_global_linked(const char *module_name,
 #endif
 }

+#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0
+WASMExport *
+loader_find_export(const WASMModuleCommon *module, const char *module_name,
+                   const char *field_name, uint8 export_kind, char *error_buf,
+                   uint32 error_buf_size)
+{
+    WASMExport *exports = NULL, *result = NULL, *export;
+    uint32 export_count = 0, i;
+#if WASM_ENABLE_AOT != 0
+    if (module->module_type == Wasm_Module_AoT) {
+        AOTModule *aot_module = (AOTModule *)module;
+        exports = (WASMExport *)aot_module->exports;
+        export_count = aot_module->export_count;
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (module->module_type == Wasm_Module_Bytecode) {
+        WASMModule *wasm_module = (WASMModule *)module;
+        exports = wasm_module->exports;
+        export_count = wasm_module->export_count;
+    }
+#endif
+    for (i = 0, export = exports; i < export_count; ++i, ++export) {
+        if (export->kind == export_kind && !strcmp(field_name, export->name)) {
+            result = export;
+            goto exit;
+        }
+    }
+    if (i == export_count) {
+        LOG_DEBUG("can not find an export %d named %s in the module %s",
+                  export_kind, field_name, module_name);
+        set_error_buf(error_buf, error_buf_size,
+                      "unknown import or incompatible import type");
+    }
+exit:
+    return result;
+}
+#endif
+
+#if WASM_ENABLE_MULTI_MODULE != 0
+WASMModuleCommon *
+wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module,
+                               const char *sub_module_name)
+{
+    WASMRegisteredModule *node = NULL;
+#if WASM_ENABLE_AOT != 0
+    if (parent_module->module_type == Wasm_Module_AoT) {
+        node = bh_list_first_elem(
+            ((AOTModule *)parent_module)->import_module_list);
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (parent_module->module_type == Wasm_Module_Bytecode) {
+        node = bh_list_first_elem(
+            ((WASMModule *)parent_module)->import_module_list);
+    }
+#endif
+    while (node && strcmp(sub_module_name, node->module_name)) {
+        node = bh_list_elem_next(node);
+    }
+    return node ? node->module : NULL;
+}
+
+bool
+wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module,
+                                 const char *sub_module_name,
+                                 WASMModuleCommon *sub_module)
+{
+    /* register sub_module into its parent sub module list */
+    WASMRegisteredModule *node = NULL;
+    bh_list_status ret;
+
+    if (wasm_runtime_search_sub_module(parent_module, sub_module_name)) {
+        LOG_DEBUG("%s has been registered in its parent", sub_module_name);
+        return true;
+    }
+
+    node = loader_malloc(sizeof(WASMRegisteredModule), NULL, 0);
+    if (!node) {
+        return false;
+    }
+
+    node->module_name = sub_module_name;
+    node->module = sub_module;
+#if WASM_ENABLE_AOT != 0
+    if (parent_module->module_type == Wasm_Module_AoT) {
+        ret = bh_list_insert(((AOTModule *)parent_module)->import_module_list,
+                             node);
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (parent_module->module_type == Wasm_Module_Bytecode) {
+        ret = bh_list_insert(((WASMModule *)parent_module)->import_module_list,
+                             node);
+    }
+#endif
+    bh_assert(BH_LIST_SUCCESS == ret);
+    (void)ret;
+    return true;
+}
+
+WASMModuleCommon *
+wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module,
+                                  const char *sub_module_name, char *error_buf,
+                                  uint32 error_buf_size)
+{
+    WASMModuleCommon *sub_module = NULL;
+    bool ret = false;
+    uint8 *buffer = NULL;
+    uint32 buffer_size = 0;
+
+    /* check the registered module list of the parent */
+    sub_module = wasm_runtime_search_sub_module(parent_module, sub_module_name);
+    if (sub_module) {
+        LOG_DEBUG("%s has been loaded before", sub_module_name);
+        return sub_module;
+    }
+
+    /* check the global registered module list */
+    sub_module = wasm_runtime_find_module_registered(sub_module_name);
+    if (sub_module) {
+        LOG_DEBUG("%s has been loaded", sub_module_name);
+        goto wasm_runtime_register_sub_module;
+    }
+    LOG_VERBOSE("loading %s", sub_module_name);
+    if (!reader) {
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "no sub module reader to load %s", sub_module_name);
+        return NULL;
+    }
+    /* start to maintain a loading module list */
+    ret = wasm_runtime_is_loading_module(sub_module_name);
+    if (ret) {
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "found circular dependency on %s", sub_module_name);
+        return NULL;
+    }
+    ret = wasm_runtime_add_loading_module(sub_module_name, error_buf,
+                                          error_buf_size);
+    if (!ret) {
+        LOG_DEBUG("can not add %s into loading module list\n", sub_module_name);
+        return NULL;
+    }
+
+    ret = reader(parent_module->module_type, sub_module_name, &buffer,
+                 &buffer_size);
+    if (!ret) {
+        LOG_DEBUG("read the file of %s failed", sub_module_name);
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "unknown import", sub_module_name);
+        goto delete_loading_module;
+    }
+    if (get_package_type(buffer, buffer_size) != parent_module->module_type) {
+        LOG_DEBUG("moudle %s type error", sub_module_name);
+        goto delete_loading_module;
+    }
+    if (get_package_type(buffer, buffer_size) == Wasm_Module_Bytecode) {
+#if WASM_ENABLE_INTERP != 0
+        sub_module = (WASMModuleCommon *)wasm_load(buffer, buffer_size, false,
+                                                   error_buf, error_buf_size);
+#endif
+    }
+    else if (get_package_type(buffer, buffer_size) == Wasm_Module_AoT) {
+#if WASM_ENABLE_AOT != 0
+        sub_module = (WASMModuleCommon *)aot_load_from_aot_file(
+            buffer, buffer_size, error_buf, error_buf_size);
+#endif
+    }
+    if (!sub_module) {
+        LOG_DEBUG("error: can not load the sub_module %s", sub_module_name);
+        /* others will be destroyed in runtime_destroy() */
+        goto destroy_file_buffer;
+    }
+    wasm_runtime_delete_loading_module(sub_module_name);
+    /* register on a global list */
+    ret = wasm_runtime_register_module_internal(
+        sub_module_name, (WASMModuleCommon *)sub_module, buffer, buffer_size,
+        error_buf, error_buf_size);
+    if (!ret) {
+        LOG_DEBUG("error: can not register module %s globally\n",
+                  sub_module_name);
+        /* others will be unloaded in runtime_destroy() */
+        goto unload_module;
+    }
+
+    /* register into its parent list */
+wasm_runtime_register_sub_module:
+    ret = wasm_runtime_register_sub_module(parent_module, sub_module_name,
+                                           sub_module);
+    if (!ret) {
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "failed to register sub module %s", sub_module_name);
+        /* since it is in the global module list, no need to
+         * unload the module. the runtime_destroy() will do it
+         */
+        return NULL;
+    }
+
+    return sub_module;
+
+unload_module:
+    wasm_runtime_unload(sub_module);
+
+destroy_file_buffer:
+    if (destroyer) {
+        destroyer(buffer, buffer_size);
+    }
+    else {
+        LOG_WARNING("need to release the reading buffer of %s manually",
+                    sub_module_name);
+    }
+
+delete_loading_module:
+    wasm_runtime_delete_loading_module(sub_module_name);
+    return NULL;
+}
+
+bool
+wasm_runtime_sub_module_instantiate(WASMModuleCommon *module,
+                                    WASMModuleInstanceCommon *module_inst,
+                                    uint32 stack_size, uint32 heap_size,
+                                    char *error_buf, uint32 error_buf_size)
+{
+    bh_list *sub_module_inst_list = NULL;
+    WASMRegisteredModule *sub_module_list_node = NULL;
+
+#if WASM_ENABLE_AOT != 0
+    if (module->module_type == Wasm_Module_AoT) {
+        sub_module_inst_list =
+            ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e)
+                ->sub_module_inst_list;
+        sub_module_list_node =
+            bh_list_first_elem(((AOTModule *)module)->import_module_list);
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (module->module_type == Wasm_Module_Bytecode) {
+        sub_module_inst_list =
+            ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e)
+                ->sub_module_inst_list;
+        sub_module_list_node =
+            bh_list_first_elem(((WASMModule *)module)->import_module_list);
+    }
+#endif
+    while (sub_module_list_node) {
+        WASMSubModInstNode *sub_module_inst_list_node = NULL;
+        WASMModuleCommon *sub_module = sub_module_list_node->module;
+        WASMModuleInstanceCommon *sub_module_inst = NULL;
+        sub_module_inst = wasm_runtime_instantiate_internal(
+            sub_module, NULL, NULL, stack_size, heap_size, error_buf,
+            error_buf_size);
+        if (!sub_module_inst) {
+            LOG_DEBUG("instantiate %s failed",
+                      sub_module_list_node->module_name);
+            return false;
+        }
+        sub_module_inst_list_node = loader_malloc(sizeof(WASMSubModInstNode),
+                                                  error_buf, error_buf_size);
+        if (!sub_module_inst_list_node) {
+            LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ:%d",
+                      sizeof(WASMSubModInstNode));
+            if (sub_module_inst)
+                wasm_runtime_deinstantiate_internal(sub_module_inst, false);
+            return false;
+        }
+        sub_module_inst_list_node->module_inst =
+            (WASMModuleInstance *)sub_module_inst;
+        sub_module_inst_list_node->module_name =
+            sub_module_list_node->module_name;
+        bh_list_status ret =
+            bh_list_insert(sub_module_inst_list, sub_module_inst_list_node);
+        bh_assert(BH_LIST_SUCCESS == ret);
+        (void)ret;
+        sub_module_list_node = bh_list_elem_next(sub_module_list_node);
+    }
+
+    return true;
+}
+
+void
+wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst)
+{
+    bh_list *list = NULL;
+#if WASM_ENABLE_AOT != 0
+    if (module_inst->module_type == Wasm_Module_AoT) {
+        list = ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e)
+                   ->sub_module_inst_list;
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (module_inst->module_type == Wasm_Module_Bytecode) {
+        list =
+            ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e)
+                ->sub_module_inst_list;
+    }
+#endif
+
+    WASMSubModInstNode *node = bh_list_first_elem(list);
+    while (node) {
+        WASMSubModInstNode *next_node = bh_list_elem_next(node);
+        bh_list_remove(list, node);
+        wasm_runtime_deinstantiate_internal(
+            (WASMModuleInstanceCommon *)node->module_inst, false);
+        wasm_runtime_free(node);
+        node = next_node;
+    }
+}
+#endif /* end of WASM_ENABLE_MULTI_MODULE */
 #if WASM_ENABLE_MODULE_INST_CONTEXT != 0
 void *
 wasm_runtime_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst,