christoph/wamr
1
Fork 0
Code Releases Activity

Support integrate 3rd-party toolchains into wamrc (#1237)

Browse Source 
Support integrating 3rd-party toolchain llc compiler or asm compiler
into wamrc by setting environment variable WAMRC_LLC_COMPILER
or WAMRC_ASM_COMPILER, wamrc will use these tools to generate
object file from LLVM IR firstly, and then refactor the object file into
aot file.
This commit is contained in:
Xu Jun
2022-06-20 13:13:41 +08:00
committed by GitHub
parent bc6eda2803
commit 53b775aa4b
8 changed files with 241 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/iwasm/compilation/aot.h
View File

@ -15,7 +15,9 @@
extern "C" {
#endif
#ifndef AOT_FUNC_PREFIX
#define AOT_FUNC_PREFIX "aot_func#"
#endif
typedef InitializerExpression AOTInitExpr;
typedef WASMType AOTFuncType;

107
core/iwasm/compilation/aot_compiler.c
View File

@ -2886,6 +2886,36 @@ aot_compile_wasm(AOTCompContext *comp_ctx)
return true;
}
#if !(defined(_WIN32) || defined(_WIN32_))
char *
aot_generate_tempfile_name(const char *prefix, const char *extension,
char *buffer, uint32 len)
{
int fd, name_len;
name_len = snprintf(buffer, len, "%s-XXXXXX", prefix);
if ((fd = mkstemp(buffer)) <= 0) {
aot_set_last_error("make temp file failed.");
return NULL;
}
/* close and remove temp file */
close(fd);
unlink(buffer);
/* Check if buffer length is enough */
/* name_len + '.' + extension + '\0' */
if (name_len + 1 + strlen(extension) + 1 > len) {
aot_set_last_error("temp file name too long.");
return NULL;
}
snprintf(buffer + name_len, len - name_len, ".%s", extension);
return buffer;
}
#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */
#if WASM_ENABLE_LAZY_JIT == 0
bool
aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name)
@ -2915,6 +2945,83 @@ aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name)
bh_print_time("Begin to emit object file");
#if !(defined(_WIN32) || defined(_WIN32_))
if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) {
char cmd[1024];
int ret;
if (comp_ctx->external_llc_compiler) {
char bc_file_name[64];
if (!aot_generate_tempfile_name("wamrc-bc", "bc", bc_file_name,
sizeof(bc_file_name))) {
return false;
}
if (LLVMWriteBitcodeToFile(comp_ctx->module, bc_file_name) != 0) {
aot_set_last_error("emit llvm bitcode file failed.");
return false;
}
snprintf(cmd, sizeof(cmd), "%s %s -o %s %s",
comp_ctx->external_llc_compiler,
comp_ctx->llc_compiler_flags ? comp_ctx->llc_compiler_flags
: "-O3 -c",
file_name, bc_file_name);
LOG_VERBOSE("invoking external LLC compiler:\n\t%s", cmd);
ret = system(cmd);
/* remove temp bitcode file */
unlink(bc_file_name);
if (ret != 0) {
aot_set_last_error("failed to compile LLVM bitcode to obj file "
"with external LLC compiler.");
return false;
}
}
else if (comp_ctx->external_asm_compiler) {
char asm_file_name[64];
if (!aot_generate_tempfile_name("wamrc-asm", "s", asm_file_name,
sizeof(asm_file_name))) {
return false;
}
if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine,
comp_ctx->module, asm_file_name,
LLVMAssemblyFile, &err)
!= 0) {
if (err) {
LLVMDisposeMessage(err);
err = NULL;
}
aot_set_last_error("emit elf to assembly file failed.");
return false;
}
snprintf(cmd, sizeof(cmd), "%s %s -o %s %s",
comp_ctx->external_asm_compiler,
comp_ctx->asm_compiler_flags ? comp_ctx->asm_compiler_flags
: "-O3 -c",
file_name, asm_file_name);
LOG_VERBOSE("invoking external ASM compiler:\n\t%s", cmd);
ret = system(cmd);
/* remove temp assembly file */
unlink(asm_file_name);
if (ret != 0) {
aot_set_last_error("failed to compile Assembly file to obj "
"file with external ASM compiler.");
return false;
}
}
return true;
}
#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */
if (!strncmp(LLVMGetTargetName(target), "arc", 3))
/* Emit to assmelby file instead for arc target
as it cannot emit to object file */

4
core/iwasm/compilation/aot_compiler.h
View File

@ -371,6 +371,10 @@ aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data,
bool
aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name);
char *
aot_generate_tempfile_name(const char *prefix, const char *extension,
char *buffer, uint32 len);
#ifdef __cplusplus
} /* end of extern "C" */
#endif

35
core/iwasm/compilation/aot_emit_aot_file.c
View File

@ -2692,8 +2692,41 @@ aot_obj_data_create(AOTCompContext *comp_ctx)
memset(obj_data, 0, sizeof(AOTObjectData));
bh_print_time("Begin to emit object file");
if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) {
#if defined(_WIN32) || defined(_WIN32_)
aot_set_last_error("external toolchain not supported on Windows");
goto fail;
#else
/* Generate a temp file name */
int ret;
char obj_file_name[64];
if (!strncmp(LLVMGetTargetName(target), "arc", 3)) {
if (!aot_generate_tempfile_name("wamrc-obj", "o", obj_file_name,
sizeof(obj_file_name))) {
goto fail;
}
if (!aot_emit_object_file(comp_ctx, obj_file_name)) {
goto fail;
}
/* create memory buffer from object file */
ret = LLVMCreateMemoryBufferWithContentsOfFile(
obj_file_name, &obj_data->mem_buf, &err);
/* remove temp object file */
unlink(obj_file_name);
if (ret != 0) {
if (err) {
LLVMDisposeMessage(err);
err = NULL;
}
aot_set_last_error("create mem buffer with file failed.");
goto fail;
}
#endif /* end of defined(_WIN32) || defined(_WIN32_) */
}
else if (!strncmp(LLVMGetTargetName(target), "arc", 3)) {
#if defined(_WIN32) || defined(_WIN32_)
aot_set_last_error("emit object file on Windows is unsupported.");
goto fail;

45
core/iwasm/compilation/aot_llvm.c
View File

@ -1660,6 +1660,51 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
opt_level = option->opt_level;
size_level = option->size_level;
/* verify external llc compiler */
comp_ctx->external_llc_compiler = getenv("WAMRC_LLC_COMPILER");
if (comp_ctx->external_llc_compiler) {
#if defined(_WIN32) || defined(_WIN32_)
comp_ctx->external_llc_compiler = NULL;
LOG_WARNING("External LLC compiler not supported on Windows.");
#else
if (access(comp_ctx->external_llc_compiler, X_OK) != 0) {
LOG_WARNING("WAMRC_LLC_COMPILER [%s] not found, fallback to "
"default pipeline",
comp_ctx->external_llc_compiler);
comp_ctx->external_llc_compiler = NULL;
}
else {
comp_ctx->llc_compiler_flags = getenv("WAMRC_LLC_FLAGS");
LOG_VERBOSE("Using external LLC compiler [%s]",
comp_ctx->external_llc_compiler);
}
#endif
}
/* verify external asm compiler */
if (!comp_ctx->external_llc_compiler) {
comp_ctx->external_asm_compiler = getenv("WAMRC_ASM_COMPILER");
if (comp_ctx->external_asm_compiler) {
#if defined(_WIN32) || defined(_WIN32_)
comp_ctx->external_asm_compiler = NULL;
LOG_WARNING("External ASM compiler not supported on Windows.");
#else
if (access(comp_ctx->external_asm_compiler, X_OK) != 0) {
LOG_WARNING(
"WAMRC_ASM_COMPILER [%s] not found, fallback to "
"default pipeline",
comp_ctx->external_asm_compiler);
comp_ctx->external_asm_compiler = NULL;
}
else {
comp_ctx->asm_compiler_flags = getenv("WAMRC_ASM_FLAGS");
LOG_VERBOSE("Using external ASM compiler [%s]",
comp_ctx->external_asm_compiler);
}
#endif
}
}
if (arch) {
/* Add default sub-arch if not specified */
if (!strcmp(arch, "arm"))

15
core/iwasm/compilation/aot_llvm.h
View File

@ -14,6 +14,7 @@
#include "llvm-c/Object.h"
#include "llvm-c/ExecutionEngine.h"
#include "llvm-c/Analysis.h"
#include "llvm-c/BitWriter.h"
#include "llvm-c/Transforms/Utils.h"
#include "llvm-c/Transforms/Scalar.h"
#include "llvm-c/Transforms/Vectorize.h"
@ -350,6 +351,20 @@ typedef struct AOTCompContext {
uint32 func_ctx_count;
char **custom_sections_wp;
uint32 custom_sections_count;
/* 3rd-party toolchains */
/* External llc compiler, if specified, wamrc will emit the llvm-ir file and
* invoke the llc compiler to generate object file.
* This can be used when we want to benefit from the optimization of other
* LLVM based toolchains */
const char *external_llc_compiler;
const char *llc_compiler_flags;
/* External asm compiler, if specified, wamrc will emit the text-based
* assembly file (.s) and invoke the llc compiler to generate object file.
* This will be useful when the upstream LLVM doesn't support to emit object
* file for some architecture (such as arc) */
const char *external_asm_compiler;
const char *asm_compiler_flags;
} AOTCompContext;
enum {