christoph/wamr
1
Fork 0
Code Releases Activity

Implement SIMD latest opcodes and update LLVM to 13.0 (#758)

Browse Source 
Implement the latest SIMD opcodes and update LLVM 13.0,
update the llvm build scripts, update the sample workloads‘ build scripts,
and build customized wasi-sdk to build some workloads.
Also refine the CI rules.

Signed-off-by: Wenyong Huang <wenyong.huang@intel.com>
This commit is contained in:
Wenyong Huang
2021-09-17 19:12:57 +08:00
committed by GitHub
parent 7e60a5db8d
commit 7be0d385a6
82 changed files with 5266 additions and 4698 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
core/iwasm/compilation/simd/simd_access_lanes.c
View File

@ -8,39 +8,6 @@
#include "../aot_emit_exception.h"
#include "../../aot/aot_runtime.h"
static bool
is_target_x86(AOTCompContext *comp_ctx)
{
return !strncmp(comp_ctx->target_arch, "x86_64", 6) ||
!strncmp(comp_ctx->target_arch, "i386", 4);
}
static LLVMValueRef
build_intx16_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const int *element_value)
{
LLVMValueRef vector, elements[16];
unsigned i;
for (i = 0; i < 16; i++) {
if (!(elements[i] =
LLVMConstInt(element_type, element_value[i], true))) {
HANDLE_FAILURE("LLVMConstInst");
goto fail;
}
}
if (!(vector = LLVMConstVector(elements, 16))) {
HANDLE_FAILURE("LLVMConstVector");
goto fail;
}
return vector;
fail:
return NULL;
}
bool
aot_compile_simd_shuffle(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
@ -67,7 +34,8 @@ aot_compile_simd_shuffle(AOTCompContext *comp_ctx,
}
/* build a vector <16 x i32> */
if (!(mask = build_intx16_vector(comp_ctx, I32_TYPE, values))) {
if (!(mask =
simd_build_const_integer_vector(comp_ctx, I32_TYPE, values, 16))) {
goto fail;
}
@ -77,29 +45,20 @@ aot_compile_simd_shuffle(AOTCompContext *comp_ctx,
goto fail;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
PUSH_V128(result);
return true;
fail:
return false;
}
/*TODO: llvm.experimental.vector.*/
/* shufflevector is not an option, since it requires *mask as a const */
bool
aot_compile_simd_swizzle_x86(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_swizzle_x86(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMValueRef vector, mask, max_lanes, condition, mask_lanes, result;
LLVMTypeRef param_types[2];
int max_lane_id[16] = { 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16 },
mask_lane_id[16] = { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 };
if (!(mask = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE,
"mask"))) {
@ -112,7 +71,15 @@ aot_compile_simd_swizzle_x86(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
}
/* icmp uge <16 x i8> mask, <16, 16, 16, 16, ...> */
if (!(max_lanes = build_intx16_vector(comp_ctx, INT8_TYPE, max_lane_id))) {
if (!(max_lanes = simd_build_splat_const_integer_vector(
comp_ctx, INT8_TYPE, 16, 16))) {
goto fail;
}
/* if the highest bit of every i8 of mask is 1, means doesn't pick up from vector */
/* select <16 x i1> %condition, <16 x i8> <0x80, 0x80, ...>, <16 x i8> %mask */
if (!(mask_lanes = simd_build_splat_const_integer_vector(
comp_ctx, INT8_TYPE, 0x80, 16))) {
goto fail;
}
@ -122,13 +89,6 @@ aot_compile_simd_swizzle_x86(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
goto fail;
}
/* if the highest bit of every i8 of mask is 1, means doesn't pick up from vector */
/* select <16 x i1> %condition, <16 x i8> <0x80, 0x80, ...>, <16 x i8> %mask */
if (!(mask_lanes =
build_intx16_vector(comp_ctx, INT8_TYPE, mask_lane_id))) {
goto fail;
}
if (!(mask = LLVMBuildSelect(comp_ctx->builder, condition, mask_lanes,
mask, "mask"))) {
HANDLE_FAILURE("LLVMBuildSelect");
@ -158,17 +118,13 @@ fail:
}
static bool
aot_compile_simd_swizzle_common(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_swizzle_common(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMValueRef vector, mask, default_lane_value, condition, max_lane_id,
result, idx, id, replace_with_zero, elem, elem_or_zero, undef;
uint8 i;
int const_lane_ids[16] = { 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16 },
const_zeors[16] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
if (!(mask = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE,
"mask"))) {
goto fail;
@ -185,8 +141,8 @@ aot_compile_simd_swizzle_common(AOTCompContext *comp_ctx, AOTFuncContext *func_c
}
/* icmp uge <16 x i8> mask, <16, 16, 16, 16, ...> */
if (!(max_lane_id =
build_intx16_vector(comp_ctx, INT8_TYPE, const_lane_ids))) {
if (!(max_lane_id = simd_build_splat_const_integer_vector(
comp_ctx, INT8_TYPE, 16, 16))) {
goto fail;
}
@ -197,8 +153,8 @@ aot_compile_simd_swizzle_common(AOTCompContext *comp_ctx, AOTFuncContext *func_c
}
/* if the id is out of range (>=16), set the id as 0 */
if (!(default_lane_value =
build_intx16_vector(comp_ctx, INT8_TYPE, const_zeors))) {
if (!(default_lane_value = simd_build_splat_const_integer_vector(
comp_ctx, INT8_TYPE, 0, 16))) {
goto fail;
}
@ -277,9 +233,9 @@ aot_compile_simd_extract(AOTCompContext *comp_ctx,
LLVMTypeRef result_type,
unsigned aot_value_type)
{
LLVMValueRef vector, idx, result;
LLVMValueRef vector, lane, result;
if (!(idx = I8_CONST(lane_id))) {
if (!(lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
@ -291,7 +247,7 @@ aot_compile_simd_extract(AOTCompContext *comp_ctx,
}
/* extractelement <vector_type> %vector, i8 lane_id*/
if (!(result = LLVMBuildExtractElement(comp_ctx->builder, vector, idx,
if (!(result = LLVMBuildExtractElement(comp_ctx->builder, vector, lane,
"element"))) {
HANDLE_FAILURE("LLVMBuildExtractElement");
goto fail;
@ -390,23 +346,20 @@ aot_compile_simd_replace(AOTCompContext *comp_ctx,
bool need_reduce,
LLVMTypeRef element_type)
{
LLVMValueRef vector, new_value, idx, result;
LLVMValueRef vector, new_value, lane, result;
POP(new_value, new_value_type);
if (!(idx = I8_CONST(lane_id))) {
HANDLE_FAILURE("LLVMConstInt");
if (!(lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id))) {
goto fail;
}
/* bitcast <2 x i64> %0 to <vector_type> */
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
goto fail;
}
/* bitcast <new_value_type> to <element_type> */
/* trunc <new_value_type> to <element_type> */
if (need_reduce) {
if (!(new_value = LLVMBuildTrunc(comp_ctx->builder, new_value,
element_type, "element"))) {
@ -415,23 +368,15 @@ aot_compile_simd_replace(AOTCompContext *comp_ctx,
}
}
/* insertelement <vector_type> %vector, <element_type> %element, i8 idx */
/* insertelement <vector_type> %vector, <element_type> %element, i32 lane */
if (!(result = LLVMBuildInsertElement(comp_ctx->builder, vector, new_value,
idx, "new_vector"))) {
lane, "new_vector"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
goto fail;
}
/* bitcast <vector_type> %result to <2 x i64> */
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "reesult");
PUSH_V128(result);
return true;
fail:
return false;
}

20
core/iwasm/compilation/simd/simd_access_lanes.h
View File

@ -82,6 +82,26 @@ aot_compile_simd_replace_f64x2(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 lane_id);
bool
aot_compile_simd_load8_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 lane_id);
bool
aot_compile_simd_load16_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 lane_id);
bool
aot_compile_simd_load32_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 lane_id);
bool
aot_compile_simd_load64_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 lane_id);
#ifdef __cplusplus
} /* end of extern "C" */
#endif

159
core/iwasm/compilation/simd/simd_bit_shifts.c
View File

@ -8,121 +8,112 @@
#include "../aot_emit_exception.h"
#include "../../aot/aot_runtime.h"
enum integer_shift {
e_shift_i8x16,
e_shift_i16x8,
e_shift_i32x4,
e_shift_i64x2,
};
static bool
simd_shift(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntShift shift_op,
LLVMTypeRef vector_type,
LLVMTypeRef element_type,
unsigned lane_width)
enum integer_shift itype)
{
LLVMValueRef vector, offset, width, undef, zeros, result;
LLVMTypeRef zeros_type;
LLVMValueRef vector, offset, result = NULL;
LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE,
V128_i32x4_TYPE, V128_i64x2_TYPE };
LLVMTypeRef element_type[] = { INT8_TYPE, INT16_TYPE, I32_TYPE, I64_TYPE };
LLVMValueRef undef[] = { LLVM_CONST(i8x16_undef), LLVM_CONST(i16x8_undef),
LLVM_CONST(i32x4_undef),
LLVM_CONST(i64x2_undef) };
LLVMValueRef mask[] = { LLVM_CONST(i8x16_vec_zero),
LLVM_CONST(i16x8_vec_zero),
LLVM_CONST(i32x4_vec_zero),
LLVM_CONST(i64x2_vec_zero) };
LLVMValueRef lane_bits[] = {
LLVM_CONST(i32_eight),
LLVMConstInt(I32_TYPE, 16, true),
LLVMConstInt(I32_TYPE, 32, true),
LLVMConstInt(I32_TYPE, 64, true),
};
POP_I32(offset);
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
goto fail;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
vector_type[itype], "vec"))) {
return false;
}
if (!(width = LLVMConstInt(I32_TYPE, lane_width, true))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
/* offset mod LaneBits */
if (!lane_bits[itype]
|| !(offset = LLVMBuildSRem(comp_ctx->builder, offset,
lane_bits[itype], "offset_fix"))) {
HANDLE_FAILURE("LLVMBuildSRem");
return false;
}
/* change type */
if (itype < e_shift_i32x4) {
offset = LLVMBuildTrunc(comp_ctx->builder, offset, element_type[itype],
"offset_trunc");
}
else if (itype == e_shift_i64x2) {
offset = LLVMBuildZExt(comp_ctx->builder, offset, element_type[itype],
"offset_ext");
}
if (!offset) {
HANDLE_FAILURE("LLVMBuildZext/LLVMBuildTrunc");
return false;
}
/* splat to a vector */
if (!(offset =
LLVMBuildInsertElement(comp_ctx->builder, undef[itype], offset,
I32_ZERO, "offset_vector_base"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
return false;
}
if (!(offset =
LLVMBuildURem(comp_ctx->builder, offset, width, "remainder"))) {
HANDLE_FAILURE("LLVMBuildURem");
goto fail;
}
if (I64_TYPE == element_type) {
if (!(offset = LLVMBuildZExt(comp_ctx->builder, offset, element_type,
"offset_scalar"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
}
else {
if (!(offset = LLVMBuildTruncOrBitCast(
comp_ctx->builder, offset, element_type, "offset_scalar"))) {
HANDLE_FAILURE("LLVMBuildTrunc");
goto fail;
}
}
/* create a vector with offset */
if (!(undef = LLVMGetUndef(vector_type))) {
HANDLE_FAILURE("LLVMGetUndef");
goto fail;
}
if (!(zeros_type = LLVMVectorType(I32_TYPE, 128 / lane_width))) {
HANDLE_FAILURE("LVMVectorType");
goto fail;
}
if (!(zeros = LLVMConstNull(zeros_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
if (!(offset = LLVMBuildInsertElement(comp_ctx->builder, undef, offset,
I32_ZERO, "base_vector"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
goto fail;
}
if (!(offset = LLVMBuildShuffleVector(comp_ctx->builder, offset, undef,
zeros, "offset_vector"))) {
LLVMBuildShuffleVector(comp_ctx->builder, offset, undef[itype],
mask[itype], "offset_vector"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
goto fail;
return false;
}
switch (shift_op) {
case INT_SHL:
{
if (!(result =
LLVMBuildShl(comp_ctx->builder, vector, offset, "shl"))) {
HANDLE_FAILURE("LLVMBuildShl");
goto fail;
}
result = LLVMBuildShl(comp_ctx->builder, vector, offset, "shl");
break;
}
case INT_SHR_S:
{
if (!(result = LLVMBuildAShr(comp_ctx->builder, vector, offset,
"ashr"))) {
HANDLE_FAILURE("LLVMBuildAShr");
goto fail;
}
result = LLVMBuildAShr(comp_ctx->builder, vector, offset, "ashr");
break;
}
case INT_SHR_U:
{
if (!(result = LLVMBuildLShr(comp_ctx->builder, vector, offset,
"lshr"))) {
HANDLE_FAILURE("LLVMBuildLShr");
goto fail;
}
result = LLVMBuildLShr(comp_ctx->builder, vector, offset, "lshr");
break;
}
default:
{
bh_assert(0);
goto fail;
break;
}
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"result"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
if (!result) {
HANDLE_FAILURE("LLVMBuildShl/LLVMBuildLShr/LLVMBuildAShr");
goto fail;
}
PUSH_V128(result);
return true;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
fail:
return false;
}
@ -132,8 +123,7 @@ aot_compile_simd_i8x16_shift(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntShift shift_op)
{
return simd_shift(comp_ctx, func_ctx, shift_op, V128_i8x16_TYPE, INT8_TYPE,
8);
return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i8x16);
}
bool
@ -141,8 +131,7 @@ aot_compile_simd_i16x8_shift(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntShift shift_op)
{
return simd_shift(comp_ctx, func_ctx, shift_op, V128_i16x8_TYPE,
INT16_TYPE, 16);
return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i16x8);
}
bool
@ -150,8 +139,7 @@ aot_compile_simd_i32x4_shift(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntShift shift_op)
{
return simd_shift(comp_ctx, func_ctx, shift_op, V128_i32x4_TYPE, I32_TYPE,
32);
return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i32x4);
}
bool
@ -159,6 +147,5 @@ aot_compile_simd_i64x2_shift(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntShift shift_op)
{
return simd_shift(comp_ctx, func_ctx, shift_op, V128_i64x2_TYPE, I64_TYPE,
64);
return simd_shift(comp_ctx, func_ctx, shift_op, e_shift_i64x2);
}

114
core/iwasm/compilation/simd/simd_bitmask_extracts.c
View File

@ -8,70 +8,92 @@
#include "../aot_emit_exception.h"
#include "../../aot/aot_runtime.h"
enum integer_bitmask_type {
e_bitmask_i8x16,
e_bitmask_i16x8,
e_bitmask_i32x4,
e_bitmask_i64x2,
};
/* TODO: should use a much clever intrinsic */
static bool
simd_build_bitmask(const AOTCompContext *comp_ctx,
const AOTFuncContext *func_ctx,
uint8 length,
LLVMTypeRef vector_type,
LLVMTypeRef element_type,
const char *intrinsic)
enum integer_bitmask_type itype)
{
LLVMValueRef vector, zeros, mask, mask_elements[16], cond, result;
LLVMTypeRef param_types[1], vector_ext_type;
const uint32 numbers[16] = { 0x1, 0x2, 0x4, 0x8, 0x10, 0x20,
0x40, 0x80, 0x100, 0x200, 0x400, 0x800,
0x1000, 0x2000, 0x4000, 0x8000 };
LLVMValueRef vector, mask, result;
uint8 i;
LLVMTypeRef vector_ext_type;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
uint32 lanes[] = { 16, 8, 4, 2 };
uint32 lane_bits[] = { 8, 16, 32, 64 };
LLVMTypeRef element_type[] = { INT8_TYPE, INT16_TYPE, I32_TYPE, I64_TYPE };
LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE,
V128_i32x4_TYPE, V128_i64x2_TYPE };
int32 mask_element[16] = { 0 };
const char *intrinsic[] = {
"llvm.vector.reduce.or.v16i64",
"llvm.vector.reduce.or.v8i64",
"llvm.vector.reduce.or.v4i64",
"llvm.vector.reduce.or.v2i64",
};
LLVMValueRef ashr_distance;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
vector_type[itype], "vec"))) {
goto fail;
}
if (!(vector_ext_type = LLVMVectorType(I32_TYPE, length))) {
/* fill every bit in a lange with its sign bit */
if (!(ashr_distance = simd_build_splat_const_integer_vector(
comp_ctx, element_type[itype], lane_bits[itype] - 1,
lanes[itype]))) {
goto fail;
}
if (!(vector = LLVMBuildAShr(comp_ctx->builder, vector, ashr_distance,
"vec_ashr"))) {
HANDLE_FAILURE("LLVMBuildAShr");
goto fail;
}
if (!(vector_ext_type = LLVMVectorType(I64_TYPE, lanes[itype]))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
if (!(vector = LLVMBuildSExt(comp_ctx->builder, vector, vector_ext_type,
"vec_ext"))) {
HANDLE_FAILURE("LLVMBuildSExt");
goto fail;
}
if (!(zeros = LLVMConstNull(vector_ext_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
for (i = 0; i < 16; i++) {
if (!(mask_elements[i] = LLVMConstInt(I32_TYPE, numbers[i], false))) {
HANDLE_FAILURE("LLVMConstInt");
if (e_bitmask_i64x2 != itype) {
if (!(vector = LLVMBuildSExt(comp_ctx->builder, vector,
vector_ext_type, "zext_to_i64"))) {
goto fail;
}
}
if (!(mask = LLVMConstVector(mask_elements, length))) {
HANDLE_FAILURE("LLVMConstVector");
for (i = 0; i < 16; i++) {
mask_element[i] = 0x1 << i;
}
if (!(mask = simd_build_const_integer_vector(
comp_ctx, I64_TYPE, mask_element, lanes[itype]))) {
goto fail;
}
if (!(cond = LLVMBuildICmp(comp_ctx->builder, LLVMIntSLT, vector, zeros,
"lt_zero"))) {
HANDLE_FAILURE("LLVMBuildICmp");
if (!(vector =
LLVMBuildAnd(comp_ctx->builder, vector, mask, "mask_bits"))) {
HANDLE_FAILURE("LLVMBuildAnd");
goto fail;
}
if (!(result =
LLVMBuildSelect(comp_ctx->builder, cond, mask, zeros, "select"))) {
HANDLE_FAILURE("LLVMBuildSelect");
aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic[itype],
I64_TYPE, &vector_ext_type, 1, vector))) {
goto fail;
}
param_types[0] = vector_ext_type;
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, I32_TYPE,
param_types, 1, result))) {
HANDLE_FAILURE("LLVMBuildCall");
if (!(result =
LLVMBuildTrunc(comp_ctx->builder, result, I32_TYPE, "to_i32"))) {
HANDLE_FAILURE("LLVMBuildTrunc");
goto fail;
}
@ -86,24 +108,26 @@ bool
aot_compile_simd_i8x16_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_build_bitmask(comp_ctx, func_ctx, 16, V128_i8x16_TYPE,
INT8_TYPE,
"llvm.experimental.vector.reduce.or.v16i32");
return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i8x16);
}
bool
aot_compile_simd_i16x8_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_build_bitmask(comp_ctx, func_ctx, 8, V128_i16x8_TYPE,
INT16_TYPE,
"llvm.experimental.vector.reduce.or.v8i32");
return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i16x8);
}
bool
aot_compile_simd_i32x4_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_build_bitmask(comp_ctx, func_ctx, 4, V128_i32x4_TYPE, I32_TYPE,
"llvm.experimental.vector.reduce.or.v4i32");
return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i32x4);
}
bool
aot_compile_simd_i64x2_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_build_bitmask(comp_ctx, func_ctx, e_bitmask_i64x2);
}

14
core/iwasm/compilation/simd/simd_bitmask_extracts.h
View File

@ -13,17 +13,23 @@ extern "C" {
#endif
bool
aot_compile_simd_i8x16_bitmask(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_i8x16_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_bitmask(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_i16x8_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i32x4_bitmask(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_i32x4_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i64x2_bitmask(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
#ifdef __cplusplus
} /* end of extern "C" */
#endif
#endif /* end of _SIMD_BITMASK_EXTRACTS_H_ */

4
core/iwasm/compilation/simd/simd_bitwise_ops.c
View File

@ -86,7 +86,7 @@ fail:
/* v128.or(v128.and(v1, c), v128.and(v2, v128.not(c))) */
static bool
v128_bitwise_bit_select(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
v128_bitwise_bitselect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef vector1, vector2, vector3, result;
@ -138,7 +138,7 @@ aot_compile_simd_v128_bitwise(AOTCompContext *comp_ctx,
case V128_NOT:
return v128_bitwise_not(comp_ctx, func_ctx);
case V128_BITSELECT:
return v128_bitwise_bit_select(comp_ctx, func_ctx);
return v128_bitwise_bitselect(comp_ctx, func_ctx);
default:
bh_assert(0);
return false;

206
core/iwasm/compilation/simd/simd_bool_reductions.c
View File

@ -8,145 +8,62 @@
#include "../aot_emit_exception.h"
#include "../../aot/aot_runtime.h"
static bool
simd_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
LLVMTypeRef element_type,
const char *intrinsic)
{
LLVMValueRef vector, zeros, non_zero, result;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
goto fail;
}
if (!(zeros = LLVMConstNull(vector_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
/* icmp eq <N x iX> %vector, zeroinitialize */
if (!(non_zero = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, vector, zeros,
"non_zero"))) {
HANDLE_FAILURE("LLVMBuildICmp");
goto fail;
}
/* zext <N x i1> to <N x iX> */
if (!(non_zero = LLVMBuildZExt(comp_ctx->builder, non_zero, vector_type,
"non_zero_ex"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, element_type,
&vector_type, 1, non_zero))) {
HANDLE_FAILURE("LLVMBuildCall");
goto fail;
}
if (!(zeros = LLVMConstNull(element_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
if (!(result = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, result, zeros,
"gt_zero"))) {
HANDLE_FAILURE("LLVMBuildICmp");
goto fail;
}
if (!(result =
LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, "ret"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
PUSH_I32(result);
return true;
fail:
return false;
}
bool
aot_compile_simd_i8x16_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_any_true(comp_ctx, func_ctx, V128_i8x16_TYPE, INT8_TYPE,
"llvm.experimental.vector.reduce.add.v16i8");
}
bool
aot_compile_simd_i16x8_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_any_true(comp_ctx, func_ctx, V128_i16x8_TYPE, INT16_TYPE,
"llvm.experimental.vector.reduce.add.v8i16");
}
bool
aot_compile_simd_i32x4_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_any_true(comp_ctx, func_ctx, V128_i32x4_TYPE, I32_TYPE,
"llvm.experimental.vector.reduce.add.v4i32");
}
enum integer_all_true {
e_int_all_true_v16i8,
e_int_all_true_v8i16,
e_int_all_true_v4i32,
e_int_all_true_v2i64,
};
static bool
simd_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
LLVMTypeRef element_type,
const char *intrinsic)
enum integer_all_true itype)
{
LLVMValueRef vector, zeros, is_zero, result;
LLVMValueRef vector, result;
LLVMTypeRef vector_i1_type;
LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE,
V128_i32x4_TYPE, V128_i64x2_TYPE };
uint32 lanes[] = { 16, 8, 4, 2 };
const char *intrinsic[] = {
"llvm.vector.reduce.and.v16i1",
"llvm.vector.reduce.and.v8i1",
"llvm.vector.reduce.and.v4i1",
"llvm.vector.reduce.and.v2i1",
};
LLVMValueRef zero[] = {
LLVM_CONST(i8x16_vec_zero),
LLVM_CONST(i16x8_vec_zero),
LLVM_CONST(i32x4_vec_zero),
LLVM_CONST(i64x2_vec_zero),
};
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
if (!(vector_i1_type = LLVMVectorType(INT1_TYPE, lanes[itype]))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
if (!(zeros = LLVMConstNull(vector_type))) {
HANDLE_FAILURE("LLVMConstNull");
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
vector_type[itype], "vector"))) {
goto fail;
}
/* icmp eq <N x iX> %vector, zeroinitialize */
if (!(is_zero = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, vector, zeros,
"is_zero"))) {
/* compare with zero */
if (!(result = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, vector,
zero[itype], "ne_zero"))) {
HANDLE_FAILURE("LLVMBuildICmp");
goto fail;
}
/* zext <N x i1> to <N x iX> */
if (!(is_zero = LLVMBuildZExt(comp_ctx->builder, is_zero, vector_type,
"is_zero_ex"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, element_type,
&vector_type, 1, is_zero))) {
HANDLE_FAILURE("LLVMBuildCall");
goto fail;
}
if (!(zeros = LLVMConstNull(element_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
if (!(result = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, result, zeros,
"none"))) {
HANDLE_FAILURE("LLVMBuildICmp");
/* check zero */
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx,
intrinsic[itype], INT1_TYPE,
&vector_i1_type, 1, result))) {
goto fail;
}
if (!(result =
LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, "ret"))) {
LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, "to_i32"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
@ -162,22 +79,61 @@ bool
aot_compile_simd_i8x16_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_all_true(comp_ctx, func_ctx, V128_i8x16_TYPE, INT8_TYPE,
"llvm.experimental.vector.reduce.add.v16i8");
return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v16i8);
}
bool
aot_compile_simd_i16x8_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_all_true(comp_ctx, func_ctx, V128_i16x8_TYPE, INT16_TYPE,
"llvm.experimental.vector.reduce.add.v8i16");
return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v8i16);
}
bool
aot_compile_simd_i32x4_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_all_true(comp_ctx, func_ctx, V128_i32x4_TYPE, I32_TYPE,
"llvm.experimental.vector.reduce.add.v4i32");
return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v4i32);
}
bool
aot_compile_simd_i64x2_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_all_true(comp_ctx, func_ctx, e_int_all_true_v2i64);
}
bool
aot_compile_simd_v128_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMTypeRef vector_type;
LLVMValueRef vector, result;
if (!(vector_type = LLVMVectorType(INT1_TYPE, 128))) {
return false;
}
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vector"))) {
goto fail;
}
if (!(result = aot_call_llvm_intrinsic(
comp_ctx, func_ctx, "llvm.vector.reduce.or.v128i1", INT1_TYPE,
&vector_type, 1, vector))) {
goto fail;
}
if (!(result =
LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, "to_i32"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
PUSH_I32(result);
return true;
fail:
return false;
}

20
core/iwasm/compilation/simd/simd_bool_reductions.h
View File

@ -12,18 +12,6 @@
extern "C" {
#endif
bool
aot_compile_simd_i8x16_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i32x4_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i8x16_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
@ -36,6 +24,14 @@ bool
aot_compile_simd_i32x4_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i64x2_all_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_v128_any_true(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
#ifdef __cplusplus
} /* end of extern "C" */
#endif

117
core/iwasm/compilation/simd/simd_common.c
View File

@ -44,4 +44,119 @@ simd_bitcast_and_push_v128(const AOTCompContext *comp_ctx,
return true;
fail:
return false;
}
}
LLVMValueRef
simd_lane_id_to_llvm_value(AOTCompContext *comp_ctx, uint8 lane_id)
{
LLVMValueRef lane_indexes[] = {
LLVM_CONST(i32_zero), LLVM_CONST(i32_one),
LLVM_CONST(i32_two), LLVM_CONST(i32_three),
LLVM_CONST(i32_four), LLVM_CONST(i32_five),
LLVM_CONST(i32_six), LLVM_CONST(i32_seven),
LLVM_CONST(i32_eight), LLVM_CONST(i32_nine),
LLVM_CONST(i32_ten), LLVM_CONST(i32_eleven),
LLVM_CONST(i32_twelve), LLVM_CONST(i32_thirteen),
LLVM_CONST(i32_fourteen), LLVM_CONST(i32_fifteen),
};
return lane_id < 16 ? lane_indexes[lane_id] : NULL;
}
LLVMValueRef
simd_build_const_integer_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const int *element_value,
uint32 length)
{
LLVMValueRef vector = NULL;
LLVMValueRef *elements;
unsigned i;
if (!(elements = wasm_runtime_malloc(sizeof(LLVMValueRef) * length))) {
return NULL;
}
for (i = 0; i < length; i++) {
if (!(elements[i] =
LLVMConstInt(element_type, element_value[i], true))) {
HANDLE_FAILURE("LLVMConstInst");
goto fail;
}
}
if (!(vector = LLVMConstVector(elements, length))) {
HANDLE_FAILURE("LLVMConstVector");
goto fail;
}
fail:
wasm_runtime_free(elements);
return vector;
}
LLVMValueRef
simd_build_splat_const_integer_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const int64 element_value,
uint32 length)
{
LLVMValueRef vector = NULL, element;
LLVMValueRef *elements;
unsigned i;
if (!(elements = wasm_runtime_malloc(sizeof(LLVMValueRef) * length))) {
return NULL;
}
if (!(element = LLVMConstInt(element_type, element_value, true))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
for (i = 0; i < length; i++) {
elements[i] = element;
}
if (!(vector = LLVMConstVector(elements, length))) {
HANDLE_FAILURE("LLVMConstVector");
goto fail;
}
fail:
wasm_runtime_free(elements);
return vector;
}
LLVMValueRef
simd_build_splat_const_float_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const float element_value,
uint32 length)
{
LLVMValueRef vector = NULL, element;
LLVMValueRef *elements;
unsigned i;
if (!(elements = wasm_runtime_malloc(sizeof(LLVMValueRef) * length))) {
return NULL;
}
if (!(element = LLVMConstReal(element_type, element_value))) {
HANDLE_FAILURE("LLVMConstReal");
goto fail;
}
for (i = 0; i < length; i++) {
elements[i] = element;
}
if (!(vector = LLVMConstVector(elements, length))) {
HANDLE_FAILURE("LLVMConstVector");
goto fail;
}
fail:
wasm_runtime_free(elements);
return vector;
}

27
core/iwasm/compilation/simd/simd_common.h
View File

@ -8,6 +8,13 @@
#include "../aot_compiler.h"
static inline bool
is_target_x86(AOTCompContext *comp_ctx)
{
return !strncmp(comp_ctx->target_arch, "x86_64", 6)
|| !strncmp(comp_ctx->target_arch, "i386", 4);
}
LLVMValueRef
simd_pop_v128_and_bitcast(const AOTCompContext *comp_ctx,
const AOTFuncContext *func_ctx,
@ -20,4 +27,24 @@ simd_bitcast_and_push_v128(const AOTCompContext *comp_ctx,
LLVMValueRef vector,
const char *name);
LLVMValueRef
simd_lane_id_to_llvm_value(AOTCompContext *comp_ctx, uint8 lane_id);
LLVMValueRef
simd_build_const_integer_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const int *element_value,
uint32 length);
LLVMValueRef
simd_build_splat_const_integer_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const int64 element_value,
uint32 length);
LLVMValueRef
simd_build_splat_const_float_vector(const AOTCompContext *comp_ctx,
const LLVMTypeRef element_type,
const float element_value,
uint32 length);
#endif /* _SIMD_COMMON_H_ */

8
core/iwasm/compilation/simd/simd_comparisons.c
View File

@ -160,6 +160,14 @@ aot_compile_simd_i32x4_compare(AOTCompContext *comp_ctx,
return interger_vector_compare(comp_ctx, func_ctx, cond, V128_i32x4_TYPE);
}
bool
aot_compile_simd_i64x2_compare(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntCond cond)
{
return interger_vector_compare(comp_ctx, func_ctx, cond, V128_i64x2_TYPE);
}
static bool
float_vector_compare(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,

5
core/iwasm/compilation/simd/simd_comparisons.h
View File

@ -27,6 +27,11 @@ aot_compile_simd_i32x4_compare(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntCond cond);
bool
aot_compile_simd_i64x2_compare(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
IntCond cond);
bool
aot_compile_simd_f32x4_compare(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,

118
core/iwasm/compilation/simd/simd_construct_values.c
View File

@ -3,6 +3,7 @@
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "simd_common.h"
#include "simd_construct_values.h"
#include "../aot_emit_exception.h"
#include "../interpreter/wasm_opcode.h"
@ -14,23 +15,19 @@ aot_compile_simd_v128_const(AOTCompContext *comp_ctx,
const uint8 *imm_bytes)
{
uint64 imm1, imm2;
LLVMValueRef undef, first_long, agg1, second_long, agg2;
LLVMValueRef first_long, agg1, second_long, agg2;
wasm_runtime_read_v128(imm_bytes, &imm1, &imm2);
if (!(undef = LLVMGetUndef(V128_i64x2_TYPE))) {
HANDLE_FAILURE("LLVMGetUndef");
goto fail;
}
/* %agg1 = insertelement <2 x i64> undef, i16 0, i64 ${*imm} */
if (!(first_long = I64_CONST(imm1))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
if (!(agg1 = LLVMBuildInsertElement(comp_ctx->builder, undef, first_long,
I32_ZERO, "agg1"))) {
if (!(agg1 =
LLVMBuildInsertElement(comp_ctx->builder, LLVM_CONST(i64x2_undef),
first_long, I32_ZERO, "agg1"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
goto fail;
}
@ -48,7 +45,6 @@ aot_compile_simd_v128_const(AOTCompContext *comp_ctx,
}
PUSH_V128(agg2);
return true;
fail:
return false;
@ -57,134 +53,88 @@ fail:
bool
aot_compile_simd_splat(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 splat_opcode)
uint8 opcode)
{
LLVMValueRef value, undef, base, mask, new_vector, result;
LLVMTypeRef all_zero_ty;
uint32 opcode_index = opcode - SIMD_i8x16_splat;
LLVMValueRef value = NULL, base, new_vector;
LLVMValueRef undefs[] = {
LLVM_CONST(i8x16_undef), LLVM_CONST(i16x8_undef),
LLVM_CONST(i32x4_undef), LLVM_CONST(i64x2_undef),
LLVM_CONST(f32x4_undef), LLVM_CONST(f64x2_undef),
};
LLVMValueRef masks[] = {
LLVM_CONST(i32x16_zero), LLVM_CONST(i32x8_zero),
LLVM_CONST(i32x4_zero), LLVM_CONST(i32x2_zero),
LLVM_CONST(i32x4_zero), LLVM_CONST(i32x2_zero),
};
switch (splat_opcode) {
switch (opcode) {
case SIMD_i8x16_splat:
{
LLVMValueRef input;
POP_I32(input);
/* trunc i32 %input to i8 */
if (!(value = LLVMBuildTrunc(comp_ctx->builder, input, INT8_TYPE,
"trunc"))) {
HANDLE_FAILURE("LLVMBuildTrunc");
goto fail;
}
undef = LLVMGetUndef(V128_i8x16_TYPE);
if (!(all_zero_ty = LLVMVectorType(I32_TYPE, 16))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
value =
LLVMBuildTrunc(comp_ctx->builder, input, INT8_TYPE, "trunc");
break;
}
case SIMD_i16x8_splat:
{
LLVMValueRef input;
POP_I32(input);
/* trunc i32 %input to i16 */
if (!(value = LLVMBuildTrunc(comp_ctx->builder, input, INT16_TYPE,
"trunc"))) {
HANDLE_FAILURE("LLVMBuildTrunc");
goto fail;
}
undef = LLVMGetUndef(V128_i16x8_TYPE);
if (!(all_zero_ty = LLVMVectorType(I32_TYPE, 8))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
value =
LLVMBuildTrunc(comp_ctx->builder, input, INT16_TYPE, "trunc");
break;
}
case SIMD_i32x4_splat:
{
POP_I32(value);
undef = LLVMGetUndef(V128_i32x4_TYPE);
if (!(all_zero_ty = LLVMVectorType(I32_TYPE, 4))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
case SIMD_i64x2_splat:
{
POP(value, VALUE_TYPE_I64);
undef = LLVMGetUndef(V128_i64x2_TYPE);
if (!(all_zero_ty = LLVMVectorType(I32_TYPE, 2))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
case SIMD_f32x4_splat:
{
POP(value, VALUE_TYPE_F32);
undef = LLVMGetUndef(V128_f32x4_TYPE);
if (!(all_zero_ty = LLVMVectorType(I32_TYPE, 4))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
case SIMD_f64x2_splat:
{
POP(value, VALUE_TYPE_F64);
undef = LLVMGetUndef(V128_f64x2_TYPE);
if (!(all_zero_ty = LLVMVectorType(I32_TYPE, 2))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
default:
{
bh_assert(0);
goto fail;
break;
}
}
if (!undef) {
HANDLE_FAILURE("LVMGetUndef");
if (!value) {
goto fail;
}
/* insertelement <n x ty> undef, ty %value, i32 0 */
if (!(base = LLVMBuildInsertElement(comp_ctx->builder, undef, value,
I32_ZERO, "base"))) {
if (!(base =
LLVMBuildInsertElement(comp_ctx->builder, undefs[opcode_index],
value, I32_ZERO, "base"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
goto fail;
}
/* <n x i32> zeroinitializer */
if (!(mask = LLVMConstNull(all_zero_ty))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
/* shufflevector <ty1> %base, <ty2> undef, <n x i32> zeroinitializer */
if (!(new_vector = LLVMBuildShuffleVector(comp_ctx->builder, base, undef,
mask, "new_vector"))) {
if (!(new_vector = LLVMBuildShuffleVector(
comp_ctx->builder, base, undefs[opcode_index], masks[opcode_index],
"new_vector"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
goto fail;
}
/* bitcast <ty> <value> to <2 x i64> */
if (!(result = LLVMBuildBitCast(comp_ctx->builder, new_vector,
V128_i64x2_TYPE, "ret"))) {
HANDLE_FAILURE("LLVMBuidlCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, new_vector,
"result");
fail:
return false;
}

1303
core/iwasm/compilation/simd/simd_conversions.c
View File

File diff suppressed because it is too large Load Diff

66
core/iwasm/compilation/simd/simd_conversions.h
View File

@ -23,27 +23,77 @@ aot_compile_simd_i16x8_narrow_i32x4(AOTCompContext *comp_ctx,
bool is_signed);
bool
aot_compile_simd_i16x8_widen_i8x16(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_low,
bool is_signed);
aot_compile_simd_i32x4_narrow_i64x2(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_i32x4_widen_i16x8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_low,
bool is_signed);
aot_compile_simd_i16x8_extend_i8x16(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_low,
bool is_signed);
bool
aot_compile_simd_i32x4_extend_i16x8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_low,
bool is_signed);
bool
aot_compile_simd_i64x2_extend_i32x4(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool lower_half,
bool is_signed);
bool
aot_compile_simd_i32x4_trunc_sat_f32x4(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_i32x4_trunc_sat_f64x2(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_f32x4_convert_i32x4(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_f64x2_convert_i32x4(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_i16x8_extadd_pairwise_i8x16(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_i32x4_extadd_pairwise_i16x8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_signed);
bool
aot_compile_simd_i16x8_q15mulr_sat(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_extmul_i8x16(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_low,
bool is_signed);
bool
aot_compile_simd_i32x4_extmul_i16x8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool is_low,
bool is_signed);
bool
aot_compile_simd_i64x2_extmul_i32x4(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool lower_half,
bool is_signed);
#ifdef __cplusplus
} /* end of extern "C" */
#endif

420
core/iwasm/compilation/simd/simd_floating_point.c
View File

@ -9,111 +9,45 @@
#include "../aot_emit_numberic.h"
#include "../../aot/aot_runtime.h"
static LLVMValueRef
simd_v128_float_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
FloatArithmetic arith_op,
LLVMValueRef lhs,
LLVMValueRef rhs)
{
LLVMValueRef result;
LLVMRealPredicate op;
op = FLOAT_MIN == arith_op ? LLVMRealULT : LLVMRealUGT;
if (!(result = LLVMBuildFCmp(comp_ctx->builder, op, lhs, rhs, "cmp"))) {
HANDLE_FAILURE("LLVMBuildFCmp");
goto fail;
}
if (!(result =
LLVMBuildSelect(comp_ctx->builder, result, lhs, rhs, "select"))) {
HANDLE_FAILURE("LLVMBuildSelect");
goto fail;
}
return result;
fail:
return NULL;
}
static bool
simd_v128_float_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
FloatArithmetic arith_op,
LLVMTypeRef vector_type)
{
LLVMValueRef lhs, rhs, result;
LLVMValueRef lhs, rhs, result = NULL;
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
goto fail;
if (!(rhs =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
return false;
}
switch (arith_op) {
case FLOAT_ADD:
if (!(result =
LLVMBuildFAdd(comp_ctx->builder, lhs, rhs, "sum"))) {
HANDLE_FAILURE("LLVMBuildFAdd");
goto fail;
}
result = LLVMBuildFAdd(comp_ctx->builder, lhs, rhs, "sum");
break;
case FLOAT_SUB:
if (!(result = LLVMBuildFSub(comp_ctx->builder, lhs, rhs,
"difference"))) {
HANDLE_FAILURE("LLVMBuildFSub");
goto fail;
}
result = LLVMBuildFSub(comp_ctx->builder, lhs, rhs, "difference");
break;
case FLOAT_MUL:
if (!(result =
LLVMBuildFMul(comp_ctx->builder, lhs, rhs, "product"))) {
HANDLE_FAILURE("LLVMBuildFMul");
goto fail;
}
result = LLVMBuildFMul(comp_ctx->builder, lhs, rhs, "product");
break;
case FLOAT_DIV:
if (!(result =
LLVMBuildFDiv(comp_ctx->builder, lhs, rhs, "quotient"))) {
HANDLE_FAILURE("LLVMBuildFDiv");
goto fail;
}
break;
case FLOAT_MIN:
if (!(result = simd_v128_float_cmp(comp_ctx, func_ctx, FLOAT_MIN,
lhs, rhs))) {
goto fail;
}
break;
case FLOAT_MAX:
if (!(result = simd_v128_float_cmp(comp_ctx, func_ctx, FLOAT_MAX,
lhs, rhs))) {
goto fail;
}
result = LLVMBuildFDiv(comp_ctx->builder, lhs, rhs, "quotient");
break;
default:
result = NULL;
bh_assert(0);
break;
return false;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
if (!result) {
HANDLE_FAILURE(
"LLVMBuildFAdd/LLVMBuildFSub/LLVMBuildFMul/LLVMBuildFDiv");
return false;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
@ -139,30 +73,19 @@ simd_v128_float_neg(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type)
{
LLVMValueRef number, result;
LLVMValueRef vector, result;
if (!(number = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"number"))) {
goto fail;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vector"))) {
return false;
}
if (!(result = LLVMBuildFNeg(comp_ctx->builder, number, "neg"))) {
if (!(result = LLVMBuildFNeg(comp_ctx->builder, vector, "neg"))) {
HANDLE_FAILURE("LLVMBuildFNeg");
goto fail;
return false;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
@ -178,119 +101,310 @@ aot_compile_simd_f64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
}
static bool
simd_v128_float_intrinsic(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
const char *intrinsic)
simd_float_intrinsic(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
const char *intrinsic)
{
LLVMValueRef number, result;
LLVMValueRef vector, result;
LLVMTypeRef param_types[1] = { vector_type };
if (!(number = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"number"))) {
goto fail;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vector"))) {
return false;
}
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, vector_type,
param_types, 1, number))) {
if (!(result =
aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, vector_type,
param_types, 1, vector))) {
HANDLE_FAILURE("LLVMBuildCall");
goto fail;
return false;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_f32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.fabs.v4f32");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.fabs.v4f32");
}
bool
aot_compile_simd_f64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.fabs.v2f64");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.fabs.v2f64");
}
bool
aot_compile_simd_f32x4_round(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.round.v4f32");
}
bool
aot_compile_simd_f64x2_round(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.round.v2f64");
}
bool
aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.sqrt.v4f32");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.sqrt.v4f32");
}
bool
aot_compile_simd_f64x2_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.sqrt.v2f64");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.sqrt.v2f64");
}
bool
aot_compile_simd_f32x4_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.ceil.v4f32");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.ceil.v4f32");
}
bool
aot_compile_simd_f64x2_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.ceil.v2f64");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.ceil.v2f64");
}
bool
aot_compile_simd_f32x4_floor(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_f32x4_floor(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.floor.v4f32");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.floor.v4f32");
}
bool
aot_compile_simd_f64x2_floor(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_f64x2_floor(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.floor.v2f64");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.floor.v2f64");
}
bool
aot_compile_simd_f32x4_trunc(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_f32x4_trunc(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.trunc.v4f32");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.trunc.v4f32");
}
bool
aot_compile_simd_f64x2_trunc(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_f64x2_trunc(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.trunc.v2f64");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.trunc.v2f64");
}
bool
aot_compile_simd_f32x4_nearest(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_f32x4_nearest(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.rint.v4f32");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE,
"llvm.rint.v4f32");
}
bool
aot_compile_simd_f64x2_nearest(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
aot_compile_simd_f64x2_nearest(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.rint.v2f64");
return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE,
"llvm.rint.v2f64");
}
static bool
simd_float_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
FloatArithmetic arith_op,
LLVMTypeRef vector_type)
{
LLVMValueRef lhs, rhs, result;
LLVMRealPredicate op = FLOAT_MIN == arith_op ? LLVMRealULT : LLVMRealUGT;
if (!(rhs =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
return false;
}
if (!(result = LLVMBuildFCmp(comp_ctx->builder, op, lhs, rhs, "cmp"))) {
HANDLE_FAILURE("LLVMBuildFCmp");
return false;
}
if (!(result =
LLVMBuildSelect(comp_ctx->builder, result, lhs, rhs, "select"))) {
HANDLE_FAILURE("LLVMBuildSelect");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
/*TODO: sugggest non-IA platforms check with "llvm.minimum.*" and "llvm.maximum.*" firstly */
bool
aot_compile_simd_f32x4_min_max(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min)
{
return simd_float_cmp(comp_ctx, func_ctx, run_min ? FLOAT_MIN : FLOAT_MAX,
V128_f32x4_TYPE);
}
bool
aot_compile_simd_f64x2_min_max(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min)
{
return simd_float_cmp(comp_ctx, func_ctx, run_min ? FLOAT_MIN : FLOAT_MAX,
V128_f64x2_TYPE);
}
static bool
simd_float_pmin_max(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
const char *intrinsic)
{
LLVMValueRef lhs, rhs, result;
LLVMTypeRef param_types[2];
param_types[0] = vector_type;
param_types[1] = vector_type;
if (!(rhs =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
return false;
}
if (!(result =
aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, vector_type,
param_types, 2, lhs, rhs))) {
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_f32x4_pmin_pmax(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min)
{
return simd_float_pmin_max(comp_ctx, func_ctx, V128_f32x4_TYPE,
run_min ? "llvm.minnum.v4f32"
: "llvm.maxnum.v4f32");
}
bool
aot_compile_simd_f64x2_pmin_pmax(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min)
{
return simd_float_pmin_max(comp_ctx, func_ctx, V128_f64x2_TYPE,
run_min ? "llvm.minnum.v2f64"
: "llvm.maxnum.v2f64");
}
bool
aot_compile_simd_f64x2_demote(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMValueRef vector, elem_0, elem_1, result;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_f64x2_TYPE, "vector"))) {
return false;
}
if (!(elem_0 = LLVMBuildExtractElement(comp_ctx->builder, vector,
LLVM_CONST(i32_zero), "elem_0"))
|| !(elem_1 = LLVMBuildExtractElement(
comp_ctx->builder, vector, LLVM_CONST(i32_one), "elem_1"))) {
HANDLE_FAILURE("LLVMBuildExtractElement");
return false;
}
/* fptrunc <f64> elem to <f32> */
if (!(elem_0 = LLVMBuildFPTrunc(comp_ctx->builder, elem_0, F32_TYPE,
"elem_0_trunc"))
|| !(elem_1 = LLVMBuildFPTrunc(comp_ctx->builder, elem_1, F32_TYPE,
"elem_1_trunc"))) {
HANDLE_FAILURE("LLVMBuildFPTrunc");
return false;
}
if (!(result = LLVMBuildInsertElement(
comp_ctx->builder, LLVM_CONST(f32x4_vec_zero), elem_0,
LLVM_CONST(i32_zero), "new_vector_0"))
|| !(result =
LLVMBuildInsertElement(comp_ctx->builder, result, elem_1,
LLVM_CONST(i32_one), "new_vector_1"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_f32x4_promote(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMValueRef vector, elem_0, elem_1, result;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_f32x4_TYPE, "vector"))) {
return false;
}
if (!(elem_0 = LLVMBuildExtractElement(comp_ctx->builder, vector,
LLVM_CONST(i32_zero), "elem_0"))
|| !(elem_1 = LLVMBuildExtractElement(
comp_ctx->builder, vector, LLVM_CONST(i32_one), "elem_1"))) {
HANDLE_FAILURE("LLVMBuildExtractElement");
return false;
}
/* fpext <f32> elem to <f64> */
if (!(elem_0 =
LLVMBuildFPExt(comp_ctx->builder, elem_0, F64_TYPE, "elem_0_ext"))
|| !(elem_1 = LLVMBuildFPExt(comp_ctx->builder, elem_1, F64_TYPE,
"elem_1_ext"))) {
HANDLE_FAILURE("LLVMBuildFPExt");
return false;
}
if (!(result = LLVMBuildInsertElement(
comp_ctx->builder, LLVM_CONST(f64x2_vec_zero), elem_0,
LLVM_CONST(i32_zero), "new_vector_0"))
|| !(result =
LLVMBuildInsertElement(comp_ctx->builder, result, elem_1,
LLVM_CONST(i32_one), "new_vector_1"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}

66
core/iwasm/compilation/simd/simd_floating_point.h
View File

@ -35,34 +35,80 @@ bool
aot_compile_simd_f64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f32x4_round(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f64x2_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f64x2_round(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f64x2_ceil(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f64x2_sqrt(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_floor(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f32x4_ceil(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f64x2_floor(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f64x2_ceil(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_trunc(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f32x4_floor(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f64x2_trunc(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f64x2_floor(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_nearest(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f32x4_trunc(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f64x2_nearest(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
aot_compile_simd_f64x2_trunc(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_nearest(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f64x2_nearest(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_min_max(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min);
bool
aot_compile_simd_f64x2_min_max(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min);
bool
aot_compile_simd_f32x4_pmin_pmax(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min);
bool
aot_compile_simd_f64x2_pmin_pmax(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
bool run_min);
bool
aot_compile_simd_f64x2_demote(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_f32x4_promote(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
#ifdef __cplusplus
} /* end of extern "C" */

481
core/iwasm/compilation/simd/simd_int_arith.c
View File

@ -9,59 +9,41 @@
#include "../../aot/aot_runtime.h"
static bool
simd_v128_integer_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
LLVMValueRef lhs,
LLVMValueRef rhs)
simd_integer_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
LLVMTypeRef vector_type)
{
LLVMValueRef result;
LLVMValueRef lhs, rhs, result = NULL;
if (!(rhs =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
return false;
}
switch (arith_op) {
case V128_ADD:
if (!(result = LLVMBuildAdd(comp_ctx->builder, lhs, rhs, "sum"))) {
HANDLE_FAILURE("LLVMBuildAdd");
goto fail;
}
result = LLVMBuildAdd(comp_ctx->builder, lhs, rhs, "sum");
break;
case V128_SUB:
if (!(result =
LLVMBuildSub(comp_ctx->builder, lhs, rhs, "difference"))) {
HANDLE_FAILURE("LLVMBuildSub");
goto fail;
}
result = LLVMBuildSub(comp_ctx->builder, lhs, rhs, "difference");
break;
case V128_MUL:
if (!(result =
LLVMBuildMul(comp_ctx->builder, lhs, rhs, "product"))) {
HANDLE_FAILURE("LLVMBuildMul");
goto fail;
}
break;
case V128_NEG:
if (!(result = LLVMBuildNeg(comp_ctx->builder, lhs, "neg"))) {
HANDLE_FAILURE("LLVMBuildNeg");
goto fail;
}
result = LLVMBuildMul(comp_ctx->builder, lhs, rhs, "product");
break;
default:
result = NULL;
bh_assert(0);
HANDLE_FAILURE("Unsupport arith_op");
break;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
if (!result) {
HANDLE_FAILURE("LLVMBuildAdd/LLVMBuildSub/LLVMBuildMul");
return false;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
@ -69,21 +51,7 @@ aot_compile_simd_i8x16_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op)
{
LLVMValueRef lhs, rhs;
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i8x16_TYPE,
"lhs"))) {
goto fail;
}
return simd_v128_integer_arith(comp_ctx, func_ctx, arith_op, lhs, rhs);
fail:
return false;
return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i8x16_TYPE);
}
bool
@ -91,21 +59,7 @@ aot_compile_simd_i16x8_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op)
{
LLVMValueRef lhs, rhs;
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i16x8_TYPE,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i16x8_TYPE,
"lhs"))) {
goto fail;
}
return simd_v128_integer_arith(comp_ctx, func_ctx, arith_op, lhs, rhs);
fail:
return false;
return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i16x8_TYPE);
}
bool
@ -113,21 +67,7 @@ aot_compile_simd_i32x4_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op)
{
LLVMValueRef lhs, rhs;
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i32x4_TYPE,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i32x4_TYPE,
"lhs"))) {
goto fail;
}
return simd_v128_integer_arith(comp_ctx, func_ctx, arith_op, lhs, rhs);
fail:
return false;
return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i32x4_TYPE);
}
bool
@ -135,73 +75,354 @@ aot_compile_simd_i64x2_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op)
{
LLVMValueRef lhs, rhs;
return simd_integer_arith(comp_ctx, func_ctx, arith_op, V128_i64x2_TYPE);
}
POP_V128(rhs);
POP_V128(lhs);
static bool
simd_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMTypeRef type)
{
LLVMValueRef vector, result;
return simd_v128_integer_arith(comp_ctx, func_ctx, arith_op, lhs, rhs);
fail:
return false;
if (!(vector =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, type, "vector"))) {
return false;
}
if (!(result = LLVMBuildNeg(comp_ctx->builder, vector, "neg"))) {
HANDLE_FAILURE("LLVMBuildNeg");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_i8x16_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef number;
if (!(number = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_i8x16_TYPE, "number"))) {
goto fail;
}
return simd_v128_integer_arith(comp_ctx, func_ctx, V128_NEG, number, NULL);
fail:
return false;
return simd_neg(comp_ctx, func_ctx, V128_i8x16_TYPE);
}
bool
aot_compile_simd_i16x8_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef number;
if (!(number = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_i16x8_TYPE, "number"))) {
goto fail;
}
return simd_v128_integer_arith(comp_ctx, func_ctx, V128_NEG, number, NULL);
fail:
return false;
return simd_neg(comp_ctx, func_ctx, V128_i16x8_TYPE);
}
bool
aot_compile_simd_i32x4_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef number;
if (!(number = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_i32x4_TYPE, "number"))) {
goto fail;
}
return simd_v128_integer_arith(comp_ctx, func_ctx, V128_NEG, number, NULL);
fail:
return false;
return simd_neg(comp_ctx, func_ctx, V128_i32x4_TYPE);
}
bool
aot_compile_simd_i64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef number;
POP_V128(number);
return simd_v128_integer_arith(comp_ctx, func_ctx, V128_NEG, number, NULL);
fail:
return false;
return simd_neg(comp_ctx, func_ctx, V128_i64x2_TYPE);
}
bool
aot_compile_simd_i8x16_popcnt(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMValueRef vector, result;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_i8x16_TYPE, "vector"))) {
return false;
}
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx,
"llvm.ctpop.v16i8", V128_i8x16_TYPE,
&V128_i8x16_TYPE, 1, vector))) {
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
static bool
simd_v128_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
V128Arithmetic arith_op,
bool is_signed)
{
LLVMValueRef lhs, rhs, result;
LLVMIntPredicate op;
if (!(rhs =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
return false;
}
if (V128_MIN == arith_op) {
op = is_signed ? LLVMIntSLT : LLVMIntULT;
}
else {
op = is_signed ? LLVMIntSGT : LLVMIntUGT;
}
if (!(result = LLVMBuildICmp(comp_ctx->builder, op, lhs, rhs, "cmp"))) {
HANDLE_FAILURE("LLVMBuildICmp");
return false;
}
if (!(result =
LLVMBuildSelect(comp_ctx->builder, result, lhs, rhs, "select"))) {
HANDLE_FAILURE("LLVMBuildSelect");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_i8x16_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
return simd_v128_cmp(comp_ctx, func_ctx, V128_i8x16_TYPE, arith_op,
is_signed);
}
bool
aot_compile_simd_i16x8_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
return simd_v128_cmp(comp_ctx, func_ctx, V128_i16x8_TYPE, arith_op,
is_signed);
}
bool
aot_compile_simd_i32x4_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
return simd_v128_cmp(comp_ctx, func_ctx, V128_i32x4_TYPE, arith_op,
is_signed);
}
/* llvm.abs.* */
static bool
simd_v128_abs(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
char *intrinsic,
LLVMTypeRef vector_type)
{
LLVMValueRef vector, result;
LLVMTypeRef param_types[] = { vector_type, INT1_TYPE };
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
return false;
}
if (!(result = aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic,
vector_type, param_types, 2, vector,
/* is_int_min_poison */
LLVM_CONST(i1_zero)))) {
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_i8x16_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v16i8",
V128_i8x16_TYPE);
}
bool
aot_compile_simd_i16x8_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v8i16",
V128_i16x8_TYPE);
}
bool
aot_compile_simd_i32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v4i32",
V128_i32x4_TYPE);
}
bool
aot_compile_simd_i64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, "llvm.abs.v2i64",
V128_i64x2_TYPE);
}
enum integer_avgr_u {
e_avgr_u_i8x16,
e_avgr_u_i16x8,
e_avgr_u_i32x4,
};
/* TODO: try int_x86_mmx_pavg_b and int_x86_mmx_pavg_w */
/* (v1 + v2 + 1) / 2 */
static bool
simd_v128_avg(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
enum integer_avgr_u itype)
{
LLVMValueRef lhs, rhs, ones, result;
LLVMTypeRef vector_ext_type;
LLVMTypeRef vector_type[] = {
V128_i8x16_TYPE,
V128_i16x8_TYPE,
V128_i32x4_TYPE,
};
unsigned lanes[] = { 16, 8, 4 };
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
vector_type[itype], "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
vector_type[itype], "lhs"))) {
return false;
}
if (!(vector_ext_type = LLVMVectorType(I64_TYPE, lanes[itype]))) {
HANDLE_FAILURE("LLVMVectorType");
return false;
}
if (!(lhs = LLVMBuildZExt(comp_ctx->builder, lhs, vector_ext_type,
"zext_to_i64"))
|| !(rhs = LLVMBuildZExt(comp_ctx->builder, rhs, vector_ext_type,
"zext_to_i64"))) {
HANDLE_FAILURE("LLVMBuildZExt");
return false;
}
/* by default, add will do signed/unsigned overflow */
if (!(result = LLVMBuildAdd(comp_ctx->builder, lhs, rhs, "l_add_r"))) {
HANDLE_FAILURE("LLVMBuildAdd");
return false;
}
if (!(ones = simd_build_splat_const_integer_vector(comp_ctx, I64_TYPE, 1,
lanes[itype]))) {
return false;
}
if (!(result = LLVMBuildAdd(comp_ctx->builder, result, ones, "plus_1"))) {
HANDLE_FAILURE("LLVMBuildAdd");
return false;
}
if (!(result = LLVMBuildLShr(comp_ctx->builder, result, ones, "avg"))) {
HANDLE_FAILURE("LLVMBuildLShr");
return false;
}
if (!(result = LLVMBuildTrunc(comp_ctx->builder, result,
vector_type[itype], "to_orig_type"))) {
HANDLE_FAILURE("LLVMBuildTrunc");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_i8x16_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i8x16);
}
bool
aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i16x8);
}
bool
aot_compile_simd_i32x4_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i32x4);
}
bool
aot_compile_simd_i32x4_dot_i16x8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
LLVMValueRef vec1, vec2, even_mask, odd_mask, zero, result;
LLVMTypeRef vector_ext_type;
LLVMValueRef even_element[] = {
LLVM_CONST(i32_zero),
LLVM_CONST(i32_two),
LLVM_CONST(i32_four),
LLVM_CONST(i32_six),
};
LLVMValueRef odd_element[] = {
LLVM_CONST(i32_one),
LLVM_CONST(i32_three),
LLVM_CONST(i32_five),
LLVM_CONST(i32_seven),
};
if (!(vec1 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, V128_i16x8_TYPE,
"vec1"))
|| !(vec2 = simd_pop_v128_and_bitcast(comp_ctx, func_ctx,
V128_i16x8_TYPE, "vec2"))) {
return false;
}
if (!(vector_ext_type = LLVMVectorType(I32_TYPE, 8))) {
HANDLE_FAILURE("LLVMVectorType");
return false;
}
/* sext <v8i16> to <v8i32> */
if (!(vec1 = LLVMBuildSExt(comp_ctx->builder, vec1, vector_ext_type,
"vec1_v8i32"))
|| !(vec2 = LLVMBuildSExt(comp_ctx->builder, vec2, vector_ext_type,
"vec2_v8i32"))) {
HANDLE_FAILURE("LLVMBuildSExt");
return false;
}
if (!(result = LLVMBuildMul(comp_ctx->builder, vec1, vec2, "product"))) {
HANDLE_FAILURE("LLVMBuildMul");
return false;
}
/* pick elements with even indexes and odd indexes */
if (!(even_mask = LLVMConstVector(even_element, 4))
|| !(odd_mask = LLVMConstVector(odd_element, 4))) {
HANDLE_FAILURE("LLVMConstVector");
return false;
}
if (!(zero =
simd_build_splat_const_integer_vector(comp_ctx, I32_TYPE, 0, 8))) {
return false;
}
if (!(vec1 = LLVMBuildShuffleVector(comp_ctx->builder, result, zero,
even_mask, "even_result"))
|| !(vec2 = LLVMBuildShuffleVector(comp_ctx->builder, result, zero,
odd_mask, "odd_result"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
return false;
}
if (!(result = LLVMBuildAdd(comp_ctx->builder, vec1, vec2, "new_vec"))) {
HANDLE_FAILURE("LLVMBuildAdd");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}

50
core/iwasm/compilation/simd/simd_int_arith.h
View File

@ -44,6 +44,56 @@ aot_compile_simd_i32x4_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i64x2_neg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i8x16_popcnt(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i8x16_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
bool
aot_compile_simd_i16x8_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
bool
aot_compile_simd_i32x4_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
bool
aot_compile_simd_i8x16_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i8x16_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i32x4_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i32x4_dot_i16x8(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
#ifdef __cplusplus
} /* end of extern "C" */
#endif

424
core/iwasm/compilation/simd/simd_load_store.c
View File

@ -3,6 +3,7 @@
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "simd_common.h"
#include "simd_load_store.h"
#include "../aot_emit_exception.h"
#include "../aot_emit_memory.h"
@ -23,68 +24,23 @@ simd_load(AOTCompContext *comp_ctx,
if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset,
data_length))) {
HANDLE_FAILURE("aot_check_memory_overflow");
goto fail;
return NULL;
}
if (!(maddr = LLVMBuildBitCast(comp_ctx->builder, maddr, ptr_type,
"data_ptr"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
return NULL;
}
if (!(data = LLVMBuildLoad(comp_ctx->builder, maddr, "data"))) {
HANDLE_FAILURE("LLVMBuildLoad");
goto fail;
return NULL;
}
LLVMSetAlignment(data, 1);
return data;
fail:
return NULL;
}
/* data_length in bytes */
static LLVMValueRef
simd_splat(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMValueRef element,
LLVMTypeRef vectory_type,
unsigned lane_count)
{
LLVMValueRef undef, zeros, vector;
LLVMTypeRef zeros_type;
if (!(undef = LLVMGetUndef(vectory_type))) {
HANDLE_FAILURE("LLVMGetUndef");
goto fail;
}
if (!(zeros_type = LLVMVectorType(I32_TYPE, lane_count))) {
HANDLE_FAILURE("LVMVectorType");
goto fail;
}
if (!(zeros = LLVMConstNull(zeros_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
if (!(vector = LLVMBuildInsertElement(comp_ctx->builder, undef, element,
I32_ZERO, "base"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
goto fail;
}
if (!(vector = LLVMBuildShuffleVector(comp_ctx->builder, vector, undef,
zeros, "vector"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
goto fail;
}
return vector;
fail:
return NULL;
}
bool
@ -97,40 +53,10 @@ aot_compile_simd_v128_load(AOTCompContext *comp_ctx,
if (!(result =
simd_load(comp_ctx, func_ctx, align, offset, 16, V128_PTR_TYPE))) {
goto fail;
return false;
}
PUSH_V128(result);
return true;
fail:
return false;
}
bool
aot_compile_simd_v128_store(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint32 align,
uint32 offset)
{
LLVMValueRef maddr, value, result;
POP_V128(value);
if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset, 16)))
return false;
if (!(maddr = LLVMBuildBitCast(comp_ctx->builder, maddr, V128_PTR_TYPE,
"data_ptr"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
if (!(result = LLVMBuildStore(comp_ctx->builder, value, maddr))) {
HANDLE_FAILURE("LLVMBuildStore");
goto fail;
}
LLVMSetAlignment(result, 1);
return true;
fail:
@ -140,162 +66,272 @@ fail:
bool
aot_compile_simd_load_extend(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 load_opcode,
uint8 opcode,
uint32 align,
uint32 offset)
{
LLVMValueRef sub_vector, result;
LLVMTypeRef sub_vector_type, vector_type;
bool is_signed;
uint32 data_length;
switch (load_opcode) {
case SIMD_i16x8_load8x8_s:
case SIMD_i16x8_load8x8_u:
{
data_length = 8;
vector_type = V128_i16x8_TYPE;
is_signed = (load_opcode == SIMD_i16x8_load8x8_s);
if (!(sub_vector_type = LLVMVectorType(INT8_TYPE, 8))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
case SIMD_i32x4_load16x4_s:
case SIMD_i32x4_load16x4_u:
{
data_length = 8;
vector_type = V128_i32x4_TYPE;
is_signed = (load_opcode == SIMD_i32x4_load16x4_s);
if (!(sub_vector_type = LLVMVectorType(INT16_TYPE, 4))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
case SIMD_i64x2_load32x2_s:
case SIMD_i64x2_load32x2_u:
{
data_length = 8;
vector_type = V128_i64x2_TYPE;
is_signed = (load_opcode == SIMD_i64x2_load32x2_s);
if (!(sub_vector_type = LLVMVectorType(I32_TYPE, 2))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
break;
}
default:
{
bh_assert(0);
goto fail;
}
}
uint32 opcode_index = opcode - SIMD_v128_load8x8_s;
bool signeds[] = { true, false, true, false, true, false };
LLVMTypeRef vector_types[] = {
V128_i16x8_TYPE, V128_i16x8_TYPE, V128_i32x4_TYPE,
V128_i32x4_TYPE, V128_i64x2_TYPE, V128_i64x2_TYPE,
};
LLVMTypeRef sub_vector_types[] = {
LLVMVectorType(INT8_TYPE, 8), LLVMVectorType(INT8_TYPE, 8),
LLVMVectorType(INT16_TYPE, 4), LLVMVectorType(INT16_TYPE, 4),
LLVMVectorType(I32_TYPE, 2), LLVMVectorType(I32_TYPE, 2),
};
LLVMTypeRef sub_vector_type = sub_vector_types[opcode_index];
/* to vector ptr type */
if (!(sub_vector_type = LLVMPointerType(sub_vector_type, 0))) {
if (!sub_vector_type
|| !(sub_vector_type = LLVMPointerType(sub_vector_type, 0))) {
HANDLE_FAILURE("LLVMPointerType");
goto fail;
return false;
}
if (!(sub_vector = simd_load(comp_ctx, func_ctx, align, offset,
data_length, sub_vector_type))) {
goto fail;
if (!(sub_vector = simd_load(comp_ctx, func_ctx, align, offset, 8,
sub_vector_type))) {
return false;
}
if (is_signed) {
if (signeds[opcode_index]) {
if (!(result = LLVMBuildSExt(comp_ctx->builder, sub_vector,
vector_type, "vector"))) {
vector_types[opcode_index], "vector"))) {
HANDLE_FAILURE("LLVMBuildSExt");
goto fail;
return false;
}
}
else {
if (!(result = LLVMBuildZExt(comp_ctx->builder, sub_vector,
vector_type, "vector"))) {
vector_types[opcode_index], "vector"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
return false;
}
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"result"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
PUSH_V128(result);
return true;
fail:
return false;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_load_splat(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 load_opcode,
uint8 opcode,
uint32 align,
uint32 offset)
{
uint32 opcode_index = opcode - SIMD_v128_load8_splat;
LLVMValueRef element, result;
LLVMTypeRef element_ptr_type, vector_type;
unsigned data_length, lane_count;
LLVMTypeRef element_ptr_types[] = { INT8_PTR_TYPE, INT16_PTR_TYPE,
INT32_PTR_TYPE, INT64_PTR_TYPE };
uint32 data_lengths[] = { 1, 2, 4, 8 };
LLVMValueRef undefs[] = {
LLVM_CONST(i8x16_undef),
LLVM_CONST(i16x8_undef),
LLVM_CONST(i32x4_undef),
LLVM_CONST(i64x2_undef),
};
LLVMValueRef masks[] = {
LLVM_CONST(i32x16_zero),
LLVM_CONST(i32x8_zero),
LLVM_CONST(i32x4_zero),
LLVM_CONST(i32x2_zero),
};
switch (load_opcode) {
case SIMD_v8x16_load_splat:
data_length = 1;
lane_count = 16;
element_ptr_type = INT8_PTR_TYPE;
vector_type = V128_i8x16_TYPE;
break;
case SIMD_v16x8_load_splat:
data_length = 2;
lane_count = 8;
element_ptr_type = INT16_PTR_TYPE;
vector_type = V128_i16x8_TYPE;
break;
case SIMD_v32x4_load_splat:
data_length = 4;
lane_count = 4;
element_ptr_type = INT32_PTR_TYPE;
vector_type = V128_i32x4_TYPE;
break;
case SIMD_v64x2_load_splat:
data_length = 8;
lane_count = 2;
element_ptr_type = INT64_PTR_TYPE;
vector_type = V128_i64x2_TYPE;
break;
default:
bh_assert(0);
goto fail;
if (!(element = simd_load(comp_ctx, func_ctx, align, offset,
data_lengths[opcode_index],
element_ptr_types[opcode_index]))) {
return false;
}
if (!(element = simd_load(comp_ctx, func_ctx, align, offset, data_length,
element_ptr_type))) {
goto fail;
if (!(result =
LLVMBuildInsertElement(comp_ctx->builder, undefs[opcode_index],
element, I32_ZERO, "base"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
return false;
}
if (!(result = simd_splat(comp_ctx, func_ctx, element, vector_type,
lane_count))) {
goto fail;
if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, result,
undefs[opcode_index],
masks[opcode_index], "vector"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
return false;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"result"))) {
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
aot_compile_simd_load_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 opcode,
uint32 align,
uint32 offset,
uint8 lane_id)
{
LLVMValueRef element, vector;
uint32 opcode_index = opcode - SIMD_v128_load8_lane;
uint32 data_lengths[] = { 1, 2, 4, 8 };
LLVMTypeRef element_ptr_types[] = { INT8_PTR_TYPE, INT16_PTR_TYPE,
INT32_PTR_TYPE, INT64_PTR_TYPE };
LLVMTypeRef vector_types[] = { V128_i8x16_TYPE, V128_i16x8_TYPE,
V128_i32x4_TYPE, V128_i64x2_TYPE };
LLVMValueRef lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id);
if (!(vector = simd_pop_v128_and_bitcast(
comp_ctx, func_ctx, vector_types[opcode_index], "src"))) {
return false;
}
if (!(element = simd_load(comp_ctx, func_ctx, align, offset,
data_lengths[opcode_index],
element_ptr_types[opcode_index]))) {
return false;
}
if (!(vector = LLVMBuildInsertElement(comp_ctx->builder, vector, element,
lane, "dst"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, vector, "result");
}
bool
aot_compile_simd_load_zero(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 opcode,
uint32 align,
uint32 offset)
{
LLVMValueRef element, result, mask;
uint32 opcode_index = opcode - SIMD_v128_load32_zero;
uint32 data_lengths[] = { 4, 8 };
LLVMTypeRef element_ptr_types[] = { INT32_PTR_TYPE, INT64_PTR_TYPE };
LLVMValueRef zero[] = {
LLVM_CONST(i32x4_vec_zero),
LLVM_CONST(i64x2_vec_zero),
};
LLVMValueRef undef[] = {
LLVM_CONST(i32x4_undef),
LLVM_CONST(i64x2_undef),
};
uint32 mask_length[] = { 4, 2 };
LLVMValueRef mask_element[][4] = {
{ LLVM_CONST(i32_zero), LLVM_CONST(i32_four), LLVM_CONST(i32_five),
LLVM_CONST(i32_six) },
{ LLVM_CONST(i32_zero), LLVM_CONST(i32_two) },
};
if (!(element = simd_load(comp_ctx, func_ctx, align, offset,
data_lengths[opcode_index],
element_ptr_types[opcode_index]))) {
return false;
}
if (!(result =
LLVMBuildInsertElement(comp_ctx->builder, undef[opcode_index],
element, I32_ZERO, "vector"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
return false;
}
/* fill in other lanes with zero */
if (!(mask = LLVMConstVector(mask_element[opcode_index],
mask_length[opcode_index]))) {
HANDLE_FAILURE("LLConstVector");
return false;
}
if (!(result = LLVMBuildShuffleVector(comp_ctx->builder, result,
zero[opcode_index], mask,
"fill_in_zero"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
return false;
}
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
/* data_length in bytes */
static bool
simd_store(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint32 align,
uint32 offset,
uint32 data_length,
LLVMValueRef value,
LLVMTypeRef value_ptr_type)
{
LLVMValueRef maddr, result;
if (!(maddr = aot_check_memory_overflow(comp_ctx, func_ctx, offset,
data_length)))
return false;
if (!(maddr = LLVMBuildBitCast(comp_ctx->builder, maddr, value_ptr_type,
"data_ptr"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
return false;
}
PUSH_V128(result);
if (!(result = LLVMBuildStore(comp_ctx->builder, value, maddr))) {
HANDLE_FAILURE("LLVMBuildStore");
return false;
}
LLVMSetAlignment(result, 1);
return true;
}
bool
aot_compile_simd_v128_store(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint32 align,
uint32 offset)
{
LLVMValueRef value;
POP_V128(value);
return simd_store(comp_ctx, func_ctx, align, offset, 16, value,
V128_PTR_TYPE);
fail:
return false;
}
bool
aot_compile_simd_store_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 opcode,
uint32 align,
uint32 offset,
uint8 lane_id)
{
LLVMValueRef element, vector;
uint32 data_lengths[] = { 1, 2, 4, 8 };
LLVMTypeRef element_ptr_types[] = { INT8_PTR_TYPE, INT16_PTR_TYPE,
INT32_PTR_TYPE, INT64_PTR_TYPE };
uint32 opcode_index = opcode - SIMD_v128_store8_lane;
LLVMTypeRef vector_types[] = { V128_i8x16_TYPE, V128_i16x8_TYPE,
V128_i32x4_TYPE, V128_i64x2_TYPE };
LLVMValueRef lane = simd_lane_id_to_llvm_value(comp_ctx, lane_id);
if (!(vector = simd_pop_v128_and_bitcast(
comp_ctx, func_ctx, vector_types[opcode_index], "src"))) {
return false;
}
if (!(element = LLVMBuildExtractElement(comp_ctx->builder, vector, lane,
"element"))) {
HANDLE_FAILURE("LLVMBuildExtractElement");
return false;
}
return simd_store(comp_ctx, func_ctx, align, offset,
data_lengths[opcode_index], element,
element_ptr_types[opcode_index]);
}

39
core/iwasm/compilation/simd/simd_load_store.h
View File

@ -18,26 +18,49 @@ aot_compile_simd_v128_load(AOTCompContext *comp_ctx,
uint32 align,
uint32 offset);
bool
aot_compile_simd_v128_store(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint32 align,
uint32 offset);
bool
aot_compile_simd_load_extend(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 load_opcode,
uint8 opcode,
uint32 align,
uint32 offset);
bool
aot_compile_simd_load_splat(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 load_opcode,
uint8 opcode,
uint32 align,
uint32 offset);
bool
aot_compile_simd_load_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 opcode,
uint32 align,
uint32 offset,
uint8 lane_id);
bool
aot_compile_simd_load_zero(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 opcode,
uint32 align,
uint32 offset);
bool
aot_compile_simd_v128_store(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint32 align,
uint32 offset);
bool
aot_compile_simd_store_lane(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
uint8 opcode,
uint32 align,
uint32 offset,
uint8 lane_id);
#ifdef __cplusplus
} /* end of extern "C" */
#endif

361
core/iwasm/compilation/simd/simd_sat_int_arith.c
View File

@ -9,46 +9,32 @@
#include "../../aot/aot_runtime.h"
static bool
simd_v128_integer_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
char *intrinsics_s_u[2],
bool is_signed)
simd_sat_int_arith(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
const char *intrinsics)
{
LLVMValueRef lhs, rhs, result;
LLVMTypeRef param_types[2];
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
goto fail;
if (!(rhs =
simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type, "rhs"))
|| !(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
return false;
}
param_types[0] = vector_type;
param_types[1] = vector_type;
if (!(result = aot_call_llvm_intrinsic(
comp_ctx, func_ctx, is_signed ? intrinsics_s_u[0] : intrinsics_s_u[1],
vector_type, param_types, 2, lhs, rhs))) {
if (!(result =
aot_call_llvm_intrinsic(comp_ctx, func_ctx, intrinsics,
vector_type, param_types, 2, lhs, rhs))) {
HANDLE_FAILURE("LLVMBuildCall");
goto fail;
return false;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
return simd_bitcast_and_push_v128(comp_ctx, func_ctx, result, "result");
}
bool
@ -57,27 +43,14 @@ aot_compile_simd_i8x16_saturate(AOTCompContext *comp_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
char *intrinsics[2] = { 0 };
bool result = false;
switch (arith_op) {
case V128_ADD:
intrinsics[0] = "llvm.sadd.sat.v16i8";
intrinsics[1] = "llvm.uadd.sat.v16i8";
result = simd_v128_integer_arith(
comp_ctx, func_ctx, V128_i8x16_TYPE, intrinsics, is_signed);
break;
case V128_SUB:
intrinsics[0] = "llvm.ssub.sat.v16i8";
intrinsics[1] = "llvm.usub.sat.v16i8";
result = simd_v128_integer_arith(
comp_ctx, func_ctx, V128_i8x16_TYPE, intrinsics, is_signed);
break;
default:
bh_assert(0);
break;
}
char *intrinsics[][2] = {
{ "llvm.sadd.sat.v16i8", "llvm.uadd.sat.v16i8" },
{ "llvm.ssub.sat.v16i8", "llvm.usub.sat.v16i8" },
};
return result;
return simd_sat_int_arith(comp_ctx, func_ctx, V128_i8x16_TYPE,
is_signed ? intrinsics[arith_op][0]
: intrinsics[arith_op][1]);
}
bool
@ -86,282 +59,28 @@ aot_compile_simd_i16x8_saturate(AOTCompContext *comp_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
char *intrinsics[2] = { 0 };
bool result = false;
switch (arith_op) {
case V128_ADD:
intrinsics[0] = "llvm.sadd.sat.v8i16";
intrinsics[1] = "llvm.uadd.sat.v8i16";
result = simd_v128_integer_arith(
comp_ctx, func_ctx, V128_i16x8_TYPE, intrinsics, is_signed);
break;
case V128_SUB:
intrinsics[0] = "llvm.ssub.sat.v8i16";
intrinsics[1] = "llvm.usub.sat.v8i16";
result = simd_v128_integer_arith(
comp_ctx, func_ctx, V128_i16x8_TYPE, intrinsics, is_signed);
break;
default:
bh_assert(0);
break;
}
char *intrinsics[][2] = {
{ "llvm.sadd.sat.v8i16", "llvm.uadd.sat.v8i16" },
{ "llvm.ssub.sat.v8i16", "llvm.usub.sat.v8i16" },
};
return result;
}
static bool
simd_v128_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
V128Arithmetic arith_op,
bool is_signed)
{
LLVMValueRef lhs, rhs, result;
LLVMIntPredicate op;
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
goto fail;
}
if (V128_MIN == arith_op) {
op = is_signed ? LLVMIntSLT : LLVMIntULT;
}
else {
op = is_signed ? LLVMIntSGT : LLVMIntUGT;
}
if (!(result = LLVMBuildICmp(comp_ctx->builder, op, lhs, rhs, "cmp"))) {
HANDLE_FAILURE("LLVMBuildICmp");
goto fail;
}
if (!(result =
LLVMBuildSelect(comp_ctx->builder, result, lhs, rhs, "select"))) {
HANDLE_FAILURE("LLVMBuildSelect");
goto fail;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
return simd_sat_int_arith(comp_ctx, func_ctx, V128_i16x8_TYPE,
is_signed ? intrinsics[arith_op][0]
: intrinsics[arith_op][1]);
}
bool
aot_compile_simd_i8x16_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
aot_compile_simd_i32x4_saturate(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
return simd_v128_cmp(comp_ctx, func_ctx, V128_i8x16_TYPE, arith_op,
is_signed);
char *intrinsics[][2] = {
{ "llvm.sadd.sat.v4i32", "llvm.uadd.sat.v4i32" },
{ "llvm.ssub.sat.v4i32", "llvm.usub.sat.v4i32" },
};
return simd_sat_int_arith(comp_ctx, func_ctx, V128_i16x8_TYPE,
is_signed ? intrinsics[arith_op][0]
: intrinsics[arith_op][1]);
}
bool
aot_compile_simd_i16x8_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
return simd_v128_cmp(comp_ctx, func_ctx, V128_i16x8_TYPE, arith_op,
is_signed);
}
bool
aot_compile_simd_i32x4_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed)
{
return simd_v128_cmp(comp_ctx, func_ctx, V128_i32x4_TYPE, arith_op,
is_signed);
}
static bool
simd_v128_abs(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type)
{
LLVMValueRef vector, negs, zeros, cond, result;
if (!(vector = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"vec"))) {
goto fail;
}
if (!(negs = LLVMBuildNeg(comp_ctx->builder, vector, "neg"))) {
HANDLE_FAILURE("LLVMBuildNeg");
goto fail;
}
if (!(zeros = LLVMConstNull(vector_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
if (!(cond = LLVMBuildICmp(comp_ctx->builder, LLVMIntSGE, vector, zeros,
"ge_zero"))) {
HANDLE_FAILURE("LLVMBuildICmp");
goto fail;
}
if (!(result = LLVMBuildSelect(comp_ctx->builder, cond, vector, negs,
"select"))) {
HANDLE_FAILURE("LLVMBuildSelect");
goto fail;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
}
bool
aot_compile_simd_i8x16_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, V128_i8x16_TYPE);
}
bool
aot_compile_simd_i16x8_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, V128_i16x8_TYPE);
}
bool
aot_compile_simd_i32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
return simd_v128_abs(comp_ctx, func_ctx, V128_i32x4_TYPE);
}
/* (v1 + v2 + 1) / 2 */
static bool
simd_v128_avg(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
LLVMTypeRef vector_type,
LLVMTypeRef element_type,
unsigned lane_width)
{
LLVMValueRef lhs, rhs, undef, zeros, ones, result;
LLVMTypeRef ext_type;
if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"rhs"))) {
goto fail;
}
if (!(lhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type,
"lhs"))) {
goto fail;
}
if (!(ext_type = LLVMVectorType(I32_TYPE, lane_width))) {
HANDLE_FAILURE("LLVMVectorType");
goto fail;
}
if (!(lhs = LLVMBuildZExt(comp_ctx->builder, lhs, ext_type, "left_ext"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
if (!(rhs =
LLVMBuildZExt(comp_ctx->builder, rhs, ext_type, "right_ext"))) {
HANDLE_FAILURE("LLVMBuildZExt");
goto fail;
}
if (!(undef = LLVMGetUndef(ext_type))) {
HANDLE_FAILURE("LLVMGetUndef");
goto fail;
}
if (!(zeros = LLVMConstNull(ext_type))) {
HANDLE_FAILURE("LLVMConstNull");
goto fail;
}
if (!(ones = LLVMConstInt(I32_TYPE, 1, true))) {
HANDLE_FAILURE("LLVMConstInt");
goto fail;
}
if (!(ones = LLVMBuildInsertElement(comp_ctx->builder, undef, ones,
I32_ZERO, "base_ones"))) {
HANDLE_FAILURE("LLVMBuildInsertElement");
goto fail;
}
if (!(ones = LLVMBuildShuffleVector(comp_ctx->builder, ones, undef, zeros,
"ones"))) {
HANDLE_FAILURE("LLVMBuildShuffleVector");
goto fail;
}
if (!(result = LLVMBuildAdd(comp_ctx->builder, lhs, rhs, "a_add_b"))) {
HANDLE_FAILURE("LLVMBuildAdd");
goto fail;
}
if (!(result = LLVMBuildAdd(comp_ctx->builder, result, ones, "plus_1"))) {
HANDLE_FAILURE("LLVMBuildAdd");
goto fail;
}
if (!(result = LLVMBuildLShr(comp_ctx->builder, result, ones, "avg"))) {
HANDLE_FAILURE("LLVMBuildLShr");
goto fail;
}
if (!(result = LLVMBuildTrunc(comp_ctx->builder, result, vector_type,
"avg_trunc"))) {
HANDLE_FAILURE("LLVMBuildTrunc");
goto fail;
}
if (!(result = LLVMBuildBitCast(comp_ctx->builder, result, V128_i64x2_TYPE,
"ret"))) {
HANDLE_FAILURE("LLVMBuildBitCast");
goto fail;
}
/* push result into the stack */
PUSH_V128(result);
return true;
fail:
return false;
}
bool
aot_compile_simd_i8x16_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_avg(comp_ctx, func_ctx, V128_i8x16_TYPE, INT8_TYPE, 16);
}
bool
aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx)
{
return simd_v128_avg(comp_ctx, func_ctx, V128_i16x8_TYPE, INT16_TYPE, 8);
}

38
core/iwasm/compilation/simd/simd_sat_int_arith.h
View File

@ -25,40 +25,10 @@ aot_compile_simd_i16x8_saturate(AOTCompContext *comp_ctx,
bool is_signed);
bool
aot_compile_simd_i8x16_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
bool
aot_compile_simd_i16x8_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
bool
aot_compile_simd_i32x4_cmp(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
bool
aot_compile_simd_i8x16_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx);
bool
aot_compile_simd_i8x16_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
bool
aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx);
aot_compile_simd_i32x4_saturate(AOTCompContext *comp_ctx,
AOTFuncContext *func_ctx,
V128Arithmetic arith_op,
bool is_signed);
#ifdef __cplusplus
} /* end of extern "C" */
#endif