diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 22040615..a0e26759 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -397,7 +397,7 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, return is_timeout ? 2 : 0; } -uint8 +uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count) { diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index 5e863c19..f05e5957 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -57,7 +57,7 @@ uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64); -uint8 +uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 66c257c8..569f0466 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -9,6 +9,9 @@ #include "wasm_opcode.h" #include "wasm_loader.h" #include "../common/wasm_exec_env.h" +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif typedef int32 CellType_I32; typedef int64 CellType_I64; @@ -243,6 +246,11 @@ LOAD_I16(void *addr) goto out_of_bounds; \ } while (0) +#define CHECK_ATOMIC_MEMORY_ACCESS() do { \ + if (((uintptr_t)maddr & ((1 << align) - 1)) != 0) \ + goto unaligned_atomic; \ + } while (0) + static inline uint32 rotl32(uint32 n, uint32 c) { @@ -748,6 +756,98 @@ trunc_f64_to_int(WASMModuleInstance *module, local_type = cur_func->local_types[local_idx - param_count]; \ } while (0) +#define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U: \ + { \ + uint32 readv, sval; \ + \ + sval = POP_I32(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint32)(*(uint8*)maddr); \ + *(uint8*)maddr = (uint8)(readv op sval); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint32)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = LOAD_I32(maddr); \ + STORE_U32(maddr, readv op sval); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + PUSH_I32(readv); \ + break; \ + } \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U: \ + { \ + uint64 readv, sval; \ + \ + sval = (uint64)POP_I64(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)(*(uint8*)maddr); \ + *(uint8*)maddr = (uint8)(readv op sval); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)LOAD_U32(maddr); \ + STORE_U32(maddr, (uint32)(readv op sval)); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else { \ + uint64 op_result; \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)LOAD_I64(maddr); \ + op_result = readv op sval; \ + STORE_I64(maddr, op_result); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + PUSH_I64(readv); \ + break; \ + } + static inline int32 sign_ext_8_32(int8 val) { @@ -2684,6 +2784,336 @@ label_pop_csp_n: HANDLE_OP_END (); } +#if WASM_ENABLE_SHARED_MEMORY != 0 + HANDLE_OP (WASM_OP_ATOMIC_PREFIX): + { + uint32 offset, align; + int32 addr; + + opcode = *frame_ip++; + + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + switch (opcode) { + case WASM_OP_ATOMIC_NOTIFY: + { + uint32 count, ret; + + count = POP_I32(); + addr = POP_I32(); + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + ret = wasm_runtime_atomic_notify((WASMModuleInstanceCommon*)module, + maddr, count); + bh_assert((int32)ret >= 0); + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT32: + { + uint64 timeout; + uint32 expect, addr, ret; + + timeout = POP_I64(); + expect = POP_I32(); + addr = POP_I32(); + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + ret = wasm_runtime_atomic_wait((WASMModuleInstanceCommon*)module, maddr, + (uint64)expect, timeout, false); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT64: + { + uint64 timeout, expect; + uint32 ret; + + timeout = POP_I64(); + expect = POP_I64(); + addr = POP_I32(); + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + ret = wasm_runtime_atomic_wait((WASMModuleInstanceCommon*)module, + maddr, expect, timeout, true); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + { + uint32 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = (uint32)(*(uint8*)maddr); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = (uint32)LOAD_U16(maddr); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = LOAD_I32(maddr); + os_mutex_unlock(&memory->mem_lock); + } + + PUSH_I32(readv); + break; + } + + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + { + uint64 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = (uint64)(*(uint8*)maddr); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U16(maddr); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U32(maddr); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + readv = LOAD_I64(maddr); + os_mutex_unlock(&memory->mem_lock); + } + + PUSH_I64(readv); + break; + } + + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + { + uint32 sval; + + sval = (uint32)POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_STORE8) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + *(uint8*)maddr = (uint8)sval; + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + STORE_U16(maddr, (uint16)sval); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + STORE_U32(maddr, frame_sp[1]); + os_mutex_unlock(&memory->mem_lock); + } + break; + } + + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + { + uint64 sval; + + sval = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_STORE8) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + *(uint8*)maddr = (uint8)sval; + os_mutex_unlock(&memory->mem_lock); + } + else if(opcode == WASM_OP_ATOMIC_I64_STORE16) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + STORE_U16(maddr, (uint16)sval); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + STORE_U32(maddr, (uint32)sval); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + os_mutex_lock(&memory->mem_lock); + STORE_U32(maddr, frame_sp[1]); + STORE_U32(maddr + 4, frame_sp[2]); + os_mutex_unlock(&memory->mem_lock); + } + break; + } + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + { + uint32 readv, sval, expect; + + sval = POP_I32(); + expect = POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = (uint32)(*(uint8*)maddr); + if (readv == expect) + *(uint8*)maddr = (uint8)(sval); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = (uint32)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = LOAD_I32(maddr); + if (readv == expect) + STORE_U32(maddr, sval); + os_mutex_unlock(&memory->mem_lock); + } + PUSH_I32(readv); + break; + } + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + { + uint64 readv, sval, expect; + + sval = (uint64)POP_I64(); + expect = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)(*(uint8*)maddr); + if (readv == expect) + *(uint8*)maddr = (uint8)(sval); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U32(maddr); + if (readv == expect) + STORE_U32(maddr, (uint32)(sval)); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_I64(maddr); + if (readv == expect) { + STORE_I64(maddr, sval); + } + os_mutex_unlock(&memory->mem_lock); + } + PUSH_I64(readv); + break; + } + + DEF_ATOMIC_RMW_OPCODE(ADD, +); + DEF_ATOMIC_RMW_OPCODE(SUB, -); + DEF_ATOMIC_RMW_OPCODE(AND, &); + DEF_ATOMIC_RMW_OPCODE(OR, |); + DEF_ATOMIC_RMW_OPCODE(XOR, ^); + /* xchg, ignore the read value, and store the given value: + readv * 0 + sval */ + DEF_ATOMIC_RMW_OPCODE(XCHG, *0 +); + } + + HANDLE_OP_END (); + } +#endif + HANDLE_OP (WASM_OP_IMPDEP): frame = prev_frame; frame_ip = frame->ip; @@ -2827,6 +3257,12 @@ label_pop_csp_n: HANDLE_OP_END (); } +#if WASM_ENABLE_SHARED_MEMORY != 0 + unaligned_atomic: + wasm_set_exception(module, "unaligned atomic"); + goto got_exception; +#endif + out_of_bounds: wasm_set_exception(module, "out of bounds memory access"); diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 2e31552a..166dc107 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -9,6 +9,9 @@ #include "wasm_opcode.h" #include "wasm_loader.h" #include "../common/wasm_exec_env.h" +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif typedef int32 CellType_I32; typedef int64 CellType_I64; @@ -245,6 +248,11 @@ LOAD_I16(void *addr) goto out_of_bounds; \ } while (0) +#define CHECK_ATOMIC_MEMORY_ACCESS(align) do { \ + if (((uintptr_t)maddr & (align - 1)) != 0) \ + goto unaligned_atomic; \ + } while (0) + static inline uint32 rotl32(uint32 n, uint32 c) { @@ -540,6 +548,98 @@ read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign) frame_ip += 6; \ } while (0) +#define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U: \ + { \ + uint32 readv, sval; \ + \ + sval = POP_I32(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(1); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint32)(*(uint8*)maddr); \ + *(uint8*)maddr = (uint8)(readv op sval); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(2); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint32)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(4); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = LOAD_I32(maddr); \ + STORE_U32(maddr, readv op sval); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + PUSH_I32(readv); \ + break; \ + } \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U: \ + case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U: \ + { \ + uint64 readv, sval; \ + \ + sval = (uint64)POP_I64(); \ + addr = POP_I32(); \ + \ + if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(1); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)(*(uint8*)maddr); \ + *(uint8*)maddr = (uint8)(readv op sval); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(2); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)LOAD_U16(maddr); \ + STORE_U16(maddr, (uint16)(readv op sval)); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(4); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)LOAD_U32(maddr); \ + STORE_U32(maddr, (uint32)(readv op sval)); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + else { \ + uint64 op_result; \ + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); \ + CHECK_ATOMIC_MEMORY_ACCESS(8); \ + \ + os_mutex_lock(&memory->mem_lock); \ + readv = (uint64)LOAD_I64(maddr); \ + op_result = readv op sval; \ + STORE_I64(maddr, op_result); \ + os_mutex_unlock(&memory->mem_lock); \ + } \ + PUSH_I64(readv); \ + break; \ + } + #define DEF_OP_MATH(src_type, src_op_type, method) do { \ SET_OPERAND(src_type, 2, method(GET_OPERAND(src_type, 0))); \ frame_ip += 4; \ @@ -2665,6 +2765,334 @@ recover_br_info: HANDLE_OP_END (); } +#if WASM_ENABLE_SHARED_MEMORY != 0 + HANDLE_OP (WASM_OP_ATOMIC_PREFIX): + { + uint32 offset; + int32 addr; + + GET_OPCODE(); + + offset = read_uint32(frame_ip); + + switch (opcode) { + case WASM_OP_ATOMIC_NOTIFY: + { + uint32 count, ret; + + count = POP_I32(); + addr = POP_I32(); + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + ret = wasm_runtime_atomic_notify((WASMModuleInstanceCommon*)module, + maddr, count); + bh_assert((int32)ret >= 0); + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT32: + { + uint64 timeout; + uint32 expect, addr, ret; + + timeout = POP_I64(); + expect = POP_I32(); + addr = POP_I32(); + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + ret = wasm_runtime_atomic_wait((WASMModuleInstanceCommon*)module, maddr, + (uint64)expect, timeout, false); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + case WASM_OP_ATOMIC_WAIT64: + { + uint64 timeout, expect; + uint32 ret; + + timeout = POP_I64(); + expect = POP_I64(); + addr = POP_I32(); + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(8); + + ret = wasm_runtime_atomic_wait((WASMModuleInstanceCommon*)module, + maddr, expect, timeout, true); + if (ret == (uint32)-1) + goto got_exception; + + PUSH_I32(ret); + break; + } + + case WASM_OP_ATOMIC_I32_LOAD: + case WASM_OP_ATOMIC_I32_LOAD8_U: + case WASM_OP_ATOMIC_I32_LOAD16_U: + { + uint32 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(1); + os_mutex_lock(&memory->mem_lock); + readv = (uint32)(*(uint8*)maddr); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(2); + os_mutex_lock(&memory->mem_lock); + readv = (uint32)LOAD_U16(maddr); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + os_mutex_lock(&memory->mem_lock); + readv = LOAD_I32(maddr); + os_mutex_unlock(&memory->mem_lock); + } + + PUSH_I32(readv); + break; + } + + case WASM_OP_ATOMIC_I64_LOAD: + case WASM_OP_ATOMIC_I64_LOAD8_U: + case WASM_OP_ATOMIC_I64_LOAD16_U: + case WASM_OP_ATOMIC_I64_LOAD32_U: + { + uint64 readv; + + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(1); + os_mutex_lock(&memory->mem_lock); + readv = (uint64)(*(uint8*)maddr); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(2); + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U16(maddr); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U32(maddr); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(8); + os_mutex_lock(&memory->mem_lock); + readv = LOAD_I64(maddr); + os_mutex_unlock(&memory->mem_lock); + } + + PUSH_I64(readv); + break; + } + case WASM_OP_ATOMIC_I32_STORE: + case WASM_OP_ATOMIC_I32_STORE8: + case WASM_OP_ATOMIC_I32_STORE16: + { + uint32 sval; + + sval = (uint32)POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I32_STORE8) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(1); + os_mutex_lock(&memory->mem_lock); + *(uint8*)maddr = (uint8)sval; + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(2); + os_mutex_lock(&memory->mem_lock); + STORE_U16(maddr, (uint16)sval); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + os_mutex_lock(&memory->mem_lock); + STORE_U32(maddr, sval); + os_mutex_unlock(&memory->mem_lock); + } + break; + } + + case WASM_OP_ATOMIC_I64_STORE: + case WASM_OP_ATOMIC_I64_STORE8: + case WASM_OP_ATOMIC_I64_STORE16: + case WASM_OP_ATOMIC_I64_STORE32: + { + uint64 sval; + + sval = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_I64_STORE8) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(1); + os_mutex_lock(&memory->mem_lock); + *(uint8*)maddr = (uint8)sval; + os_mutex_unlock(&memory->mem_lock); + } + else if(opcode == WASM_OP_ATOMIC_I64_STORE16) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(2); + os_mutex_lock(&memory->mem_lock); + STORE_U16(maddr, (uint16)sval); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + os_mutex_lock(&memory->mem_lock); + STORE_U32(maddr, (uint32)sval); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(8); + os_mutex_lock(&memory->mem_lock); + STORE_I64(maddr, sval); + os_mutex_unlock(&memory->mem_lock); + } + break; + } + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + { + uint32 readv, sval, expect; + + sval = POP_I32(); + expect = POP_I32(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(1); + + os_mutex_lock(&memory->mem_lock); + readv = (uint32)(*(uint8*)maddr); + if (readv == expect) + *(uint8*)maddr = (uint8)(sval); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(2); + + os_mutex_lock(&memory->mem_lock); + readv = (uint32)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + os_mutex_lock(&memory->mem_lock); + readv = LOAD_I32(maddr); + if (readv == expect) + STORE_U32(maddr, sval); + os_mutex_unlock(&memory->mem_lock); + } + PUSH_I32(readv); + break; + } + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + { + uint64 readv, sval, expect; + + sval = (uint64)POP_I64(); + expect = (uint64)POP_I64(); + addr = POP_I32(); + + if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 1, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(1); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)(*(uint8*)maddr); + if (readv == expect) + *(uint8*)maddr = (uint8)(sval); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 2, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(2); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U16(maddr); + if (readv == expect) + STORE_U16(maddr, (uint16)(sval)); + os_mutex_unlock(&memory->mem_lock); + } + else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 4, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(4); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_U32(maddr); + if (readv == expect) + STORE_U32(maddr, (uint32)(sval)); + os_mutex_unlock(&memory->mem_lock); + } + else { + CHECK_BULK_MEMORY_OVERFLOW(addr + offset, 8, maddr); + CHECK_ATOMIC_MEMORY_ACCESS(8); + + os_mutex_lock(&memory->mem_lock); + readv = (uint64)LOAD_I64(maddr); + if (readv == expect) { + STORE_I64(maddr, sval); + } + os_mutex_unlock(&memory->mem_lock); + } + PUSH_I64(readv); + break; + } + + DEF_ATOMIC_RMW_OPCODE(ADD, +); + DEF_ATOMIC_RMW_OPCODE(SUB, -); + DEF_ATOMIC_RMW_OPCODE(AND, &); + DEF_ATOMIC_RMW_OPCODE(OR, |); + DEF_ATOMIC_RMW_OPCODE(XOR, ^); + /* xchg, ignore the read value, and store the given value: + readv * 0 + sval */ + DEF_ATOMIC_RMW_OPCODE(XCHG, *0 +); + } + + HANDLE_OP_END (); + } +#endif + HANDLE_OP (WASM_OP_IMPDEP): frame = prev_frame; frame_ip = frame->ip; @@ -2854,6 +3282,12 @@ recover_br_info: (void)frame_ip_end; +#if WASM_ENABLE_SHARED_MEMORY != 0 + unaligned_atomic: + wasm_set_exception(module, "unaligned atomic"); + goto got_exception; +#endif + out_of_bounds: wasm_set_exception(module, "out of bounds memory access"); diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index fb12b4ae..175f4d03 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -6596,6 +6596,9 @@ fail_data_cnt_sec_require: error_buf_size)) { goto fail; } +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif } switch (opcode) { case WASM_OP_ATOMIC_NOTIFY: diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index bfa835af..ded22b7c 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -5348,6 +5348,9 @@ handle_op_block_and_loop: CHECK_MEMORY(); read_leb_uint32(p, p_end, align); /* align */ read_leb_uint32(p, p_end, mem_offset); /* offset */ +#if WASM_ENABLE_FAST_INTERP != 0 + emit_uint32(loader_ctx, mem_offset); +#endif } switch (opcode) { case WASM_OP_ATOMIC_NOTIFY: diff --git a/core/iwasm/interpreter/wasm_opcode.h b/core/iwasm/interpreter/wasm_opcode.h index d0e29e7e..f546088b 100644 --- a/core/iwasm/interpreter/wasm_opcode.h +++ b/core/iwasm/interpreter/wasm_opcode.h @@ -369,6 +369,15 @@ typedef enum WASMAtomicEXTOpcode { } #endif +/* Opcode prefix controlled by features */ +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define DEF_ATOMIC_PREFIX_HANDLE(_name) \ + _name[WASM_OP_ATOMIC_PREFIX] = \ + HANDLE_OPCODE (WASM_OP_ATOMIC_PREFIX); /* 0xfe */ +#else +#define DEF_ATOMIC_PREFIX_HANDLE(_name) +#endif + /* * Macro used to generate computed goto tables for the C interpreter. */ @@ -591,5 +600,6 @@ static type _name[WASM_INSTRUCTION_NUM] = { \ do { \ _name[WASM_OP_MISC_PREFIX] = \ HANDLE_OPCODE (WASM_OP_MISC_PREFIX); /* 0xfc */ \ + DEF_ATOMIC_PREFIX_HANDLE(_name) \ } while (0) #endif /* end of _WASM_OPCODE_H */ diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index c2d9ab9c..42829c7a 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -91,6 +91,7 @@ memories_deinstantiate(WASMModuleInstance *module_inst, continue; #endif #if WASM_ENABLE_SHARED_MEMORY != 0 + os_mutex_destroy(&memories[0]->mem_lock); if (memories[i]->is_shared) { int32 ref_count = shared_memory_dec_reference( @@ -192,6 +193,11 @@ memory_instantiate(WASMModuleInstance *module_inst, memory->heap_base_offset = -(int32)heap_size; #if WASM_ENABLE_SHARED_MEMORY != 0 + if (0 != os_mutex_init(&memory->mem_lock)) { + mem_allocator_destroy(memory->heap_handle); + wasm_runtime_free(memory); + return NULL; + } if (is_shared_memory) { memory->is_shared = true; if (!shared_memory_set_memory_inst( @@ -200,6 +206,8 @@ memory_instantiate(WASMModuleInstance *module_inst, set_error_buf(error_buf, error_buf_size, "Instantiate memory failed:" "allocate memory failed."); + os_mutex_destroy(&memory->mem_lock); + mem_allocator_destroy(memory->heap_handle); wasm_runtime_free(memory); return NULL; } diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 361a7611..fcb9a1e6 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -50,6 +50,12 @@ typedef struct WASMMemoryInstance { /* to indicate which module instance create it */ WASMModuleInstance *owner; #endif + +#if WASM_ENABLE_SHARED_MEMORY != 0 + /* mutex lock for the memory, used in atomic operation */ + korp_mutex mem_lock; +#endif + /* Base address, the layout is: heap_data + memory data memory data init size is: num_bytes_per_page * cur_page_count diff --git a/doc/pthread_library.md b/doc/pthread_library.md index fd6964aa..cfc6bece 100644 --- a/doc/pthread_library.md +++ b/doc/pthread_library.md @@ -67,7 +67,6 @@ You can also build this program with WASI, but we need to make some changes to w ``` > Note:
>1. Remember to back up the original sysroot files ->2. wasi-sdk 9.0 or above are not supported, please use 7.0 or 8.0 Then build the program with this command: ``` bash