Add standalone cases (#3536)

tests/standalone/test-pthread/create_join.c

@@ -0,0 +1,68 @@
+/*
+ * This file is copied from https://web.dev/articles/wasm-threads
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+
+/* Calculate Fibonacci numbers shared function */
+int
+fibonacci(int iterations)
+{
+    int val = 1;
+    int last = 0;
+
+    if (iterations == 0) {
+        return 0;
+    }
+    for (int i = 1; i < iterations; i++) {
+        int seq;
+
+        seq = val + last;
+        last = val;
+        val = seq;
+    }
+    return val;
+}
+
+int bg = 42;
+
+/* Start function for the background thread */
+void *
+bg_func(void *arg)
+{
+    int *iter = (void *)arg;
+
+    *iter = fibonacci(*iter);
+    printf("bg number: %d\n", *iter);
+    return arg;
+}
+
+/* Foreground thread and main entry point */
+int
+main(int argc, char *argv[])
+{
+    int fg_val = 54;
+    int bg_val = 42;
+    pthread_t bg_thread;
+
+    /* Create the background thread */
+    if (pthread_create(&bg_thread, NULL, bg_func, &bg_val)) {
+        printf("Thread create failed");
+        return 1;
+    }
+
+    /* Calculate on the foreground thread */
+    fg_val = fibonacci(fg_val);
+
+    /* Wait for background thread to finish */
+    if (pthread_join(bg_thread, NULL)) {
+        printf("Thread join failed");
+        return 2;
+    }
+
+    /* Show the result from background and foreground threads */
+    printf("Fib(42) is %d, Fib(6 * 9) is %d\n", bg_val, fg_val);
+
+    return 0;
+}

tests/standalone/test-pthread/main_thread_return.c

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+
+/* Start function for the background thread */
+void *
+bg_func(void *arg)
+{
+    uint32_t i;
+
+    printf("Thread start.\n");
+
+    for (i = 0; i < 100000000; i++)
+        arg += i;
+    return arg;
+}
+
+/* Foreground thread and main entry point */
+int
+main(int argc, char *argv[])
+{
+    pthread_t bg_thread;
+
+    if (pthread_create(&bg_thread, NULL, bg_func, NULL)) {
+        printf("Thread create failed");
+        return 1;
+    }
+    printf("Thread created.\n");
+
+    /* Return the process directly, runtime should terminate
+        the sub threads and exit the whole wasm module */
+    printf("Process exit.\n");
+    printf("Test success.\n");
+
+    return 0;
+}

tests/standalone/test-pthread/pthread_cond.c

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+pthread_mutex_t mutex;
+static pthread_cond_t cond;
+
+typedef struct test {
+    int test1;
+    int test2;
+} Test;
+
+Test t1;
+
+void *
+thread(void *arg)
+{
+    pthread_mutex_lock(&mutex);
+    printf("thread signal\n");
+    pthread_cond_signal(&cond);
+    pthread_mutex_unlock(&mutex);
+
+    return NULL;
+}
+
+int
+main()
+{
+    pthread_t p;
+
+    pthread_mutex_init(&mutex, NULL);
+    pthread_cond_init(&cond, NULL);
+
+    printf("parent begin\n");
+
+    pthread_mutex_lock(&mutex);
+    pthread_create(&p, NULL, thread, NULL);
+
+    pthread_cond_wait(&cond, &mutex);
+    pthread_mutex_unlock(&mutex);
+
+    printf("parend end\n");
+
+    pthread_join(p, NULL);
+
+    pthread_cond_destroy(&cond);
+    pthread_mutex_destroy(&mutex);
+
+    return 0;
+}

tests/standalone/test-pthread/pthread_key.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+pthread_key_t key;
+pthread_mutex_t lock;
+
+struct test_struct {
+    int i;
+    int k;
+};
+
+typedef struct point {
+    int x;
+    int y;
+} Point;
+
+void *
+child1(void *arg)
+{
+    struct test_struct struct_data;
+    struct_data.i = 10;
+    struct_data.k = 5;
+    pthread_mutex_lock(&lock);
+    pthread_setspecific(key, &struct_data);
+    printf("thread1--address of struct_data is --> %p\n", &(struct_data));
+    printf("thread1--from pthread_getspecific(key) get the pointer and it "
+           "points to --> %p\n",
+           (struct test_struct *)pthread_getspecific(key));
+    printf("thread1--from pthread_getspecific(key) get the pointer and print "
+           "it's content:\nstruct_data.i:%d\nstruct_data.k: %d\n",
+           ((struct test_struct *)pthread_getspecific(key))->i,
+           ((struct test_struct *)pthread_getspecific(key))->k);
+    printf("------------------------------------------------------\n");
+    pthread_mutex_unlock(&lock);
+}
+
+void *
+child2(void *arg)
+{
+    int temp = 20;
+
+    pthread_mutex_lock(&lock);
+    printf("thread2--temp's address is %p\n", &temp);
+    pthread_setspecific(key, &temp);
+    printf("thread2--from pthread_getspecific(key) get the pointer and it "
+           "points to --> %p\n",
+           (int *)pthread_getspecific(key));
+    printf("thread2--from pthread_getspecific(key) get the pointer and print "
+           "it's content --> temp:%d\n",
+           *((int *)pthread_getspecific(key)));
+    printf("------------------------------------------------------\n");
+    pthread_mutex_unlock(&lock);
+}
+
+int
+main(void)
+{
+    Point p;
+    p.x = 10;
+    p.y = 20;
+    pthread_t tid1, tid2;
+
+    pthread_mutex_init(&lock, NULL);
+
+    pthread_key_create(&key, NULL);
+    pthread_create(&tid1, NULL, child1, NULL);
+    pthread_create(&tid2, NULL, child2, NULL);
+
+    pthread_mutex_lock(&lock);
+    printf("main--temp's address is %p\n", &p);
+    pthread_setspecific(key, &p);
+    printf("main--from pthread_getspecific(key) get the pointer and it points "
+           "to --> %p\n",
+           (int *)pthread_getspecific(key));
+    printf("main--from pthread_getspecific(key) get the pointer and print "
+           "it's content --> x:%d, y:%d\n",
+           ((Point *)pthread_getspecific(key))->x,
+           ((Point *)pthread_getspecific(key))->y);
+    pthread_mutex_unlock(&lock);
+
+    pthread_join(tid1, NULL);
+    pthread_join(tid2, NULL);
+    pthread_key_delete(key);
+    return 0;
+}

tests/standalone/test-pthread/pthread_mutex.c

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct ct_sum {
+    int sum;
+    pthread_mutex_t lock;
+} ct_sum;
+
+void *
+add1(void *cnt)
+{
+    pthread_mutex_lock(&(((ct_sum *)cnt)->lock));
+    printf("get lock thread1 id:%lu\n", pthread_self());
+
+    int i;
+    for (i = 0; i < 50; i++) {
+        (*(ct_sum *)cnt).sum += i;
+    }
+
+    pthread_mutex_unlock(&(((ct_sum *)cnt)->lock));
+    pthread_exit(NULL);
+    return 0;
+}
+
+void *
+add2(void *cnt)
+{
+    int i;
+    cnt = (ct_sum *)cnt;
+    pthread_mutex_lock(&(((ct_sum *)cnt)->lock));
+    printf("get lock thread2 id:%lu\n", pthread_self());
+
+    for (i = 50; i < 101; i++) {
+        (*(ct_sum *)cnt).sum += i;
+    }
+
+    pthread_mutex_unlock(&(((ct_sum *)cnt)->lock));
+    pthread_exit(NULL);
+    return 0;
+}
+
+int
+main(void)
+{
+    int i;
+    pthread_t ptid1, ptid2;
+    int sum = 0;
+    ct_sum cnt;
+    pthread_mutex_init(&(cnt.lock), NULL);
+    cnt.sum = 0;
+
+    pthread_create(&ptid1, NULL, add1, &cnt);
+    pthread_create(&ptid2, NULL, add2, &cnt);
+
+    pthread_join(ptid1, NULL);
+    pthread_join(ptid2, NULL);
+
+    pthread_mutex_lock(&(cnt.lock));
+    printf("sum %d\n", cnt.sum);
+    pthread_mutex_unlock(&(cnt.lock));
+
+    pthread_mutex_destroy(&(cnt.lock));
+    return 0;
+}

tests/standalone/test-pthread/run.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+if [[ $2 == "--sgx" ]]; then
+    readonly IWASM_CMD="../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
+else
+    readonly IWASM_CMD="../../../product-mini/platforms/linux/build/iwasm"
+fi
+readonly WAMRC_CMD="../../../wamr-compiler/build/wamrc"
+
+if [[ $1 != "--aot" ]]; then
+    echo "============> run create_join.wasm"
+    ${IWASM_CMD} --heap-size=16384 create_join.wasm
+    echo "============> run main_thread_return.wasm"
+    ${IWASM_CMD} --heap-size=16384 main_thread_return.wasm
+    echo "============> run thread_cancel.wasm"
+    ${IWASM_CMD} --heap-size=16384 thread_cancel.wasm
+    echo "============> run thread_exit.wasm"
+    ${IWASM_CMD} --heap-size=16384 thread_exit.wasm
+    echo "============> run wasi.wasm"
+    ${IWASM_CMD} --heap-size=16384 wasi.wasm
+    echo "============> run pthread_mutex.wasm"
+    ${IWASM_CMD} --heap-size=16384 pthread_mutex.wasm
+    echo "============> run pthread_cond.wasm"
+    ${IWASM_CMD} --heap-size=16384 pthread_cond.wasm
+    echo "============> run pthread_key.wasm"
+    ${IWASM_CMD} --heap-size=16384 pthread_key.wasm
+else
+    echo "============> compile create_join.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o create_join.aot create_join.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o create_join.aot create_join.wasm
+    fi
+    echo "============> run create_join.aot"
+    ${IWASM_CMD} --heap-size=16384 create_join.aot
+
+    echo "============> compile main_thread_return.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o main_thread_return.aot main_thread_return.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o main_thread_return.aot main_thread_return.wasm
+    fi
+    echo "============> run main_thread_return.aot"
+    ${IWASM_CMD} --heap-size=16384 main_thread_return.aot
+
+    echo "============> compile thread_cancel.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o thread_cancel.aot thread_cancel.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o thread_cancel.aot thread_cancel.wasm
+    fi
+    echo "============> run thread_cancel.aot"
+    ${IWASM_CMD} --heap-size=16384 thread_cancel.aot
+
+    echo "============> compile thread_exit.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o thread_exit.aot thread_exit.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o thread_exit.aot thread_exit.wasm
+    fi
+    echo "============> run thread_exit.aot"
+    ${IWASM_CMD} --heap-size=16384 thread_exit.aot
+
+    echo "============> compile wasi.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o wasi.aot wasi.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o wasi.aot wasi.wasm
+    fi
+    echo "============> run wasi.aot"
+    ${IWASM_CMD} --heap-size=16384 wasi.aot
+
+    echo "============> compile pthread_mutex.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o pthread_mutex.aot pthread_mutex.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o pthread_mutex.aot pthread_mutex.wasm
+    fi
+    echo "============> run pthread_mutex.aot"
+    ${IWASM_CMD} --heap-size=16384 pthread_mutex.aot
+
+    echo "============> compile pthread_cond.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o pthread_cond.aot pthread_cond.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o pthread_cond.aot pthread_cond.wasm
+    fi
+    echo "============> run pthread_cond.aot"
+    ${IWASM_CMD} --heap-size=16384 pthread_cond.aot
+
+    echo "============> compile pthread_key.wasm to aot"
+    if [[ $2 == "--sgx" ]]; then
+        ${WAMRC_CMD} -sgx --enable-multi-thread -o pthread_key.aot pthread_key.wasm
+    else
+        ${WAMRC_CMD} --enable-multi-thread -o pthread_key.aot pthread_key.wasm
+    fi
+    echo "============> run pthread_key.aot"
+    ${IWASM_CMD} --heap-size=16384 pthread_key.aot
+fi
+
+cd threads-opcode-wasm-apps
+rm -rf build && mkdir build && cd build
+cmake .. && make
+cd ../..
+
+wasm_files=$(ls threads-opcode-wasm-apps/build/*.wasm)
+
+for wasm_file in $wasm_files; do
+    wasm_file_name="${wasm_file%.wasm}"
+    # avoid keep printing warning: warning: SGX pthread_cond_timedwait isn't supported, calling pthread_cond_wait instead!
+    if [[ $2 == "--sgx" ]] && [[ ${wasm_file_name} == *"atomic_wait_notify"* ]]; then
+        echo "============> didn't run ${wasm_file_name} on sgx aot mode, it will output too much warning info"
+        continue
+    fi
+
+    if [[ $1 != "--aot" ]]; then
+        echo "============> run ${wasm_file_name}.wasm"
+        ${IWASM_CMD} --heap-size=16384 ${wasm_file_name}.wasm
+    else
+        echo "============> compile ${wasm_file_name}.wasm to aot"
+        if [[ $2 == "--sgx" ]]; then
+            ${WAMRC_CMD} -sgx --enable-multi-thread --opt-level=0 -o ${wasm_file_name}.aot ${wasm_file_name}.wasm
+        else
+            ${WAMRC_CMD} --enable-multi-thread --opt-level=0 -o ${wasm_file_name}.aot ${wasm_file_name}.wasm
+        fi
+        echo "============> run ${wasm_file_name}.aot"
+        ${IWASM_CMD} --heap-size=16384 ${wasm_file_name}.aot
+    fi
+done

tests/standalone/test-pthread/thread_cancel.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+
+/* Start function for the background thread */
+void *
+bg_func(void *arg)
+{
+    printf("Thread start.\n");
+    /* This thread will never exit unless canceled by other thread */
+    while (1)
+        ;
+}
+
+/* Foreground thread and main entry point */
+int
+main(int argc, char *argv[])
+{
+    pthread_t bg_thread;
+
+    if (pthread_create(&bg_thread, NULL, bg_func, NULL)) {
+        printf("Thread create failed");
+        return 1;
+    }
+    printf("Thread created.\n");
+
+    /* Cancel the sub thread */
+    pthread_cancel(bg_thread);
+
+    printf("Sub-thread Canceled.\n");
+    printf("Test success.\n");
+
+    return 0;
+}

tests/standalone/test-pthread/thread_exit.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+
+/* Start function for the background thread */
+void *
+bg_func(void *arg)
+{
+    printf("Thread start.\n");
+    while (1) {
+        pthread_exit(NULL);
+    }
+}
+
+/* Foreground thread and main entry point */
+int
+main(int argc, char *argv[])
+{
+    pthread_t bg_thread;
+
+    if (pthread_create(&bg_thread, NULL, bg_func, NULL)) {
+        printf("Thread create failed");
+        return 1;
+    }
+    printf("Thread created.\n");
+
+    /* Wait for background thread to finish */
+    if (pthread_join(bg_thread, NULL)) {
+        printf("Thread join failed");
+        return 2;
+    }
+
+    printf("Sub-thread exit.\n");
+    printf("Test success.\n");
+
+    return 0;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/CMakeLists.txt

@@ -0,0 +1,61 @@
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+cmake_minimum_required(VERSION 3.14)
+project(wasm-apps)
+
+set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../wamr)
+
+if (APPLE)
+    set (HAVE_FLAG_SEARCH_PATHS_FIRST 0)
+    set (CMAKE_C_LINK_FLAGS "")
+    set (CMAKE_CXX_LINK_FLAGS "")
+endif ()
+# have to be debug for peterson lock algorithm
+set(CMAKE_BUILD_TYPE Debug)
+set(CMAKE_SYSTEM_PROCESSOR wasm32)
+set (CMAKE_SYSROOT                  ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot)
+
+if (NOT DEFINED WASI_SDK_DIR)
+    set (WASI_SDK_DIR               "/opt/wasi-sdk")
+endif ()
+
+set (CMAKE_C_FLAGS                  "-nostdlib -pthread -Qunused-arguments")
+set (CMAKE_C_FLAGS                  "${CMAKE_C_FLAGS} -z stack-size=32768")
+set (CMAKE_C_COMPILER_TARGET        "wasm32")
+set (CMAKE_C_COMPILER               "${WASI_SDK_DIR}/bin/clang")
+
+set (DEFINED_SYMBOLS
+"${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt")
+
+set (CMAKE_EXE_LINKER_FLAGS
+     "-Wl,--shared-memory,--max-memory=131072,          \
+      -Wl,--no-entry,--strip-all,                       \
+      -Wl,--export=__heap_base,--export=__data_end      \
+      -Wl,--export=__wasm_call_ctors                    \
+      -Wl,--export=main -Wl,--export=__main_argc_argv   \
+      -Wl,--allow-undefined"
+      #-Wl,--allow-undefined-file=${DEFINED_SYMBOLS}"
+)
+
+add_executable(atomic_add_sub.wasm atomic_add_sub.c)
+target_link_libraries(atomic_add_sub.wasm)
+
+# use peterson lock to test the atomicity of opcode: fence store xchg 
+add_executable(peterson_native.wasm peterson_native.c)
+target_link_libraries(peterson_native.wasm)
+
+add_executable(atomic_fence.wasm atomic_fence.c)
+target_link_libraries(atomic_fence.wasm)
+
+add_executable(atomic_store.wasm atomic_store.c)
+target_link_libraries(atomic_store.wasm)
+
+add_executable(atomic_xchg.wasm atomic_xchg.c)
+target_link_libraries(atomic_xchg.wasm)
+
+add_executable(atomic_logical.wasm atomic_logical.c)
+target_link_libraries(atomic_logical.wasm)
+
+add_executable(atomic_wait_notify.wasm atomic_wait_notify.c)
+target_link_libraries(atomic_wait_notify.wasm)

tests/standalone/test-pthread/threads-opcode-wasm-apps/README.md

@@ -0,0 +1,73 @@
+# "Unit Test" Testsuite for Threads Opcode
+
+These tests are meant to **test the atomicity** of threads code. Initially, they were meant to test if the fast JIT implementation is correct, but they can also be used to test other running modes. The semantics correctness (operating with the correct number of bytes in memory and returning the correct value) have already been tested in spec tests (single-thread environment).
+
+## Test Cases Opcode Coverage
+
+> **Atomicity** of **all** threads opcodes are **fully tested** with these cases.
+>
+> **☑ Only indicates** there is a WASM test case to **test the atomicity directly.** The atomicity of other opcodes **without ☑** is **tested indirectly.** Indirect testing means that it is either implicit with other cases in this directory or tested pragmatically correct (atomicity and semantic correctness together) in pthread or WASI-threads test cases.
+
+Click the link to see the details of how each opcode is tested.
+
+- RMW (Read-Modify-Write):
+  - [CMPXCHG opcode](#cmpxchg) ☑
+  - Arithmetic:
+    - [ADD opcode](#arithmetic-add-sub-and-xchg) ☑
+    - [SUB opcode](#arithmetic-add-sub-and-xchg) ☑
+    - [XCHG opcode](#arithmetic-add-sub-and-xchg) ☑
+  - Logical:
+    - [AND opcode](#logical-or-xor-and)
+    - [OR opcode](#logical-or-xor-and)
+    - [XOR opcode](#logical-or-xor-and) ☑
+
+- [LOAD](#atomic-ldstfence)
+
+- [STORE](#atomic-ldstfence) ☑
+
+- [FENCE](#atomic-ldstfence) ☑
+
+- [WAIT & NOTIFY](#atomic-waitnotify) ☑
+
+## Details
+
+### atomic rmw
+
+#### arithmetic (`add`, `sub`) and xchg
+
+- `add, sub`: in [atomic_add_sub.c](./atomic_add_sub.c), **__atomic_fetch_add/sub()** to generate wasm opcode atomic.add/sub
+
+- `xchg`: in x86-64 implementation, wasm code atomic `store` and `xchg` generate same asm instruction xchg, should be enough to only test with store(tested in [atomic_store.c](./atomic_store.c)). But add a `atomic_xchg.c` to use **__atomic_exchange()** to generate wasm opcode xchg and test it anyways(tested in [atomic_xchg.c](./atomic_xchg.c)).
+
+#### logical `or`, `xor`, `and`
+
+- logical `or`, `xor`, `and`: those three opcodes are similar, it all generate a loop, inside which uses corresponding asm instruction do logical operation and locked cmpxchg to atomically modify the memory. So in a sense, test it will implicitly test the atomicity of cmpxchg.
+
+  in [atomic_logical.c](./atomic_logical.c), tested `xor` wasm opcode to test the atomicity of the generated loop:
+
+  - make use of operation "n `xor` n -> 0", when n range from 1 to 9999, 4 thread concurrently xor the same variable, the final result should be 0.
+  
+  The generated loop of `xor` is atomic -> generated loop of `or`, `and` is also atomic
+
+#### cmpxchg
+
+- wasm opcode `cmpxchg` already tested together with other opcodes in multiple wasi-thread cases. Logical opcodes generate asm instruction lock cmpxchg, the atomicity of generated asm code is proven in logical opcode. In [atomic_wait&&notify.c](./atomic_wait_notify.c), it also tests the opcode `cmpxchg`
+
+### atomic ld/st/fence
+
+use peterson lock algorithm, in [atomic_fence.c](./atomic_fence.c) to test the atomicity of `fence`
+
+> PS: since the interpreter is relatively slow compared to JIT/AOT mode, it's less likely(almost impossible) to trigger processor-level behavior: instructions to be ordered within a single thread
+
+The prerequisite for peterson lock properly is that load and store have to be `Sequential Consistency`, which can be achieved use:
+
+1. LOAD (without fence) and STORE + MFENCE -> use it to test `fence` opcode
+2. LOAD (without fence) and LOCK XCHG -> use it to test atomic `store` opcode
+3. MFENCE + LOAD and STORE (without fence)
+4. LOCK XADD ( 0 ) and STORE (without fence)
+
+### atomic wait&notify
+
+Actually in every pthread tests, it will generate `wait` and `notify`, it is also tested in in multiple wasi-thread cases.
+
+But add a [atomic_wait&&notify.c](./atomic_wait_notify.c) to test it anyways.

tests/standalone/test-pthread/threads-opcode-wasm-apps/atomic_add_sub.c

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <stdio.h>
+#include <pthread.h>
+
+#define MAX_NUM_THREADS 4
+#define NUM_ITER 100000
+
+int g_add_count = 0;
+int g_sub_count = 0;
+
+static void *
+thread(void *arg)
+{
+    for (int i = 0; i < NUM_ITER; i++) {
+        __atomic_fetch_add(&g_add_count, 1, __ATOMIC_SEQ_CST);
+        __atomic_fetch_sub(&g_sub_count, 1, __ATOMIC_SEQ_CST);
+    }
+
+    return NULL;
+}
+
+int
+main(int argc, char **argv)
+{
+    pthread_t tids[MAX_NUM_THREADS];
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (pthread_create(&tids[i], NULL, thread, NULL) != 0) {
+            printf("Thread creation failed\n");
+        }
+    }
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (pthread_join(tids[i], NULL) != 0) {
+            printf("Thread join failed\n");
+        }
+    }
+
+    printf("Value of counter after add update: %d (expected=%d)\n", g_add_count,
+           MAX_NUM_THREADS * NUM_ITER);
+    if (g_add_count != MAX_NUM_THREADS * NUM_ITER) {
+        __builtin_trap();
+    }
+
+    printf("Value of counter after sub update: %d (expected=%d)\n", g_sub_count,
+           -(MAX_NUM_THREADS * NUM_ITER));
+    if (g_sub_count != -(MAX_NUM_THREADS * NUM_ITER)) {
+        __builtin_trap();
+    }
+
+    return -1;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/atomic_fence.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include "peterson.h"
+
+void
+peterson_lock_acquire(peterson_lock_t *lock, int thread_id)
+{
+    // this threads wants to enter the cs
+    lock->flag[thread_id] = true;
+
+    // assume the other thread has priority
+    int other_thread = 1 - thread_id;
+
+    lock->turn = other_thread;
+
+    // generate fence in wasm
+    __atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+    while (lock->turn == other_thread && lock->flag[other_thread]) {
+        // Busy wait
+    }
+}
+
+int
+main()
+{
+    pthread_t thread1, thread2;
+
+    printf("============ test peterson lock using atomic fence ============\n");
+    run_test(&thread1, &thread2, test_peterson_lock_atomicity);
+
+    return 0;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/atomic_logical.c

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include <stdio.h>
+#include <pthread.h>
+
+// x XOR x -> 0
+// even num of thread -> g_val should end up with its original value
+#define MAX_NUM_THREADS 4
+#define NUM_ITER 199999
+
+int g_val = 5050;
+
+static void *
+thread(void *arg)
+{
+    for (int i = 0; i < NUM_ITER; i++) {
+        __atomic_fetch_xor(&g_val, i, __ATOMIC_SEQ_CST);
+    }
+
+    return NULL;
+}
+
+int
+main(int argc, char **argv)
+{
+
+    pthread_t tids[MAX_NUM_THREADS];
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (pthread_create(&tids[i], NULL, thread, NULL) != 0) {
+            printf("Thread creation failed\n");
+        }
+    }
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (pthread_join(tids[i], NULL) != 0) {
+            printf("Thread join failed\n");
+        }
+    }
+
+    printf("Global value after update: %d (expected=%d)\n", g_val, 5050);
+    if (g_val != 5050) {
+        __builtin_trap();
+    }
+
+    return -1;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/atomic_store.c

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include "peterson.h"
+
+void
+peterson_lock_acquire(peterson_lock_t *lock, int thread_id)
+{
+    // this threads wants to enter the cs
+    __atomic_store(&lock->flag[thread_id], &(bool){ true }, __ATOMIC_SEQ_CST);
+
+    // assume the other thread has priority
+    int other_thread = 1 - thread_id;
+
+    __atomic_store(&lock->turn, &other_thread, __ATOMIC_SEQ_CST);
+
+    while (lock->turn == other_thread && lock->flag[other_thread]) {
+        // Busy wait
+    }
+}
+
+int
+main()
+{
+    pthread_t thread1, thread2;
+
+    printf("============ test peterson lock using atomic store ============\n");
+    run_test(&thread1, &thread2, test_peterson_lock_atomicity);
+
+    return 0;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/atomic_wait_notify.c

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include <stdio.h>
+#include <pthread.h>
+#include <stdbool.h>
+
+#define MAX_NUM_THREADS 4
+#define NUM_ITER 100000
+
+int g_val;
+int my_mutex;
+
+int
+try_lock()
+{
+    return __atomic_compare_exchange(&my_mutex, &(int){ 0 }, &(int){ 1 }, false,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+void
+lock_acquire()
+{
+    while (!try_lock()) {
+        // expected value (1 => locked)
+        __builtin_wasm_memory_atomic_wait32(&my_mutex, 1, -1);
+    }
+}
+void
+lock_release()
+{
+    // unlock the mutex
+    __atomic_store(&my_mutex, &(int){ 0 }, __ATOMIC_SEQ_CST);
+    // notify 1 waiter
+    __builtin_wasm_memory_atomic_notify(&my_mutex, 1);
+}
+
+static void *
+thread(void *arg)
+{
+    for (int i = 0; i < NUM_ITER; i++) {
+        lock_acquire();
+        g_val++;
+        lock_release();
+    }
+
+    return NULL;
+}
+
+int
+main()
+{
+    pthread_t tids[MAX_NUM_THREADS];
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (pthread_create(&tids[i], NULL, thread, NULL) != 0) {
+            printf("Thread creation failed\n");
+        }
+    }
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (pthread_join(tids[i], NULL) != 0) {
+            printf("Thread join failed\n");
+        }
+    }
+
+    printf("Value of counter after add update: %d (expected=%d)\n", g_val,
+           MAX_NUM_THREADS * NUM_ITER);
+    if (g_val != MAX_NUM_THREADS * NUM_ITER) {
+        __builtin_trap();
+    }
+
+    return 0;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/atomic_xchg.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include "peterson.h"
+
+void
+peterson_lock_acquire(peterson_lock_t *lock, int thread_id)
+{
+    bool xchg_ret1;
+    int xchg_ret2;
+    // this threads wants to enter the cs
+    __atomic_exchange(&lock->flag[thread_id], &(bool){ true }, &xchg_ret1,
+                      __ATOMIC_SEQ_CST);
+
+    // assume the other thread has priority
+    int other_thread = 1 - thread_id;
+
+    __atomic_exchange(&lock->turn, &other_thread, &xchg_ret2, __ATOMIC_SEQ_CST);
+
+    while (lock->turn == other_thread && lock->flag[other_thread]) {
+        // Busy wait
+    }
+}
+
+int
+main()
+{
+    pthread_t thread1, thread2;
+
+    printf("============ test peterson lock using atomic xchg ============\n");
+    run_test(&thread1, &thread2, test_peterson_lock_atomicity);
+
+    return 0;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/peterson.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include <stdio.h>
+#include <pthread.h>
+#include <stdbool.h>
+
+// Peterson's algorithm for mutual exclusion
+#define ITERATIONS 15000000
+
+typedef struct {
+    bool flag[2];
+    int turn;
+} peterson_lock_t;
+
+static int counter = 0;
+static peterson_lock_t lock;
+
+void
+peterson_lock_acquire(peterson_lock_t *lock, int thread_id);
+
+void
+peterson_lock_release(peterson_lock_t *lock, int thread_id)
+{
+    lock->flag[thread_id] = false;
+}
+
+void *
+test_peterson_lock_atomicity(void *arg)
+{
+    int thread_id = (int)(long)arg;
+
+    for (int i = 0; i < ITERATIONS; ++i) {
+        peterson_lock_acquire(&lock, thread_id);
+        counter++;
+        peterson_lock_release(&lock, thread_id);
+    }
+
+    return NULL;
+}
+
+int
+run_test(pthread_t *thread1_ptr, pthread_t *thread2_ptr,
+         void *(*start_routine)(void *))
+{
+    lock.flag[0] = false;
+    lock.flag[1] = false;
+    lock.turn = 0;
+    counter = 0;
+
+    pthread_create(thread1_ptr, NULL, start_routine, (void *)0);
+    pthread_create(thread2_ptr, NULL, start_routine, (void *)1);
+
+    pthread_join(*thread1_ptr, NULL);
+    pthread_join(*thread2_ptr, NULL);
+
+    printf("Expected counter value: %d\n", ITERATIONS * 2);
+    printf("Actual counter value: %d\n", counter);
+    if (counter != ITERATIONS * 2)
+        __builtin_trap();
+
+    return 0;
+}

tests/standalone/test-pthread/threads-opcode-wasm-apps/peterson_native.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#include "peterson.h"
+
+void
+peterson_lock_acquire(peterson_lock_t *lock, int thread_id)
+{
+    // this threads wants to enter the cs
+    lock->flag[thread_id] = true;
+
+    // assume the other thread has priority
+    int other_thread = 1 - thread_id;
+
+    lock->turn = other_thread;
+
+    while (lock->turn == other_thread && lock->flag[other_thread]) {
+        // Busy wait
+    }
+}
+
+int
+main()
+{
+    pthread_t thread1, thread2;
+
+    printf("============ test naive peterson lock(should trap) ============\n");
+    printf("============ May not be able to observe in interpreter mode "
+           "============\n");
+    run_test(&thread1, &thread2, test_peterson_lock_atomicity);
+
+    return 0;
+}

tests/standalone/test-pthread/wasi.c

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2023 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <unistd.h>
+
+struct member {
+    int a;
+    char *b;
+} temp = { 8, "zieckey" };
+
+void *
+create(void *arg)
+{
+    printf("new thread ... \n");
+    sleep(5);
+
+    pthread_exit(&temp);
+}
+
+int
+main(int argc, char *argv[])
+{
+    int error;
+    pthread_t tid;
+    struct member *c;
+
+    error = pthread_create(&tid, NULL, create, NULL);
+
+    if (error) {
+        printf("new thread is not created ... \n");
+        return -1;
+    }
+    printf("main ... \n");
+
+    error = pthread_join(tid, (void *)&c);
+
+    if (error) {
+        printf("new thread is not exit ... \n");
+        return -2;
+    }
+
+    printf("c->a = %d \n", c->a);
+    printf("c->b = %s \n", c->b);
+    sleep(1);
+
+    return 0;
+}