Merge pull request #2740 from bytecodealliance/dev/wasi-libc-windows

The implementation is already in a stage where it's possible to compile WAMR
with wasi libc enabled and run wasi modules without errors.

tests/wamr-test-suites/spec-test-script/all.py

@@ -6,7 +6,7 @@
 
 import argparse
 import multiprocessing as mp
-import os
+import platform
 import pathlib
 import subprocess
 import sys
@@ -28,12 +28,26 @@ To run a single GC case:
     --aot-compiler wamrc --gc spec/test/core/xxx.wast
 """
 
-PLATFORM_NAME = os.uname().sysname.lower()
-IWASM_CMD = "../../../product-mini/platforms/" + PLATFORM_NAME + "/build/iwasm"
+def exe_file_path(base_path: str) -> str:
+    if platform.system().lower() == "windows":
+        base_path += ".exe"
+    return base_path
+
+def get_iwasm_cmd(platform: str) -> str:
+    build_path = "../../../product-mini/platforms/" + platform + "/build/"
+    exe_name = "iwasm"
+
+    if platform == "windows":
+        build_path += "RelWithDebInfo/"
+
+    return exe_file_path(build_path + exe_name)
+
+PLATFORM_NAME = platform.uname().system.lower()
+IWASM_CMD = get_iwasm_cmd(PLATFORM_NAME)
 IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
 IWASM_QEMU_CMD = "iwasm"
 SPEC_TEST_DIR = "spec/test/core"
-WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
+WAST2WASM_CMD = exe_file_path("./wabt/out/gcc/Release/wat2wasm")
 SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
 WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
 
@@ -146,8 +160,9 @@ def test_case(
     qemu_flag=False,
     qemu_firmware="",
     log="",
+    no_pty=False
 ):
-    CMD = ["python3", "runtest.py"]
+    CMD = [sys.executable, "runtest.py"]
     CMD.append("--wast2wasm")
     CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD)
     CMD.append("--interpreter")
@@ -157,6 +172,8 @@ def test_case(
         CMD.append(IWASM_QEMU_CMD)
     else:
         CMD.append(IWASM_CMD)
+    if no_pty:
+        CMD.append("--no-pty")
     CMD.append("--aot-compiler")
     CMD.append(WAMRC_CMD)
 
@@ -261,6 +278,7 @@ def test_suite(
     qemu_flag=False,
     qemu_firmware="",
     log="",
+    no_pty=False,
 ):
     suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
     if not suite_path.exists():
@@ -322,6 +340,7 @@ def test_suite(
                         qemu_flag,
                         qemu_firmware,
                         log,
+                        no_pty,
                     ],
                 )
 
@@ -359,6 +378,7 @@ def test_suite(
                     qemu_flag,
                     qemu_firmware,
                     log,
+                    no_pty,
                 )
                 successful_case += 1
             except Exception as e:
@@ -480,6 +500,8 @@ def main():
         nargs="*",
         help=f"Specify all wanted cases. If not the script will go through all cases under {SPEC_TEST_DIR}",
     )
+    parser.add_argument('--no-pty', action='store_true',
+        help="Use direct pipes instead of pseudo-tty")
 
     options = parser.parse_args()
 
@@ -509,6 +531,7 @@ def main():
             options.qemu_flag,
             options.qemu_firmware,
             options.log,
+            options.no_pty
         )
         end = time.time_ns()
         print(
@@ -532,6 +555,7 @@ def main():
                     options.qemu_flag,
                     options.qemu_firmware,
                     options.log,
+                    options.no_pty,
                 )
             else:
                 ret = True

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -5,22 +5,21 @@ from __future__ import print_function
 import argparse
 import array
 import atexit
-import fcntl
 import math
 import os
-# Pseudo-TTY and terminal manipulation
-import pty
 import re
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
-import termios
 import time
+import threading
 import traceback
 from select import select
+from queue import Queue
 from subprocess import PIPE, STDOUT, Popen
+from typing import BinaryIO, Optional, Tuple
 
 if sys.version_info[0] == 2:
     IS_PY_3 = False
@@ -52,6 +51,10 @@ def log(data, end='\n'):
     print(data, end=end)
     sys.stdout.flush()
 
+def create_tmp_file(suffix: str) -> str:
+    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp_file:
+        return tmp_file.name
+
 # TODO: do we need to support '\n' too
 import platform
 
@@ -62,6 +65,34 @@ else:
     sep = "\r\n"
 rundir = None
 
+
+class AsyncStreamReader:
+    def __init__(self, stream: BinaryIO) -> None:
+        self._queue = Queue()
+        self._reader_thread = threading.Thread(
+            daemon=True,
+            target=AsyncStreamReader._stdout_reader,
+            args=(self._queue, stream))
+        self._reader_thread.start()
+
+    def read(self) -> Optional[bytes]:
+        return self._queue.get()
+
+    def cleanup(self) -> None:
+        self._reader_thread.join()
+
+    @staticmethod
+    def _stdout_reader(queue: Queue, stdout: BinaryIO) -> None:
+        while True:
+            try:
+                queue.put(stdout.read(1))
+            except ValueError as e:
+                if stdout.closed:
+                    queue.put(None)
+                    break
+                raise e
+
+
 class Runner():
     def __init__(self, args, no_pty=False):
         self.no_pty = no_pty
@@ -77,11 +108,14 @@ class Runner():
         if no_pty:
             self.process = Popen(args, bufsize=0,
                            stdin=PIPE, stdout=PIPE, stderr=STDOUT,
-                           preexec_fn=os.setsid,
                            env=env)
             self.stdin = self.process.stdin
             self.stdout = self.process.stdout
         else:
+            import fcntl
+            # Pseudo-TTY and terminal manipulation
+            import pty
+            import termios
             # Use tty to setup an interactive environment
             master, slave = pty.openpty()
 
@@ -101,35 +135,53 @@ class Runner():
             self.stdin = os.fdopen(master, 'r+b', 0)
             self.stdout = self.stdin
 
+        if platform.system().lower() == "windows":
+            self._stream_reader = AsyncStreamReader(self.stdout)
+        else:
+            self._stream_reader = None
+
         self.buf = ""
 
+    def _read_stdout_byte(self) -> Tuple[bool, Optional[bytes]]:
+        if self._stream_reader:
+            return True, self._stream_reader.read()
+        else:
+            # select doesn't work on file descriptors on Windows.
+            # however, this method is much faster than using
+            # queue, so we keep it for non-windows platforms.
+            [outs,_,_] = select([self.stdout], [], [], 1)
+            if self.stdout in outs:
+                return True, self.stdout.read(1)
+            else:
+                return False, None
+
     def read_to_prompt(self, prompts, timeout):
         wait_until = time.time() + timeout
         while time.time() < wait_until:
-            [outs,_,_] = select([self.stdout], [], [], 1)
-            if self.stdout in outs:
-                read_byte = self.stdout.read(1)
-                if not read_byte:
-                    # EOF on macOS ends up here.
-                    break
-                read_byte = read_byte.decode('utf-8') if IS_PY_3 else read_byte
+            has_value, read_byte = self._read_stdout_byte()
+            if not has_value:
+                continue
+            if not read_byte:
+                # EOF on macOS ends up here.
+                break
+            read_byte = read_byte.decode('utf-8') if IS_PY_3 else read_byte
 
-                debug(read_byte)
-                if self.no_pty:
-                    self.buf += read_byte.replace('\n', '\r\n')
-                else:
-                    self.buf += read_byte
-                self.buf = self.buf.replace('\r\r', '\r')
+            debug(read_byte)
+            if self.no_pty:
+                self.buf += read_byte.replace('\n', '\r\n')
+            else:
+                self.buf += read_byte
+            self.buf = self.buf.replace('\r\r', '\r')
 
-                # filter the prompts
-                for prompt in prompts:
-                    pattern = re.compile(prompt)
-                    match = pattern.search(self.buf)
-                    if match:
-                        end = match.end()
-                        buf = self.buf[0:end-len(prompt)]
-                        self.buf = self.buf[end:]
-                        return buf
+            # filter the prompts
+            for prompt in prompts:
+                pattern = re.compile(prompt)
+                match = pattern.search(self.buf)
+                if match:
+                    end = match.end()
+                    buf = self.buf[0:end-len(prompt)]
+                    self.buf = self.buf[end:]
+                    return buf
         return None
 
     def writeline(self, str):
@@ -140,6 +192,8 @@ class Runner():
         self.stdin.write(str_to_write)
 
     def cleanup(self):
+        atexit.unregister(self.cleanup)
+
         if self.process:
             try:
                 self.writeline("__exit__")
@@ -157,6 +211,8 @@ class Runner():
             self.stdout = None
             if not IS_PY_3:
                 sys.exc_clear()
+            if self._stream_reader:
+                self._stream_reader.cleanup()
 
 def assert_prompt(runner, prompts, timeout, is_need_execute_result):
     # Wait for the initial prompt
@@ -402,9 +458,9 @@ def cast_v128_to_i64x2(numbers, type, lane_type):
     unpacked = struct.unpack("Q Q", packed)
     return unpacked, f"[{unpacked[0]:#x} {unpacked[1]:#x}]:{lane_type}:v128"
 
-
 def parse_simple_const_w_type(number, type):
     number = number.replace('_', '')
+    number = re.sub(r"nan\((ind|snan)\)", "nan", number)
     if type in ["i32", "i64"]:
         number = int(number, 16) if '0x' in number else int(number)
         return number, "0x{:x}:{}".format(number, type) \
@@ -941,7 +997,8 @@ def skip_test(form, skip_list):
 
 def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts):
     log("Writing WAST module to '%s'" % wast_tempfile)
-    open(wast_tempfile, 'w').write(form)
+    with open(wast_tempfile, 'w') as file:
+        file.write(form)
     log("Compiling WASM to '%s'" % wasm_tempfile)
 
     # default arguments
@@ -1063,13 +1120,10 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
 def create_tmpfiles(wast_name):
     tempfiles = []
 
-    (t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
-    (t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
-    tempfiles.append(wast_tempfile)
-    tempfiles.append(wasm_tempfile)
+    tempfiles.append(create_tmp_file(".wast"))
+    tempfiles.append(create_tmp_file(".wasm"))
     if test_aot:
-        (t3fd, aot_tempfile) = tempfile.mkstemp(suffix=".aot")
-        tempfiles.append(aot_tempfile)
+        tempfiles.append(create_tmp_file(".aot"))
 
     # add these temp file to temporal repo, will be deleted when finishing the test
     temp_file_repo.extend(tempfiles)
@@ -1138,10 +1192,10 @@ if __name__ == "__main__":
     else:
         SKIP_TESTS = C_SKIP_TESTS
 
-    (t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
-    (t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
+    wast_tempfile = create_tmp_file(".wast")
+    wasm_tempfile = create_tmp_file(".wasm")
     if test_aot:
-        (t3fd, aot_tempfile) = tempfile.mkstemp(suffix=".aot")
+        aot_tempfile = create_tmp_file(".aot")
 
     ret_code = 0
     try:
@@ -1172,17 +1226,16 @@ if __name__ == "__main__":
                     # workaround: spec test changes error message to "malformed" while iwasm still use "invalid"
                     error_msg = m.group(2).replace("malformed", "invalid")
                     log("Testing(malformed)")
-                    f = open(wasm_tempfile, 'wb')
-                    s = m.group(1)
-                    while s:
-                        res = re.match("[^\"]*\"([^\"]*)\"(.*)", s, re.DOTALL)
-                        if IS_PY_3:
-                            context = res.group(1).replace("\\", "\\x").encode("latin1").decode("unicode-escape").encode("latin1")
-                            f.write(context)
-                        else:
-                            f.write(res.group(1).replace("\\", "\\x").decode("string-escape"))
-                        s = res.group(2)
-                    f.close()
+                    with open(wasm_tempfile, 'wb') as f:
+                        s = m.group(1)
+                        while s:
+                            res = re.match("[^\"]*\"([^\"]*)\"(.*)", s, re.DOTALL)
+                            if IS_PY_3:
+                                context = res.group(1).replace("\\", "\\x").encode("latin1").decode("unicode-escape").encode("latin1")
+                                f.write(context)
+                            else:
+                                f.write(res.group(1).replace("\\", "\\x").decode("string-escape"))
+                            s = res.group(2)
 
                     # compile wasm to aot
                     if test_aot:

tests/wamr-test-suites/test_wamr.sh

@@ -53,7 +53,13 @@ ENABLE_GC_HEAP_VERIFY=0
 #unit test case arrary
 TEST_CASE_ARR=()
 SGX_OPT=""
-PLATFORM=$(uname -s | tr A-Z a-z)
+if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
+    PLATFORM=windows
+    PYTHON_EXE=python
+else
+    PLATFORM=$(uname -s | tr A-Z a-z)
+    PYTHON_EXE=python3
+fi
 PARALLELISM=0
 ENABLE_QEMU=0
 QEMU_FIRMWARE=""
@@ -330,15 +336,18 @@ function setup_wabt()
                 darwin)
                     WABT_PLATFORM=macos
                     ;;
+                windows)
+                    WABT_PLATFORM=windows
+                    ;;
                 *)
                     echo "wabt platform for ${PLATFORM} in unknown"
                     exit 1
                     ;;
             esac
             if [ ! -f /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz ]; then
-                wget \
+                curl -L \
                     https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-${WABT_PLATFORM}.tar.gz \
-                    -P /tmp
+                    -o /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz
             fi
 
             cd /tmp \
@@ -493,12 +502,16 @@ function spec_test()
         ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} "
     fi
 
+    if [[ ${PLATFORM} == "windows" ]]; then
+        ARGS_FOR_SPEC_TEST+="--no-pty "
+    fi
+
     # set log directory
     ARGS_FOR_SPEC_TEST+="--log ${REPORT_DIR}"
 
     cd ${WORK_DIR}
-    echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt"
-    python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt
+    echo "${PYTHON_EXE} ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt"
+    ${PYTHON_EXE} ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt
     if [[ ${PIPESTATUS[0]} -ne 0 ]];then
         echo -e "\nspec tests FAILED" | tee -a ${REPORT_DIR}/spec_test_report.txt
         exit 1
@@ -559,7 +572,7 @@ function wasi_certification_test()
     cd wasi-testsuite
     git reset --hard ${WASI_TESTSUITE_COMMIT}
 
-    TSAN_OPTIONS=${TSAN_OPTIONS} bash ../../wasi-test-script/run_wasi_tests.sh $1 $TARGET \
+    TSAN_OPTIONS=${TSAN_OPTIONS} bash ../../wasi-test-script/run_wasi_tests.sh $1 $TARGET $WASI_TEST_FILTER \
         | tee -a ${REPORT_DIR}/wasi_test_report.txt
     ret=${PIPESTATUS[0]}
 
@@ -689,7 +702,7 @@ function build_iwasm_with_cfg()
         && if [ -d build ]; then rm -rf build/*; else mkdir build; fi \
         && cd build \
         && cmake $* .. \
-        && make -j 4
+        && cmake --build . -j 4 --config RelWithDebInfo
     fi
 
     if [ "$?" != 0 ];then
@@ -829,6 +842,17 @@ function trigger()
         EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SANITIZER=tsan"
     fi
 
+    # Make sure we're using the builtin WASI libc implementation
+    # if we're running the wasi certification tests.
+    if [[ $TEST_CASE_ARR ]]; then
+        for test in "${TEST_CASE_ARR[@]}"; do
+            if [[ "$test" == "wasi_certification" ]]; then
+                EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIBC_UVWASI=0 -DWAMR_BUILD_LIBC_WASI=1"
+                break
+            fi
+        done
+    fi
+
     for t in "${TYPE[@]}"; do
         case $t in
             "classic-interp")

tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh

@@ -9,11 +9,25 @@ THIS_DIR=$(cd $(dirname $0) && pwd -P)
 
 readonly MODE=$1
 readonly TARGET=$2
+readonly TEST_FILTER=$3
 
 readonly WORK_DIR=$PWD
-readonly PLATFORM=$(uname -s | tr A-Z a-z)
+
+if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
+    readonly PLATFORM=windows
+    readonly PYTHON_EXE=python
+    # see https://github.com/pypa/virtualenv/commit/993ba1316a83b760370f5a3872b3f5ef4dd904c1
+    readonly VENV_BIN_DIR=Scripts
+    readonly IWASM_EXE=$(cygpath -m "${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/RelWithDebInfo/iwasm.exe")
+else
+    readonly PLATFORM=$(uname -s | tr A-Z a-z)
+    readonly VENV_BIN_DIR=bin
+    readonly PYTHON_EXE=python3
+    readonly IWASM_EXE="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm"
+fi
+
 readonly WAMR_DIR="${WORK_DIR}/../../../.."
-readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm \
+readonly IWASM_CMD="${IWASM_EXE} \
     --allow-resolve=google-public-dns-a.google.com \
     --addr-pool=::1/128,127.0.0.1/32"
 
@@ -28,8 +42,21 @@ readonly THREAD_STRESS_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/
 readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/"
 
 run_aot_tests () {
-    local tests=("$@")
+    local -n tests=$1
+    local -n excluded_tests=$2
+
     for test_wasm in ${tests[@]}; do
+        # get the base file name from the filepath
+        local test_name=${test_wasm##*/}
+        test_name=${test_name%.wasm}
+
+        for excluded_test in "${excluded_tests[@]}"; do
+            if [[ $excluded_test == "\"$test_name\"" ]]; then
+                echo "Skipping test $test_name"
+                continue 2
+            fi
+        done
+
         local iwasm="${IWASM_CMD}"
         if [[ $test_wasm =~ "stress" ]]; then
             iwasm="${IWASM_CMD_STRESS}"
@@ -37,7 +64,7 @@ run_aot_tests () {
 
         test_aot="${test_wasm%.wasm}.aot"
         test_json="${test_wasm%.wasm}.json"
- 
+
         if [ -f ${test_wasm} ]; then
             expected=$(jq .exit_code ${test_json})
         fi
@@ -52,7 +79,7 @@ run_aot_tests () {
             expected=$(jq .exit_code ${test_json})
         fi
 
-        python3 ${THIS_DIR}/pipe.py | ${iwasm} $test_aot
+        $PYTHON_EXE ${THIS_DIR}/pipe.py | ${iwasm} $test_aot
         ret=${PIPESTATUS[1]}
 
         echo "expected=$expected, actual=$ret"
@@ -63,23 +90,29 @@ run_aot_tests () {
 }
 
 if [[ $MODE != "aot" ]];then
-    python3 -m venv wasi-env && source wasi-env/bin/activate
-    python3 -m pip install -r test-runner/requirements.txt
+    $PYTHON_EXE -m venv wasi-env && source wasi-env/${VENV_BIN_DIR}/activate
+    $PYTHON_EXE -m pip install -r test-runner/requirements.txt
 
     export TEST_RUNTIME_EXE="${IWASM_CMD}"
-    python3 ${THIS_DIR}/pipe.py | TSAN_OPTIONS=${TSAN_OPTIONS} python3 test-runner/wasi_test_runner.py \
-            -r adapters/wasm-micro-runtime.py \
-            -t \
-                ${C_TESTS} \
-                ${RUST_TESTS} \
-                ${ASSEMBLYSCRIPT_TESTS} \
-                ${THREAD_PROPOSAL_TESTS} \
-                ${THREAD_INTERNAL_TESTS} \
-                ${LIB_SOCKET_TESTS} \
+
+    TEST_OPTIONS="-r adapters/wasm-micro-runtime.py \
+        -t \
+            ${C_TESTS} \
+            ${RUST_TESTS} \
+            ${ASSEMBLYSCRIPT_TESTS} \
+            ${THREAD_PROPOSAL_TESTS} \
+            ${THREAD_INTERNAL_TESTS} \
+            ${LIB_SOCKET_TESTS}"
+
+    if [ -n "$TEST_FILTER" ]; then
+        TEST_OPTIONS="${TEST_OPTIONS} --exclude-filter ${TEST_FILTER}"
+    fi
+
+    $PYTHON_EXE ${THIS_DIR}/pipe.py | TSAN_OPTIONS=${TSAN_OPTIONS} $PYTHON_EXE test-runner/wasi_test_runner.py $TEST_OPTIONS
 
     ret=${PIPESTATUS[1]}
 
-    TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" TSAN_OPTIONS=${TSAN_OPTIONS}  python3 test-runner/wasi_test_runner.py \
+    TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" TSAN_OPTIONS=${TSAN_OPTIONS} $PYTHON_EXE test-runner/wasi_test_runner.py \
             -r adapters/wasm-micro-runtime.py \
             -t \
                 ${THREAD_STRESS_TESTS}
@@ -87,9 +120,9 @@ if [[ $MODE != "aot" ]];then
     if [ "${ret}" -eq 0 ]; then
         ret=${PIPESTATUS[0]}
     fi
-    
+
     exit_code=${ret}
-    
+
     deactivate
 else
     target_option=""
@@ -101,7 +134,17 @@ else
     for testsuite in ${THREAD_STRESS_TESTS} ${THREAD_PROPOSAL_TESTS} ${THREAD_INTERNAL_TESTS}; do
         tests=$(ls ${testsuite}*.wasm)
         tests_array=($tests)
-        run_aot_tests "${tests_array[@]}"
+
+        if [ -n "$TEST_FILTER" ]; then
+            readarray -t excluded_tests_array < <(jq -c \
+                --slurpfile testsuite_manifest $testsuite/manifest.json \
+                '.[$testsuite_manifest[0].name] // {} | keys[]' \
+                $TEST_FILTER)
+        else
+            excluded_tests_array=()
+        fi
+
+        run_aot_tests tests_array excluded_tests_array
     done
 fi