[wasi-nn] Add a new wasi-nn backend openvino (#3603)

2024-07-22 17:16:41 +08:00
parent 50f28495a1
commit 058bc47102
12 changed files with 1037 additions and 147 deletions
--- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke
+++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke
@ -1,6 +1,8 @@
 # Copyright (C) 2019 Intel Corporation.  All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

+# hadolint global ignore=DL3003,DL3008,DL3009,DL3059
+
 FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye@sha256:ddc1ee022d327f024c07484c9333db3fbbfd504bc096cdb66635653a2bebb33e

 ARG DEBIAN_FRONTEND=noninteractive
@ -8,7 +10,10 @@ ENV TZ=Asian/Shanghai

 # hadolint ignore=DL3009
 RUN apt-get update \
-  && apt-get upgrade -y
+  && apt-get upgrade -y \
+  && apt-get install -y --no-install-recommends cmake
+
+RUN rustup target add wasm32-wasi

 #
 # Openvino
@ -17,33 +22,27 @@ RUN apt-get update \
 #   - https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html
 #   - https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html
 #
-# FIXME: upgrade to 2024.1 or latest after wasi-nn(rust binding) is ready
-WORKDIR /opt/intel
-RUN wget -q https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/linux/l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz
-RUN tar -xf l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz \
-  && rm l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz \
-  && mv l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64 /opt/intel/openvino
-
-WORKDIR /opt/intel/openvino
-RUN ./install_dependencies/install_openvino_dependencies.sh -y \
-  && ./setupvars.sh
-
-#
-# wasmtime
-WORKDIR /opt
-RUN wget -q https://github.com/bytecodealliance/wasmtime/releases/download/v21.0.0/wasmtime-v21.0.0-x86_64-linux.tar.xz
-RUN tar -xf wasmtime-v21.0.0-x86_64-linux.tar.xz \
-  && rm wasmtime-v21.0.0-x86_64-linux.tar.xz \
-  && ln -sf "$(realpath ./wasmtime-v21.0.0-x86_64-linux/wasmtime)" /usr/local/bin/wasmtime
+RUN wget -q https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+  && echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2023.list
+RUN apt-get update \
+  && apt-get upgrade -y \
+  && apt-get install --no-install-recommends -y openvino-2023.2.0

 #
 # wasi-nn
 # compilation requirements
-RUN rustup target add wasm32-wasi wasm32-unknown-unknown
 WORKDIR /workspaces/wasi-nn
 RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git .
-# hadolint ignore=DL3059
-#RUN ./build.sh rust
+
+WORKDIR /workspaces/wasi-nn/rust/examples/classification-example/
+RUN cargo build --target=wasm32-wasi
+
+WORKDIR /workspaces/wasi-nn/rust/examples/classification-example/build
+RUN cp ../target/wasm32-wasi/debug/wasi-nn-example.wasm . \
+  && wget -q --no-clobber https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet/mobilenet.xml \
+  && wget -q --no-clobber https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet/mobilenet.bin
 # There are model files(mobilenet*) and wasm files(wasi-nn-example.wasm) in the directory,
 # /workspaces/wasi-nn/rust/examples/classification-example/build

@ -52,14 +51,35 @@ RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git .
 WORKDIR /tmp
 RUN wget -q https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh \
  && chmod a+x ./install.sh
-RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite
+# RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite wasi_nn-openvino
+RUN ./install.sh -r yes -D -p /opt/wasmedge --plugins wasi_nn-openvino --dist ubuntu20.04 \
+  && /opt/wasmedge/bin/wasmedge --version
 ENV PATH=/opt/wasmedge/bin:${PATH}
-ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib
+# ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib

 #
 # wasmedge-wasinn-examples
 WORKDIR /workspaces/wasmedge-wasinn-examples
 RUN git clone --depth 1 https://github.com/second-state/WasmEdge-WASINN-examples.git .
+COPY core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch .
+RUN git apply ./bump_wasi_nn_to_0_6_0.patch
+# recompile with wasi-nn 0.6.0
+RUN cd openvino-mobilenet-image/rust && cargo build --target=wasm32-wasi
+RUN cd openvino-mobilenet-raw/rust && cargo build --target=wasm32-wasi
+RUN cd openvino-road-segmentation-adas/openvino-road-seg-adas && cargo build --target=wasm32-wasi
+RUN cd tflite-birds_v1-image/rust && cargo build --target=wasm32-wasi
+
+# preparation
+RUN cd openvino-mobilenet-image \
+  && ./download_mobilenet.sh . \
+  && ls -l mobilenet.xml mobilenet.bin
+
+RUN cd openvino-mobilenet-raw \
+  && ./download_mobilenet.sh . \
+  && ls -l mobilenet.xml mobilenet.bin tensor-1x224x224x3-f32.bgr
+
+# RUN apt update \
+#   && apt install -y valgrind

 #
 # iwasm. build from source
@ -67,7 +87,10 @@ WORKDIR /workspaces/wamr
 COPY . .

 WORKDIR /workspaces/wamr/product-mini/platforms/linux
-RUN cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \
+RUN OpenVINO_DIR=/usr/lib/openvino-2023.2.0 \
+    cmake -S . -B build \
+    -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \
+    -DWAMR_BUILD_WASI_NN_OPENVINO=1 -DWAMR_BUILD_WASI_NN_TFLITE=1 \
  && cmake --build build
 RUN ln -sf "$(realpath ./build/iwasm)" /usr/local/bin/iwasm

--- a/core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch
+++ b/core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch
@ -0,0 +1,47 @@
+diff --git a/openvino-mobilenet-image/rust/Cargo.toml b/openvino-mobilenet-image/rust/Cargo.toml
+index d09e0a4..c7083fb 100644
+--- a/openvino-mobilenet-image/rust/Cargo.toml
+++ b/openvino-mobilenet-image/rust/Cargo.toml
+@@ -8,6 +8,6 @@ publish = false
+ 
+ [dependencies]
+ image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"]  }
+-wasi-nn = { version = "0.4.0" }
+wasi-nn = { version = "0.6.0" }
+ 
+ [workspace]
+diff --git a/openvino-mobilenet-raw/rust/Cargo.toml b/openvino-mobilenet-raw/rust/Cargo.toml
+index 8eab25b..3f00aec 100644
+--- a/openvino-mobilenet-raw/rust/Cargo.toml
+++ b/openvino-mobilenet-raw/rust/Cargo.toml
+@@ -7,6 +7,6 @@ edition = "2021"
+ publish = false
+ 
+ [dependencies]
+-wasi-nn = { version = "0.4.0" }
+wasi-nn = { version = "0.6.0" }
+ 
+ [workspace]
+diff --git a/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml b/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml
+index 998f391..93f91e0 100644
+--- a/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml
+++ b/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml
+@@ -5,5 +5,5 @@ name = "openvino-road-seg-adas"
+ version = "0.2.0"
+ 
+ [dependencies]
+-wasi-nn = "0.4.0"
+wasi-nn = "0.6.0"
+ image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"]  }
+diff --git a/tflite-birds_v1-image/rust/Cargo.toml b/tflite-birds_v1-image/rust/Cargo.toml
+index 572ecb9..9e89e87 100644
+--- a/tflite-birds_v1-image/rust/Cargo.toml
+++ b/tflite-birds_v1-image/rust/Cargo.toml
+@@ -8,6 +8,6 @@ publish = false
+ 
+ [dependencies]
+ image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"]  }
+-wasi-nn = "0.4.0"
+wasi-nn = "0.6.0"
+ 
+ [workspace]
--- a/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py
+++ b/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py
@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #

+from dataclasses import dataclass
 from pathlib import Path
 from pprint import pprint
 import re
@ -13,72 +14,151 @@ import subprocess
 from typing import List


-def execute_tflite_birds_v1_image_once(
-    runtime_bin: str, runtime_args: List[str], cwd: Path
+@dataclass
+class WasmEdgeExampleResult:
+    class_id: int
+    possibility: float
+
+
+def execute_once(
+    runtime_bin: str,
+    runtime_args: List[str],
+    wasm_file: str,
+    wasm_args: List[str],
+    cwd: Path,
 ) -> str:
-    """
-    execute tflite_birds_v1_image example with
-
-    ```
-    iwasm --native-lib=somewhere/libwasi-nn-tflite.so --map-dir=.:. \
-        ./wasmedge-wasinn-example-tflite-bird-image.wasm  \
-        lite-model_aiy_vision_classifier_birds_V1_3.tflite \
-        bird.jpg
-    ```
-
-    or
-
-    ```
-    wasmedge --dir=.:. \
-        ./wasmedge-wasinn-example-tflite-bird-image.wasm  \
-        lite-model_aiy_vision_classifier_birds_V1_3.tflite \
-        bird.jpg
-    ```
-
-    assumption:
-    - under the right directory, tflite-birds_v1-image
-    - every materials are ready
-    """
-
-    wasm_file = "./wasmedge-wasinn-example-tflite-bird-image.wasm"
-    wasm_args = ["lite-model_aiy_vision_classifier_birds_V1_3.tflite", "bird.jpg"]
-
    cmd = [runtime_bin]
    cmd.extend(runtime_args)
    cmd.append(wasm_file)
    cmd.extend(wasm_args)

-    try:
-        p = subprocess.run(
-            cmd,
-            cwd=cwd,
-            capture_output=True,
-            check=True,
-            text=True,
-            universal_newlines=True,
-        )
-        return p.stdout
-    except subprocess.CalledProcessError as e:
-        print(e.stderr)
-        print()
-        print(e.stdout)
+    # print(f'Execute: {" ".join(cmd)}')
+
+    p = subprocess.run(
+        cmd,
+        cwd=cwd,
+        capture_output=True,
+        check=True,
+        text=True,
+        universal_newlines=True,
+    )
+    return p.stdout


-def filter_output_tflite_birds_v1_image(output: str) -> List[str]:
+def execute_openvino_road_segmentation_adas_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-road-segmentation-adas with iwasm and wasmedge
    """
-    not all output is needed for comparision

-    pick lines like: "   1.) [526](136)Cathartes burrovianus"
+    wasm_file = (
+        "./openvino-road-seg-adas/target/wasm32-wasi/debug/openvino-road-seg-adas.wasm"
+    )
+    wasm_args = [
+        "./model/road-segmentation-adas-0001.xml",
+        "./model/road-segmentation-adas-0001.bin",
+        "./image/empty_road_mapillary.jpg",
+    ]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def execute_openvino_mobilenet_raw_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-mobilenet-image with iwasm and wasmedge
+    """
+
+    wasm_file = "./rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-mobilenet.wasm"
+    wasm_args = [
+        "mobilenet.xml",
+        "mobilenet.bin",
+        "./tensor-1x224x224x3-f32.bgr",
+    ]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def execute_openvino_mobilenet_image_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-mobilenet-image with iwasm and wasmedge
+    """
+
+    wasm_file = (
+        "./rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-mobilenet-image.wasm"
+    )
+    wasm_args = [
+        "mobilenet.xml",
+        "mobilenet.bin",
+        "input.jpg",
+    ]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def execute_tflite_birds_v1_image_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-mobilenet-image with iwasm and wasmedge
+    """
+
+    wasm_file = (
+        "rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-tflite-bird-image.wasm"
+    )
+    wasm_args = ["lite-model_aiy_vision_classifier_birds_V1_3.tflite", "bird.jpg"]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def filter_output(output: str) -> List[WasmEdgeExampleResult]:
+    """
+    not all output is required for comparison
+
+    pick lines like: " 1.) [166](198)Aix galericulata"
    """
    filtered = []
-    PATTERN = re.compile(r"^\s+\d\.\)\s+\[\d+\]\(\d+\)\w+")
+    PATTERN = re.compile(r"^\s+\d\.\)\s+\[(\d+)\]\(([.0-9]+)\)\w+")
    for line in output.split("\n"):
-        if PATTERN.search(line):
-            filtered.append(line.strip())
+        m = PATTERN.search(line)
+        if m:
+            class_id, possibility = m.groups()
+            filtered.append(WasmEdgeExampleResult(class_id, possibility))

+    assert len(filtered)
    return filtered


+def compare_output(
+    iwasm_output: List[WasmEdgeExampleResult],
+    wasmedge_output: List[WasmEdgeExampleResult],
+) -> bool:
+    """
+    only compare top 2 and ignore possibility
+    """
+    return (iwasm_output[0].class_id, iwasm_output[1].class_id) == (
+        wasmedge_output[0].class_id,
+        wasmedge_output[1].class_id,
+    )
+
+
+def summarizer_result(
+    example_name: str,
+    iwasm_output: List[WasmEdgeExampleResult],
+    wasmedge_output: List[WasmEdgeExampleResult],
+):
+    if compare_output(iwasm_output, wasmedge_output):
+        print(f"- {example_name}. PASS")
+        return
+
+    print(f"- {example_name}. FAILED")
+    print("------------------------------------------------------------")
+    pprint(iwasm_output)
+    print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
+    pprint(wasmedge_output)
+    print("------------------------------------------------------------")
+
+
 def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
    iwasm_output = execute_tflite_birds_v1_image_once(
        iwasm_bin,
@ -88,33 +168,124 @@ def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
        ],
        cwd,
    )
-    iwasm_output = filter_output_tflite_birds_v1_image(iwasm_output)
+    iwasm_output = filter_output(iwasm_output)

    wasmedge_output = execute_tflite_birds_v1_image_once(
        wasmedge_bin, ["--dir=.:."], cwd
    )
-    wasmedge_output = filter_output_tflite_birds_v1_image(wasmedge_output)
+    wasmedge_output = filter_output(wasmedge_output)

-    if iwasm_output == wasmedge_output:
-        print("- tflite_birds_v1_image. PASS")
+    summarizer_result("tf_lite_birds_v1_image", iwasm_output, wasmedge_output)
+
+
+def execute_openvino_mobilenet_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
+    iwasm_output = execute_openvino_mobilenet_image_once(
+        iwasm_bin,
+        [
+            "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so",
+            "--map-dir=.:.",
+        ],
+        cwd,
+    )
+    iwasm_output = filter_output(iwasm_output)
+
+    wasmedge_output = execute_openvino_mobilenet_image_once(
+        wasmedge_bin, ["--dir=.:."], cwd
+    )
+    wasmedge_output = filter_output(wasmedge_output)
+
+    summarizer_result("openvino_mobile_image", iwasm_output, wasmedge_output)
+
+
+def execute_openvino_mobilenet_raw(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
+    iwasm_output = execute_openvino_mobilenet_raw_once(
+        iwasm_bin,
+        [
+            "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so",
+            "--map-dir=.:.",
+        ],
+        cwd,
+    )
+    iwasm_output = filter_output(iwasm_output)
+
+    wasmedge_output = execute_openvino_mobilenet_raw_once(
+        wasmedge_bin, ["--dir=.:."], cwd
+    )
+    wasmedge_output = filter_output(wasmedge_output)
+
+    summarizer_result("openvino_mobile_raw", iwasm_output, wasmedge_output)
+
+
+def execute_openvino_road_segmentation_adas(
+    iwasm_bin: str, wasmedge_bin: str, cwd: Path
+):
+    def filter_output(output: str) -> str:
+        """
+        focus on lines:
+           The size of the output buffer is 7340032 bytes
+           dump tensor to "wasinn-openvino-inference-output-1x4x512x896xf32.tensor"
+        """
+        for line in output.split("\n"):
+            if "The size of the output buffer is" in line:
+                dump_tensor_size = int(line.split(" ")[-2])
+                continue
+
+            if "dump tensor to " in line:
+                dump_tensor_file = line.split(" ")[-1]
+                continue
+
+        return (dump_tensor_file, dump_tensor_size)
+
+    iwasm_output = execute_openvino_road_segmentation_adas_once(
+        iwasm_bin,
+        [
+            "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so",
+            "--map-dir=.:.",
+        ],
+        cwd,
+    )
+    iwasm_tensor_file, iwasm_tensor_size = filter_output(iwasm_output)
+
+    wasmedge_output = execute_openvino_road_segmentation_adas_once(
+        wasmedge_bin, ["--dir=.:."], cwd
+    )
+    wasmedge_tensor_file, wasmedge_tensor_size = filter_output(wasmedge_output)
+
+    # TODO: binary compare?
+    if iwasm_tensor_size == wasmedge_tensor_size:
+        print(f"- openvino_road_segmentation_adas. PASS")
        return

-    print("- tflite_birds_v1_image. FAILED")
+    print(f"- openvino_road_segmentation_adas. FAILED")
    print("------------------------------------------------------------")
-    pprint(iwasm_output)
+    print(f"FILE:{iwasm_tensor_file}, SIZE:{iwasm_tensor_size}")
    print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
-    pprint(wasmedge_output)
+    print(f"FILE:{wasmedge_tensor_file}, SIZE:{wasmedge_tensor_size}")
    print("------------------------------------------------------------")


-def execute_wasmedge_wasinn_exmaples(iwasm_bin: str, wasmedge_bin: str):
+def execute_wasmedge_wasinn_examples(iwasm_bin: str, wasmedge_bin: str):
    assert Path.cwd().name == "wasmedge-wasinn-examples"
    assert shutil.which(iwasm_bin)
    assert shutil.which(wasmedge_bin)

-    tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image")
-    execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir)
+    # TODO: keep commenting until https://github.com/bytecodealliance/wasm-micro-runtime/pull/3597 is merged
+    # tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image")
+    # execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir)
+
+    openvino_mobile_image_dir = Path.cwd().joinpath("./openvino-mobilenet-image")
+    execute_openvino_mobilenet_image(iwasm_bin, wasmedge_bin, openvino_mobile_image_dir)
+
+    openvino_mobile_raw_dir = Path.cwd().joinpath("./openvino-mobilenet-raw")
+    execute_openvino_mobilenet_raw(iwasm_bin, wasmedge_bin, openvino_mobile_raw_dir)
+
+    openvino_road_segmentation_adas_dir = Path.cwd().joinpath(
+        "./openvino-road-segmentation-adas"
+    )
+    execute_openvino_road_segmentation_adas(
+        iwasm_bin, wasmedge_bin, openvino_road_segmentation_adas_dir
+    )


 if __name__ == "__main__":
-    execute_wasmedge_wasinn_exmaples("iwasm", "wasmedge")
+    execute_wasmedge_wasinn_examples("iwasm", "wasmedge")