christoph/wamr
1
Fork 0
Code Releases Activity

Upgrade toolkits (#1878)

Browse Source 
Upgrade the version of related toolkits:
- upgrade llvm to 15.0
- upgrade wasi-sdk to 19.0
- upgrade emsdk to 3.1.28
- upgrade wabt to 1.0.31
- upgrade binaryen to 111

And upgrade the CI scripts, sample workload build scripts, Dockerfiles, and documents.
This commit is contained in:
Wenyong Huang
2023-02-02 09:42:25 +08:00
committed by GitHub
parent 1614ce12fa
commit 27e7e160af
43 changed files with 817 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
test-tools/build-wasi-sdk/patches/wasi_libc.patch
View File

@ -1,13 +0,0 @@
diff --git a/expected/wasm32-wasi/predefined-macros.txt b/expected/wasm32-wasi/predefined-macros.txt
index c1bb19e..954f3b5 100644
--- a/expected/wasm32-wasi/predefined-macros.txt
+++ b/expected/wasm32-wasi/predefined-macros.txt
@@ -3002,6 +3002,8 @@
#define __alignof_is_defined 1
#define __bitop(x,i,o) ((x)[(i)/8] o (1<<(i)%8))
#define __bool_true_false_are_defined 1
+#define __clang_literal_encoding__ "UTF-8"
+#define __clang_wide_literal_encoding__ "UTF-32"
#define __inline inline
#define __restrict restrict
#define __tg_complex(fun,x) (__RETCAST_CX(x)( __FLTCX((x)+I) && __IS_FP(x) ? fun ## f (x) : __LDBLCX((x)+I) ? fun ## l (x) : fun(x) ))

15
test-tools/build-wasi-sdk/patches/wasi_sdk.patch
View File

@ -1,15 +0,0 @@
diff --git a/version.sh b/version.sh
index 8e7c44c..ff0d3ba 100755
--- a/version.sh
+++ b/version.sh
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
-set -e
-GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12)
-GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';)
-echo $GIT_PACKAGE_VERSION
+#set -e
+#GIT_DESCR=$(git describe --long --candidates=999 --match='wasi-sdk-*' --dirty='+m' --abbrev=12)
+#GIT_PACKAGE_VERSION=$(echo $GIT_DESCR | perl -ne 'if(/^wasi-sdk-(\d+)-(\d+)-g([0-9a-f]{7,12})([+]m)?$/) { if($2 == 0) { print "$1.$2$4" } else { print "$1.$2g$3$4" } exit } else { print "could not parse git description"; exit 1 }';)
+#echo $GIT_PACKAGE_VERSION
+echo wasi-sdk-13-eng

211
test-tools/build-wasi-sdk/build_wasi_sdk.py → test-tools/pick-up-emscripten-headers/collect_files.py
View File

@ -9,68 +9,35 @@ The script operates on such directories and files
|-- core
| `-- deps
| |-- emscripten
| `-- wasi-sdk
| `-- src
| |-- llvm-project
| `-- wasi-libc
|-- samples
| `-- workloads
| |-- include
`-- test-tools
|-- build-wasi-sdk
| |-- build_wasi_sdk.py
| |-- include
| `-- patches
`-- wasi-sdk
|-- bin
|-- lib
`-- share
`-- wasi-sysroot
|-- pick-up-emscripten_headers
| |-- collect_files.py
"""
import argparse
import hashlib
import logging
import os
import pathlib
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import urllib
import urllib.request
logger = logging.getLogger("build_wasi_sdk")
logger = logging.getLogger("pick-up-emscripten-headers")
external_repos = {
"config": {
"sha256": "302e5e7f3c4996976c58efde8b2f28f71d51357e784330eeed738e129300dc33",
"store_dir": "core/deps/wasi-sdk/src/config",
"strip_prefix": "config-191bcb948f7191c36eefe634336f5fc5c0c4c2be",
"url": "https://git.savannah.gnu.org/cgit/config.git/snapshot/config-191bcb948f7191c36eefe634336f5fc5c0c4c2be.tar.gz",
},
"emscripten": {
"sha256": "0904a65379aea3ea94087b8c12985b2fee48599b473e3bef914fec2e3941532d",
"sha256": "c5524755b785d8f4b83eb3214fdd3ac4b2e1b1a4644df4c63f06e5968f48f90e",
"store_dir": "core/deps/emscripten",
"strip_prefix": "emscripten-2.0.28",
"url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/2.0.28.tar.gz",
},
"llvm-project": {
"sha256": "dc5169e51919f2817d06615285e9da6a804f0f881dc55d6247baa25aed3cc143",
"store_dir": "core/deps/wasi-sdk/src/llvm-project",
"strip_prefix": "llvm-project-34ff6a75f58377f32a5046a29f55c4c0e58bee9e",
"url": "https://github.com/llvm/llvm-project/archive/34ff6a75f58377f32a5046a29f55c4c0e58bee9e.tar.gz",
},
"wasi-sdk": {
"sha256": "fc4fdb0e97b915241f32209492a7d0fab42c24216f87c1d5d75f46f7c70a553d",
"store_dir": "core/deps/wasi-sdk",
"strip_prefix": "wasi-sdk-1a953299860bbcc198ad8c12a21d1b2e2f738355",
"url": "https://github.com/WebAssembly/wasi-sdk/archive/1a953299860bbcc198ad8c12a21d1b2e2f738355.tar.gz",
},
"wasi-libc": {
"sha256": "f6316ca9479d3463eb1c4f6a1d1f659bf15f67cb3c1e2e83d9d11f188dccd864",
"store_dir": "core/deps/wasi-sdk/src/wasi-libc",
"strip_prefix": "wasi-libc-a78cd329aec717f149934d7362f57050c9401f60",
"url": "https://github.com/WebAssembly/wasi-libc/archive/a78cd329aec717f149934d7362f57050c9401f60.tar.gz",
},
"strip_prefix": "emscripten-3.0.0",
"url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/3.0.0.tar.gz",
}
}
# TOOD: can we use headers from wasi-libc and clang directly ?
@ -111,25 +78,25 @@ def unpack(tar_file, strip_prefix, dest_dir):
with tempfile.TemporaryDirectory() as tmp:
with tarfile.open(tar_file) as tar:
logger.debug(f"extract to {tmp}")
def is_within_directory(directory, target):
abs_directory = os.path.abspath(directory)
abs_target = os.path.abspath(target)
prefix = os.path.commonprefix([abs_directory, abs_target])
return prefix == abs_directory
def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
for member in tar.getmembers():
member_path = os.path.join(path, member.name)
if not is_within_directory(path, member_path):
raise Exception("Attempted Path Traversal in Tar File")
tar.extractall(path, members, numeric_owner=numeric_owner)
tar.extractall(path, members, numeric_owner=numeric_owner)
safe_extract(tar, tmp)
strip_prefix_dir = (
@ -160,12 +127,12 @@ def download_repo(name, root):
download_flag = store_dir.joinpath("DOWNLOADED")
if store_dir.exists() and download_flag.exists():
logger.info(
f"keep using '{store_dir.relative_to(root)}'. Or to remove it and try again"
f"bypass downloading '{store_dir.relative_to(root)}'. Or to remove it and try again if needs a new release"
)
return True
# download only when the target is neither existed nor broken
download_dir = pathlib.Path("/tmp/build_wasi_sdk/")
download_dir = pathlib.Path("/tmp/pick-up-emscripten-headers/")
download_dir.mkdir(exist_ok=True)
tar_name = pathlib.Path(external_repos[name]["url"]).name
@ -192,104 +159,67 @@ def download_repo(name, root):
download_flag.touch()
# leave download files in /tmp
logger.info(f"Has downloaed and stored in {store_dir.relative_to(root)}")
return True
def run_patch(patch_file, cwd):
if not patch_file.exists():
logger.error(f"{patch_file} not found")
def collect_headers(root, install_location):
if not install_location.exists():
logger.error(f"{install_location} does not found")
return False
with open(patch_file, "r") as f:
try:
PATCH_DRY_RUN_CMD = "patch -f -p1 --dry-run"
if subprocess.check_call(shlex.split(PATCH_DRY_RUN_CMD), stdin=f, cwd=cwd):
logger.error(f"patch dry-run {cwd} failed")
return False
install_flag = install_location.joinpath("INSTALLED").resolve()
if install_flag.exists():
logger.info(
f"bypass downloading '{install_location}'. Or to remove it and try again if needs a new one"
)
return True
PATCH_CMD = "patch -f -p1"
f.seek(0)
if subprocess.check_call(shlex.split(PATCH_CMD), stdin=f, cwd=cwd):
logger.error(f"patch {cwd} failed")
return False
except subprocess.CalledProcessError:
logger.error(f"patch {cwd} failed")
return False
return True
def build_and_install_wasi_sdk(root):
store_dir = root.joinpath(f'{external_repos["wasi-sdk"]["store_dir"]}').resolve()
if not store_dir.exists():
logger.error(f"{store_dir} does not found")
emscripten_home = root.joinpath(
f'{external_repos["emscripten"]["store_dir"]}'
).resolve()
if not emscripten_home.exists():
logger.error(f"{emscripten_home} does not found")
return False
# patch wasi-libc and wasi-sdk
patch_flag = store_dir.joinpath("PATCHED")
if not patch_flag.exists():
if not run_patch(
root.joinpath("test-tools/build-wasi-sdk/patches/wasi_libc.patch"),
store_dir.joinpath("src/wasi-libc"),
):
return False
if not run_patch(
root.joinpath("test-tools/build-wasi-sdk/patches/wasi_sdk.patch"), store_dir
):
return False
patch_flag.touch()
else:
logger.info("bypass the patch phase")
# build
build_flag = store_dir.joinpath("BUILDED")
if not build_flag.exists():
BUILD_CMD = "make build"
if subprocess.check_call(shlex.split(BUILD_CMD), cwd=store_dir):
logger.error(f"build wasi-sdk failed")
return False
build_flag.touch()
else:
logger.info("bypass the build phase")
# install
install_flag = store_dir.joinpath("INSTALLED")
binary_path = root.joinpath("test-tools").resolve()
if not install_flag.exists():
shutil.copytree(
str(store_dir.joinpath("build/install/opt").resolve()),
str(binary_path),
dirs_exist_ok=True,
)
# install headers
emscripten_headers = (
root.joinpath(external_repos["emscripten"]["store_dir"])
.joinpath("system")
.resolve()
)
wasi_sysroot_headers = binary_path.joinpath(
"wasi-sdk/share/wasi-sysroot/include"
).resolve()
for (src, dst) in emscripten_headers_src_dst:
src = emscripten_headers.joinpath(src)
dst = wasi_sysroot_headers.joinpath(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(src, dst)
install_flag.touch()
else:
logger.info("bypass the install phase")
emscripten_headers = emscripten_home.joinpath("system").resolve()
for (src, dst) in emscripten_headers_src_dst:
src = emscripten_headers.joinpath(src)
dst = install_location.joinpath(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(src, dst)
install_flag.touch()
logger.info(f"Has installed in {install_location}")
return True
def main():
parser = argparse.ArgumentParser(
description="collect headers from emscripten for workload compilation"
)
parser.add_argument(
"--install",
type=str,
required=True,
help="identify installation location",
)
parser.add_argument(
"--loglevel",
type=str,
default="INFO",
choices=[
"ERROR",
"WARNING",
"INFO",
],
help="the logging level",
)
options = parser.parse_args()
console = logging.StreamHandler()
console.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
logger.setLevel(logging.INFO)
logger.setLevel(getattr(logging, options.loglevel))
logger.addHandler(console)
logger.propagate = False
@ -305,12 +235,9 @@ def main():
if not download_repo(repo, root):
return False
# build wasi_sdk and install
if not build_and_install_wasi_sdk(root):
if not collect_headers(root, pathlib.Path(options.install)):
return False
# TODO install headers from emscripten
return True

12
test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile
View File

@ -25,17 +25,17 @@ RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/
# - wamr-sdk
## - download wasi-sdk with wget and set up to /opt/wasi-sdk
RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-14/wasi-sdk-14.0-linux.tar.gz \
&& tar -zxvf wasi-sdk-14.0-linux.tar.gz \
&& mv wasi-sdk-14.0 /opt/wasi-sdk/ \
&& rm -f wasi-sdk-14.0-linux.tar.gz
RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz \
&& tar -zxvf wasi-sdk-*-linux.tar.gz \
&& mv wasi-sdk-19.0 /opt/wasi-sdk/ \
&& rm -f wasi-sdk-*-linux.tar.gz
## - clone wamr repo
RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git
RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git
WORKDIR /root/wasm-micro-runtime/wamr-compiler
RUN ./build_llvm.sh \
&& mkdir build
&& mkdir build
WORKDIR /root/wasm-micro-runtime/wamr-compiler/build
RUN cmake .. \