diff --git a/.github/workflows/compilation_on_android_ubuntu_macos.yml b/.github/workflows/compilation_on_android_ubuntu_macos.yml index d76b810c..25ef7da3 100644 --- a/.github/workflows/compilation_on_android_ubuntu_macos.yml +++ b/.github/workflows/compilation_on_android_ubuntu_macos.yml @@ -28,11 +28,11 @@ concurrency: cancel-in-progress: true env: - AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" - FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" - JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" - LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" + JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" + LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" jobs: @@ -118,9 +118,13 @@ jobs: if: ${{ matrix.light == 'green' }} uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ${{ matrix.os }}-${{env.LLVM_CACHE_SUFFIX }} - name: Build llvm and clang from source on ubuntu id: build_llvm_ubuntu @@ -134,11 +138,6 @@ jobs: run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly working-directory: build-scripts - - name: package LLVM - if: ${{ matrix.light == 'green' && (steps.build_llvm_ubuntu.conclusion == 'success' || steps.build_llvm_macos.conclusion == 'success')}} - run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz - working-directory: core/deps/llvm/build/ - build_wamrc: needs: [build_llvm_libraries, check_repo] runs-on: ${{ matrix.os }} @@ -165,19 +164,18 @@ jobs: if: ${{ matrix.light == 'green' }} uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - name: Quit if cache miss if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }} run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Extract the LLVM package - if: ${{ matrix.light == 'green' }} - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - - name: Build wamrc if: ${{ matrix.light == 'green' }} run: | @@ -293,19 +291,18 @@ jobs: if: ${{ matrix.light == 'green' }} uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - name: Quit if cache miss if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }} run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Extract the LLVM package - if: ${{ matrix.light == 'green' }} - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - - name: Build iwasm if: ${{ matrix.light == 'green' }} run: | @@ -362,19 +359,18 @@ jobs: if: ${{ matrix.light == 'green' }} uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - name: Quit if cache miss if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }} run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Extract the LLVM package - if: ${{ matrix.light == 'green' }} - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - - name: download and install wabt if: ${{ matrix.light == 'green' }} run: | @@ -440,19 +436,18 @@ jobs: if: ${{ matrix.light == 'green' }} uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }} - name: Quit if cache miss if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }} run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Extract the LLVM package - if: ${{ matrix.light == 'green' }} - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - - name: download and install wasi-sdk if: ${{ matrix.light == 'green' }} run: | diff --git a/.github/workflows/spec_test.yml b/.github/workflows/spec_test.yml index 1daa19a2..cab19696 100644 --- a/.github/workflows/spec_test.yml +++ b/.github/workflows/spec_test.yml @@ -49,38 +49,12 @@ jobs: with: access_token: ${{ github.token }} - build_llvm_libraries: - needs: cancel_previous - runs-on: ubuntu-20.04 - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Cache LLVM libraries - id: cache_llvm - uses: actions/cache@v2 - with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz - key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - - - name: Build llvm and clang from source - id: build_llvm - if: ${{ steps.cache_llvm.outputs.cache-hit != 'true' }} - run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly - working-directory: build-scripts - - - name: package LLVM - if: ${{ steps.build_llvm.conclusion == 'success' }} - run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz - working-directory: core/deps/llvm/build/ - spec_test_default: - needs: build_llvm_libraries + needs: cancel_previous runs-on: ubuntu-20.04 strategy: matrix: - test_option: [ $DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS ] + test_option: [$DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS] steps: - name: checkout uses: actions/checkout@v2 @@ -89,17 +63,13 @@ jobs: id: cache_llvm uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - - - name: Quit if cache miss - if: steps.cache_llvm.outputs.cache-hit != 'true' - run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - - name: Extract the LLVM package - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - name: install Ninja run: sudo apt install -y ninja-build @@ -109,12 +79,12 @@ jobs: working-directory: ./tests/wamr-test-suites spec_test_extra: - needs: build_llvm_libraries + needs: cancel_previous if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }} runs-on: ubuntu-20.04 strategy: matrix: - test_option: [ $MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS ] + test_option: [$MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS] steps: - name: checkout uses: actions/checkout@v2 @@ -123,18 +93,18 @@ jobs: id: cache_llvm uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - name: Quit if cache miss if: steps.cache_llvm.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Extract the LLVM package - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - - name: install Ninja run: sudo apt install -y ninja-build @@ -144,11 +114,11 @@ jobs: spec_test_x86_32: if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }} - needs: build_llvm_libraries + needs: cancel_previous runs-on: ubuntu-20.04 strategy: matrix: - test_option: [ $DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS ] + test_option: [$DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS] steps: - name: checkout uses: actions/checkout@v2 @@ -157,18 +127,18 @@ jobs: id: cache_llvm uses: actions/cache@v2 with: - path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }} - name: Quit if cache miss if: steps.cache_llvm.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Extract the LLVM package - run: tar xf LLVM-13.0.0.tar.gz --strip-components=1 - working-directory: ./core/deps/llvm/build - - name: install Ninja and x32 support libraries run: sudo apt install -y g++-multilib libgcc-9-dev lib32gcc-9-dev ninja-build diff --git a/build-scripts/build_llvm.py b/build-scripts/build_llvm.py index 1a952799..37c94d08 100755 --- a/build-scripts/build_llvm.py +++ b/build-scripts/build_llvm.py @@ -99,7 +99,7 @@ def build_llvm(llvm_dir, platform, backends, projects): lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve() if lib_llvm_core_library.exists(): print(f"Please remove {build_dir} manually and try again") - return + return build_dir compile_options = " ".join( LLVM_COMPILE_OPTIONS @@ -126,6 +126,28 @@ def build_llvm(llvm_dir, platform, backends, projects): return build_dir +def repackage_llvm(llvm_dir): + build_dir = llvm_dir.joinpath("./build").resolve() + + packs = [f for f in build_dir.glob("LLVM-13*.tar.gz")] + if len(packs) > 1: + raise Exception("Find more than one LLVM-13*.tar.gz") + + if not packs: + return + + llvm_package = packs[0].name + # mv build/LLVM-13.0.0*.gz . + shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir)) + # rm -r build + shutil.rmtree(str(build_dir)) + # mkdir build + build_dir.mkdir() + # tar xf ./LLVM-13.0.0-*.tar.gz --strip-components=1 --directory=build + CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}" + subprocess.check_call(shlex.split(CMD), cwd=llvm_dir) + + def main(): parser = argparse.ArgumentParser(description="build necessary LLVM libraries") parser.add_argument( @@ -198,16 +220,24 @@ def main(): current_dir = current_file.parent.resolve() deps_dir = current_dir.joinpath("../core/deps").resolve() - print(f"==================== CLONE LLVM ====================") - llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"]) - llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"]) + try: + print(f"==================== CLONE LLVM ====================") + llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"]) + llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"]) - print() - print(f"==================== BUILD LLVM ====================") - build_llvm(llvm_dir, platform, options.arch, options.project) + print() + print(f"==================== BUILD LLVM ====================") + build_llvm(llvm_dir, platform, options.arch, options.project) - print() + print() + print(f"==================== PACKAGE LLVM ====================") + repackage_llvm(llvm_dir) + + print() + return True + except subprocess.CalledProcessError: + return False if __name__ == "__main__": - main() + sys.exit(0 if main() else 1)