diff --git a/.github/workflows/ci-linux-cleanroom.yml b/.github/workflows/ci-linux-cleanroom.yml new file mode 100644 index 000000000..f7a46a95d --- /dev/null +++ b/.github/workflows/ci-linux-cleanroom.yml @@ -0,0 +1,216 @@ +name: CI Ubuntu + +on: + workflow_call: + workflow_dispatch: + inputs: + force_debug_with_tmate: + type: boolean + description: 'Run the build with tmate session' + required: false + default: false + debug_with_tmate: + type: boolean + description: 'Run the build with a tmate session ONLY in case of failure' + required: false + default: false + pull_request: + merge_group: + push: + branches: + - main + +concurrency: + group: ci-build-test-cpp-ubuntu-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + build_and_ctest: + name: Build and Test (ubuntu, ASSERTIONS) + runs-on: ubuntu-22.04 + strategy: + fail-fast: true + env: + CACHE_DIR: ${{ github.workspace }}/.container-cache + CACHE_KEY: ubuntu-build-test-cpp-asserts-v2-${{ format('{0}-{1}', github.ref_name, github.run_number) }} + steps: + - name: Set unified TZ + uses: szenius/set-timezone@v2.0 + with: + timezoneLinux: "Asia/Singapore" + timezoneMacos: "Asia/Singapore" + timezoneWindows: "Singapore Standard Time" + + - name: Checking out repository + env: + BRANCH_NAME: ${{ github.ref }} + REPO_ADDRESS: ${{ github.server_url }}/${{ github.repository }} + run: | + git init + git remote add origin $REPO_ADDRESS + git -c protocol.version=2 fetch --depth 1 origin $BRANCH_NAME + git reset --hard FETCH_HEAD + git -c submodule."third_party/torch-mlir".update=none \ + -c submodule."third_party/stablehlo".update=none \ + -c submodule."third_party/XRT".update=none \ + submodule update --init --recursive --depth 1 --single-branch -j 10 + + - name: System deps + run: | + sudo apt install ccache ninja-build + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Python deps + run: | + pip install "numpy<2" pyyaml "pybind11[global]==2.10.3" nanobind + + - name: Enable cache + uses: actions/cache/restore@v3 + with: + path: ${{ env.CACHE_DIR }} + key: ${{ env.CACHE_KEY }} + restore-keys: ubuntu-build-test-cpp-asserts- + + - name: Peano dep + run: | + bash build_tools/download_peano.sh + echo "PEANO_INSTALL_DIR=$PWD/llvm-aie" >> $GITHUB_ENV + + - name: Build packages + run: | + export cache_dir="${{ env.CACHE_DIR }}" + export CCACHE_COMPILERCHECK="string:$(clang --version)" + bash build_tools/build_llvm.sh + rm -rf llvm-build + export llvm_install_dir=$PWD/llvm-install + bash build_tools/build_test_cpp.sh + + - name: Create artifacts + if: ${{ !cancelled() }} + run: | + pushd third_party/iree/third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd + tar cf llvm-dist-ubuntu-$llvm_sha_short.tar llvm-install + tar cf iree-dist-ubuntu.tar iree-install + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ubuntu_x86_64_llvm_packages + path: llvm-dist-*.tar + if-no-files-found: warn + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ubuntu_x86_64_iree_packages + path: iree-dist-ubuntu.tar + if-no-files-found: warn + + - name: Save cache + uses: actions/cache/save@v3 + if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }} + with: + path: ${{ env.CACHE_DIR }} + key: ${{ env.CACHE_KEY }} + + - name: Start tmate session + if: ${{ (failure() && inputs.debug_with_tmate) || inputs.force_debug_with_tmate }} + uses: mxschmitt/action-tmate@v3.18 + with: + limit-access-to-actor: true + + test_linux: + name: E2E Test linux + needs: build_and_ctest + strategy: + fail-fast: false + matrix: + runs-on: [linux-phoenix] + runs-on: ${{ matrix.runs-on }} + env: + XILINXD_LICENSE_FILE: /opt/xilinx/Xilinx.lic + steps: + - name: "Checking out repository" # for test scripts + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + with: + submodules: false # not required for testbench + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: ubuntu_x86_64_iree_packages + + - name: Extract artifact + run: | + tar -xvf iree-dist-ubuntu.tar + echo "IREE_INSTALL_DIR=$PWD/iree-install" >> $GITHUB_ENV + echo "PYTHONPATH=$PWD/iree-install/python_packages/iree_compiler:$PWD/iree-install/python_packages/iree_runtime" >> $GITHUB_ENV + bash build_tools/download_peano.sh + echo "PEANO_INSTALL_DIR=$PWD/llvm-aie" >> $GITHUB_ENV + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Create venv and install dependencies + run: | + python -m venv .venv + source .venv/bin/activate + pip install -r tests/requirements.txt + + - name: Query device info + run: | + source .venv/bin/activate + echo "aie-metadata" + python build_tools/ci/amdxdna_driver_utils/amdxdna_ioctl.py --aie-metadata + echo "aie-version" + python build_tools/ci/amdxdna_driver_utils/amdxdna_ioctl.py --aie-version + echo "XRT_LITE_N_CORE_ROWS=$(python build_tools/ci/amdxdna_driver_utils/amdxdna_ioctl.py --num-rows)" >> $GITHUB_ENV + echo "XRT_LITE_N_CORE_COLS=$(python build_tools/ci/amdxdna_driver_utils/amdxdna_ioctl.py --num-cols)" >> $GITHUB_ENV + + - name : E2E comparison of AIE to llvm-cpu + run: | + source .venv/bin/activate + python build_tools/ci/cpu_comparison/run.py \ + test_aie_vs_cpu \ + $PWD/iree-install \ + $PWD/llvm-aie \ + --vitis-dir /opt/Xilinx/Vitis/2024.2 \ + --reset-npu-between-runs -v \ + --xrt_lite_n_core_rows=$XRT_LITE_N_CORE_ROWS \ + --xrt_lite_n_core_cols=$XRT_LITE_N_CORE_COLS + + - name: E2E correctness matmul test + run: | + # https://stackoverflow.com/a/17567422 + # shim_xdna::bo::map_drm_bo does an mmap with MAP_LOCKED + # which can fail if limit is to low + sudo prlimit -lunlimited --pid $$ + source .venv/bin/activate + bash build_tools/ci/run_matmul_test.sh \ + test_matmuls \ + iree-install \ + $PWD/llvm-aie \ + /opt/Xilinx/Vitis/2024.2 + + - name: Python tests + run: | + source .venv/bin/activate + pytest -v tests \ + --capture=tee-sys \ + --iree-install-dir=$PWD/iree-install \ + --peano-install-dir=$PWD/llvm-aie \ + --xrt_lite_n_core_rows=$XRT_LITE_N_CORE_ROWS \ + --xrt_lite_n_core_cols=$XRT_LITE_N_CORE_COLS + + - name: XRT-LITE tests + run: | + DEVICE_TEST_DIR="$PWD/iree-install/device_tests" + for t in $(ls $DEVICE_TEST_DIR); do + $DEVICE_TEST_DIR/$t --xrt_lite_n_core_rows=$XRT_LITE_N_CORE_ROWS --xrt_lite_n_core_cols=$XRT_LITE_N_CORE_COLS + done diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index ee14374b2..2d98a21fc 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -53,20 +53,10 @@ jobs: -c submodule."third_party/stablehlo".update=none \ -c submodule."third_party/XRT".update=none \ submodule update --init --recursive --depth 1 --single-branch -j 10 - - - name: Install deps - run: | - dnf install -y almalinux-release-devel epel-release - yum remove -y openssl-devel zlib-devel || true - yum install -y protobuf-devel protobuf-compiler tmate - name: Python deps run: | - pip install "numpy<2" pyyaml "pybind11[global]==2.10.3" nanobind pytest - - - name: Run Pytest - run: | - pytest build_tools/ci + pip install "numpy<2" pyyaml "pybind11[global]==2.10.3" nanobind - name: Enable cache uses: actions/cache/restore@v3 diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 79614c5b5..120394f4a 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -58,7 +58,10 @@ jobs: git remote add origin $REPO_ADDRESS git -c protocol.version=2 fetch --depth 1 origin $BRANCH_NAME git reset --hard FETCH_HEAD - git -c submodule."third_party/torch-mlir".update=none -c submodule."third_party/stablehlo".update=none -c submodule."src/runtime_src/core/common/aiebu".update=none submodule update --init --recursive --depth 1 --single-branch -j 10 + git -c submodule."third_party/torch-mlir".update=none \ + -c submodule."third_party/stablehlo".update=none \ + -c submodule."third_party/XRT".update=none \ + submodule update --init --recursive --depth 1 --single-branch -j 10 - uses: actions/setup-python@v4 with: diff --git a/README.md b/README.md index d42c62e19..03289fdef 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,6 @@ This repository contains an early-phase IREE compiler and runtime plugin for interfacing the AMD AIE accelerator to IREE. -## Architectural Overview - -![image](https://github.com/nod-ai/iree-amd-aie/assets/74956/3fa73139-5fdf-4658-86c3-0705352c4ea0) - - ## Developer Setup **Strong recommendation**: check the CI scripts @ [.github/workflows](.github/workflows) - they do a fresh checkout and build on every commit and are written to be read by a non-CI expert. @@ -33,7 +28,7 @@ or if you want a faster checkout git \ -c submodule."third_party/torch-mlir".update=none \ -c submodule."third_party/stablehlo".update=none \ - -c submodule."src/runtime_src/core/common/aiebu".update=none \ + -c submodule."third_party/XRT".update=none \ clone \ --recursive \ --shallow-submodules \ @@ -46,12 +41,10 @@ The above avoids cloning entire repo histories, and skips unused nested submodul ### Just show me the CMake -To configure and build with XRT runtime enabled - ``` cd iree-amd-aie cmake \ - -B $WHERE_YOU_WOULD_LIKE_TO_BUILD \ + -B \ -S third_party/iree \ -DIREE_CMAKE_PLUGIN_PATHS=$PWD \ -DIREE_BUILD_PYTHON_BINDINGS=ON \ @@ -62,9 +55,9 @@ cmake \ -DIREE_TARGET_BACKEND_DEFAULTS=OFF \ -DIREE_TARGET_BACKEND_LLVM_CPU=ON \ -DIREE_BUILD_TESTS=ON \ - -DIREE_EXTERNAL_HAL_DRIVERS=xrt \ - -DCMAKE_INSTALL_PREFIX=$WHERE_YOU_WOULD_LIKE_TO_INSTALL -cmake --build $WHERE_YOU_WOULD_LIKE_TO_BUILD + -DIREE_EXTERNAL_HAL_DRIVERS=xrt-lite \ + -DCMAKE_INSTALL_PREFIX= +cmake --build ``` ### Instructions @@ -73,9 +66,9 @@ The bare minimum configure command for IREE with the amd-aie plugin ``` cmake \ - -B $WHERE_YOU_WOULD_LIKE_TO_BUILD \ - -S $IREE_REPO_SRC_DIR \ - -DIREE_CMAKE_PLUGIN_PATHS=$IREE_AMD_AIE_REPO_SRC_DIR \ + -B \ + -S \ + -DIREE_CMAKE_PLUGIN_PATHS= \ -DIREE_BUILD_PYTHON_BINDINGS=ON ``` @@ -111,7 +104,7 @@ If you're "bringing your own LLVM", i.e., you have a prebuilt/compiled distribut -DIREE_BUILD_BUNDLED_LLVM=OFF ``` -In this case you will need to supply `-DLLVM_EXTERNAL_LIT=$SOMEWHERE` (e.g., `pip install lit; SOMEWHERE=$(which lit)`). +In this case you will need to supply `-DLLVM_EXTERNAL_LIT=` (e.g., `pip install lit; SOMEWHERE=$(which lit)`). Note, getting the right/matching build of LLVM, that works with IREE is tough (besides the commit hash, there are various flags to set). To enable adventurous users to avail themselves of `-DIREE_BUILD_BUNDLED_LLVM=OFF` we cache/store/save the LLVM distribution for every successful CI run. @@ -121,65 +114,18 @@ These can then be downloaded by checking the artifacts section of any recent CI

+## Testing + Lit tests specific to AIE can be run with something like ``` -cd $WHERE_YOU_WOULD_LIKE_TO_BUILD +cd ctest -R amd-aie ``` -Other tests which run on hardware and requiring XRT are in the `build_tools` subdirectory. - -## Runtime driver setup +Other tests, which run on device, are in the `build_tools` subdirectory. -To enable the runtime driver, you need to also enable the XRT HAL +## Architectural overview (out of date) -``` - -DIREE_EXTERNAL_HAL_DRIVERS=xrt -``` - -Additional IREE-specific flags are explained at [IREE's build instructions](https://iree.dev/building-from-source/getting-started/#quickstart-clone-and-build). To use Ninja instead of Make, and clang++ instead of g++, you can add - - -``` - -G Ninja \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER=clang -``` - - -### Ubuntu Dependencies - -XRT requires a number of packages. Here are the requirements for various operating systems - -``` -apt install \ - libcurl4-openssl-dev \ - libdrm-dev \ - libelf-dev \ - libprotobuf-dev \ - libudev-dev \ - pkg-config \ - protobuf-compiler \ - python3-pybind11 \ - systemtap-sdt-dev \ - uuid-dev -``` - -### RH Based Deps - -This is an incomplete list derived by adding what is needed to our development base manylinux (AlmaLinux 8) image. +![image](https://github.com/nod-ai/iree-amd-aie/assets/74956/3fa73139-5fdf-4658-86c3-0705352c4ea0) -``` -yum install \ - libcurl-devel \ - libdrm-devel \ - libudev-devel \ - libuuid-devel \ - ncurses-devel \ - pkgconfig \ - protobuf-compiler \ - protobuf-devel \ - systemtap-sdt-devel \ - uuid-devel -``` diff --git a/iree_compiler_plugin.cmake b/iree_compiler_plugin.cmake index a707091ca..958d6de46 100644 --- a/iree_compiler_plugin.cmake +++ b/iree_compiler_plugin.cmake @@ -17,12 +17,6 @@ if("xrt" IN_LIST IREE_EXTERNAL_HAL_DRIVERS) set(IREE_AMD_AIE_ENABLE_XRT_DRIVER ON) endif() -set(IREE_AMD_AIE_ENABLE_XRT_LITE_DRIVER OFF) -if("xrt-lite" IN_LIST IREE_EXTERNAL_HAL_DRIVERS) - message(STATUS "Enabling XRT-LITE build because it is an enabled HAL driver") - set(IREE_AMD_AIE_ENABLE_XRT_LITE_DRIVER ON) -endif() - if(IREE_AMD_AIE_ENABLE_XRT_DRIVER) include(iree_aie_xrt) endif() diff --git a/iree_runtime_plugin.cmake b/iree_runtime_plugin.cmake index 0bc5637b5..d8138465e 100644 --- a/iree_runtime_plugin.cmake +++ b/iree_runtime_plugin.cmake @@ -21,19 +21,10 @@ if("xrt" IN_LIST IREE_EXTERNAL_HAL_DRIVERS) set(IREE_AMD_AIE_ENABLE_XRT_DRIVER ON) endif() -set(IREE_AMD_AIE_ENABLE_XRT_LITE_DRIVER OFF) -if("xrt-lite" IN_LIST IREE_EXTERNAL_HAL_DRIVERS) - message(STATUS "Enabling XRT-LITE build because it is an enabled HAL driver") - set(IREE_AMD_AIE_ENABLE_XRT_LITE_DRIVER ON) -endif() - if(IREE_AMD_AIE_ENABLE_XRT_DRIVER) include(iree_aie_xrt) endif() - -if(IREE_AMD_AIE_ENABLE_XRT_DRIVER OR IREE_AMD_AIE_ENABLE_XRT_LITE_DRIVER) - include(iree_aie_bootgen) -endif() +include(iree_aie_bootgen) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/runtime/src AMD-AIE) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/experimental AMD-AIE-experimental) diff --git a/runtime/src/iree-amd-aie/CMakeLists.txt b/runtime/src/iree-amd-aie/CMakeLists.txt index d861c846d..8b67676cd 100644 --- a/runtime/src/iree-amd-aie/CMakeLists.txt +++ b/runtime/src/iree-amd-aie/CMakeLists.txt @@ -5,17 +5,17 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception if(IREE_AMD_AIE_ENABLE_XRT_DRIVER) - add_subdirectory(driver/xrt) + add_subdirectory(driver/xrt) endif() -if(IREE_AMD_AIE_ENABLE_XRT_LITE_DRIVER) - add_subdirectory(driver/xrt-lite) +if("xrt-lite" IN_LIST IREE_EXTERNAL_HAL_DRIVERS) + add_subdirectory(driver/xrt-lite) endif() -# Flatbuffer schema generation does not require XRT. Moreover the generated +# Flatbuffer schema generation does not require a driver but the generated # flatbuffer header files are used by the compiler to create artefacts # (.vmfb file), and so the schema sub-directory is required even when not -# building the XRT driver code. +# building driver code. add_subdirectory(schemas) # Contains libiree_aie_runtime, i.e., suitably encapsulated calls to aie-rt.