Bump to rocm 6.3.1.

Key changes: * Moved llvm/ -> lib/llvm/ * libamdhip64 now depends on rocprofiler * rccl switches to clang++ vs hipcc * rccl depends on rocprofiler * Drop 6.1 patches * Setup rpaths to account for llvm move (required adding some goo for doing project interception in cmake) * Move rocr to top-level and eliminate rocm-core-libs
nod-ai · Jan 15, 2025 · ff8d951 · ff8d951
1 parent 91781ae
commit ff8d951
Show file tree

Hide file tree

Showing 22 changed files with 212 additions and 644 deletions.
diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml
@@ -52,7 +52,7 @@ jobs:
 
       - name: Fetch sources
         run: |
-          ./build_tools/fetch_sources.py
+          ./build_tools/fetch_sources.py --depth 1
 
       # The full checkout is very large: ~16GB, 9 of which is GIT stuff.
       # So we delete the latter. This must be done after getting any git

diff --git a/.github/workflows/build_python_packages.yml b/.github/workflows/build_python_packages.yml
@@ -57,7 +57,7 @@ jobs:
 
       - name: Fetch sources
         run: |
-          ./build_tools/fetch_sources.py
+          ./build_tools/fetch_sources.py --depth 1
 
       # The full checkout is very large: ~16GB, 9 of which is GIT stuff.
       # So we delete the latter. This must be done after getting any git

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -25,11 +25,10 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
 endif()
 
 set(ROCM_MAJOR_VERSION 6)
-set(ROCM_MINOR_VERSION 1)
-set(ROCM_PATCH_VERSION 0)
+set(ROCM_MINOR_VERSION 3)
+set(ROCM_PATCH_VERSION 1)
 set(ROCM_VERSION
-  "${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}"
-  CACHE STRING "ROCM version")
+  "${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}")
 set(AMDGPU_TARGETS "gfx90a gfx940 gfx942 gfx1100" CACHE STRING "AMDGPU Targets")
 
 ################################################################################
@@ -49,7 +48,7 @@ set(STAGING_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/staging_install")
 
 option(ALWAYS_BUILD_SUBPROJECTS
   "Don't let the brittle CMake external project machinery decide if a sub-project needs to rebuild. Always run the underlying build."
-  ON)
+  OFF)
 set(FIND_PACKAGE_OPTIONS)
 
 macro(add_package_path PackageName path)
@@ -70,12 +69,14 @@ set(DEFAULT_CMAKE_ARGS
   "-DROCM_PATH=${STAGING_INSTALL_DIR}"
   "-DCPACK_PACKAGING_INSTALL_PREFIX=${STAGING_INSTALL_DIR}"
   "-DAMDGPU_TARGETS=${AMDGPU_TARGETS}"
+  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
   -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
   -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
    # On some distributions, this will install to lib64. We would like
    # consistency in built packages, so hard-code it.
    -DCMAKE_INSTALL_LIBDIR=lib
-   -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${CMAKE_CURRENT_SOURCE_DIR}/cmake/external_project_include.cmake
+   -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${CMAKE_CURRENT_SOURCE_DIR}/cmake/therock_external_project_include.cmake
 )
 if(CMAKE_C_VISIBILITY_PRESET)
   list(APPEND DEFAULT_CMAKE_ARGS ${CMAKE_C_VISIBILITY_PRESET})
@@ -103,6 +104,25 @@ ExternalProject_Add(
 
 add_package_path(ROCM "${STAGING_INSTALL_DIR}/share/rocm/cmake")
 
+################################################################################
+# rocprofiler-register
+# This is a stub that helps runtime libraries and profiles rendezvous
+################################################################################
+
+ExternalProject_Add(
+  rocprofiler-register
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocprofiler-register
+  SOURCE_DIR "${ROCM_GIT_DIR}/rocprofiler-register"
+  CMAKE_ARGS
+    ${DEFAULT_CMAKE_ARGS}
+    ${FIND_PACKAGE_OPTIONS}
+    -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}
+  USES_TERMINAL_CONFIGURE TRUE
+  USES_TERMINAL_BUILD TRUE
+  BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS}
+)
+add_package_path(rocprofiler-register "${STAGING_INSTALL_DIR}/lib/cmake/rocprofiler-register")
+
 ################################################################################
 # LLVM
 ################################################################################
@@ -118,33 +138,34 @@ ExternalProject_Add(
     ${FIND_PACKAGE_OPTIONS}
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     # We install llvm in its own sub-directory.
-    -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}/llvm
+    -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}/lib/llvm
   USES_TERMINAL_CONFIGURE TRUE
   USES_TERMINAL_BUILD TRUE
   BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS}
 )
 
-add_package_path(amd_comgr "${STAGING_INSTALL_DIR}/llvm/lib/cmake/amd_comgr")
-add_package_path(Clang "${STAGING_INSTALL_DIR}/llvm/lib/cmake/clang")
-add_package_path(LLD "${STAGING_INSTALL_DIR}/llvm/lib/cmake/lld")
-add_package_path(LLVM "${STAGING_INSTALL_DIR}/llvm/lib/cmake/llvm")
-add_package_path(AMDDeviceLibs "${STAGING_INSTALL_DIR}/llvm/lib/cmake/AMDDeviceLibs")
+add_package_path(amd_comgr "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/amd_comgr")
+add_package_path(Clang "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/clang")
+add_package_path(LLD "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/lld")
+add_package_path(LLVM "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/llvm")
+add_package_path(AMDDeviceLibs "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/AMDDeviceLibs")
 
 ################################################################################
-# rocm-core-libs
+# ROCR-Runtime
 ################################################################################
 
 ExternalProject_Add(
-  rocm-core-libs
+  ROCR-Runtime
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ROCR-Runtime
+  SOURCE_DIR "${ROCM_GIT_DIR}/ROCR-Runtime"
   DEPENDS
-    amd-llvm
-    rocm-cmake
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm-core-libs
-  SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/components/rocm-core-libs"
+    amd-llvm  # Build dep only
+    rocprofiler-register
   CMAKE_ARGS
     ${DEFAULT_CMAKE_ARGS}
     ${FIND_PACKAGE_OPTIONS}
     -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}
+    -DBUILD_SHARED_LIBS=ON
   USES_TERMINAL_CONFIGURE TRUE
   USES_TERMINAL_BUILD TRUE
   BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS}
@@ -153,6 +174,28 @@ ExternalProject_Add(
 add_package_path(hsakmt "${STAGING_INSTALL_DIR}/lib/cmake/hsakmt")
 add_package_path(hsa-runtime64 "${STAGING_INSTALL_DIR}/lib/cmake/hsa-runtime64")
 
+################################################################################
+# rocm-core
+################################################################################
+
+ExternalProject_Add(
+  rocm-core
+  DEPENDS
+    amd-llvm
+    rocm-cmake
+    ROCR-Runtime
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm-core
+  SOURCE_DIR "${ROCM_GIT_DIR}/rocm-core"
+  CMAKE_ARGS
+    ${DEFAULT_CMAKE_ARGS}
+    ${FIND_PACKAGE_OPTIONS}
+    -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}
+    "-DTHEROCK_INSTALL_RPATH=lib$<SEMICOLON>lib/llvm/lib"
+  USES_TERMINAL_CONFIGURE TRUE
+  USES_TERMINAL_BUILD TRUE
+  BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS}
+)
+
 ################################################################################
 # rocminfo
 ################################################################################
@@ -162,7 +205,7 @@ ExternalProject_Add(
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocminfo
   SOURCE_DIR "${ROCM_GIT_DIR}/rocminfo"
   DEPENDS
-    rocm-core-libs
+    rocm-core
   CMAKE_ARGS
     ${DEFAULT_CMAKE_ARGS}
     ${FIND_PACKAGE_OPTIONS}
@@ -207,7 +250,7 @@ ExternalProject_Add(
   DEPENDS
     amd-llvm
     hipcc
-    rocm-core-libs
+    rocm-core
   CMAKE_ARGS
     ${DEFAULT_CMAKE_ARGS}
     ${FIND_PACKAGE_OPTIONS}
@@ -244,7 +287,7 @@ ExternalProject_Add(
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm_smi_lib
   SOURCE_DIR "${ROCM_GIT_DIR}/rocm_smi_lib"
   DEPENDS
-    rocm-core-libs
+    rocm-core
   CMAKE_ARGS
     ${DEFAULT_CMAKE_ARGS}
     ${FIND_PACKAGE_OPTIONS}
@@ -280,25 +323,29 @@ ExternalProject_Add(
 # RCCL
 ################################################################################
 
-ExternalProject_Add(
-  rccl
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rccl
-  SOURCE_DIR "${ROCM_GIT_DIR}/rccl"
-  DEPENDS
-    clr
-    hipcc
-    hipify
-    rocminfo
-  CMAKE_ARGS
-    ${DEFAULT_CMAKE_ARGS}
-    ${FIND_PACKAGE_OPTIONS}
-    -DCMAKE_CXX_COMPILER=${STAGING_INSTALL_DIR}/bin/hipcc
-    -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}
-    -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
-  USES_TERMINAL_CONFIGURE TRUE
-  USES_TERMINAL_BUILD TRUE
-  BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS}
-)
+# TODO: Re-enable rccl after adapting to 6.3.x build option differences.
+# Now seems to depend on rocprofiler.
+# ExternalProject_Add(
+#   rccl
+#   PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rccl
+#   SOURCE_DIR "${ROCM_GIT_DIR}/rccl"
+#   DEPENDS
+#     clr
+#     hipcc
+#     hipify
+#     rocminfo
+#   CMAKE_ARGS
+#     ${DEFAULT_CMAKE_ARGS}
+#     ${FIND_PACKAGE_OPTIONS}
+#     # TODO: Audit how the 6.3.x build finds its C++ compiler.
+#     #-DCMAKE_CXX_COMPILER=${STAGING_INSTALL_DIR}/bin/hipcc
+#     -DCMAKE_CXX_COMPILER=${STAGING_INSTALL_DIR}/lib/llvm/bin/clang++
+#     -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}
+#     -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
+#   USES_TERMINAL_CONFIGURE TRUE
+#   USES_TERMINAL_BUILD TRUE
+#   BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS}
+# )
 
 ################################################################################
 # Testing

diff --git a/README.md b/README.md
@@ -34,10 +34,13 @@ Alternatively, for a specific ROCm version e.g. 6.1.0
 
 ```
 python ./build_tools/fetch_sources.py \
-  --manifest-url https://github.com/ROCm/ROCm.git \
-  --manifest-branch refs/tags/rocm-6.1.0
+  --manifest-url http://github.com/ROCm/ROCm.git \
+  --manifest-name tools/rocm-build/rocm-6.3.1.xml \
+  --manifest-branch roc-6.3.x
 ```
 
+TODO: Figure out why this only works with the http URL.
+
 This will also apply the patches to the downloaded source files.
 
 ## Manually
@@ -46,11 +49,9 @@ Checkout the latest development branch with
 ```
 mkdir ~/github/rocm
 cd ~/github/rocm
-repo init -u https://github.com/ROCm/ROCm.git
+repo init -u http://github.com/ROCm/ROCm.git -m tools/rocm-build/rocm-6.3.1.xml -b roc-6.3.x
 repo sync -j16
 ```
-Use `-b roc-6.0.x` if you need a specific branch of ROCm sources.  
-
 Checkout out latest LLVM sources
 
 ```

diff --git a/build_tools/apply_patches.sh b/build_tools/apply_patches.sh
@@ -25,25 +25,7 @@ function apply_patch() {
 
 echo "Running from $PWD"
 
-stash_changes clr
-apply_patch clr clr-disable-hipconfig-check.patch
-apply_patch clr clr-respect-no-versioned-soname.patch
-
-stash_changes rocm-cmake
-apply_patch rocm-cmake rocm-cmake-nocheck.patch
-
-stash_changes ROCT-Thunk-Interface
-apply_patch ROCT-Thunk-Interface ROCT-Thunk-Interface-header-nest.patch
-apply_patch ROCT-Thunk-Interface ROCT-Thunk-Interface-link-dl-libs.patch
-
-stash_changes ROCR-Runtime
-apply_patch ROCR-Runtime ROCR-Runtime-intree-build.patch
-
-stash_changes HIPIFY
-apply_patch HIPIFY hipify-install-headers-in-include-hipify.patch
-
-stash_changes rocm_smi_lib
-apply_patch rocm_smi_lib rocm_smi_lib-use-cmake-generator-expr-for-target-out-path-when-stripping.patch
-
-stash_changes rccl
-apply_patch rccl rccl-overwrite-generated-files.patch
+# Example:
+# stash_changes clr
+# apply_patch clr clr-disable-hipconfig-check.patch
+# apply_patch clr clr-respect-no-versioned-soname.patch
diff --git a/build_tools/fetch_sources.py b/build_tools/fetch_sources.py
@@ -5,31 +5,38 @@
 
 import argparse
 from pathlib import Path
+import shlex
 import subprocess
 import sys
 
 DEFAULT_SOURCES_DIR = Path(__file__).resolve().parent.parent / "sources"
 
 
-def exec(args: list[str], cwd: Path):
-    print(f"++ Exec: {args} (in {cwd})")
+def exec(args: list[str | Path], cwd: Path):
+    args = [str(arg) for arg in args]
+    print(f"++ Exec [{cwd}]$ {shlex.join(args)}")
     subprocess.check_call(args, cwd=str(cwd))
 
 
 def run(args):
     repo_dir: Path = args.dir
     print(f"Setting up repo in {repo_dir}")
     repo_dir.mkdir(exist_ok=True, parents=True)
+    repo_args = [
+        "repo",
+        "init",
+        "-v",
+        "-u",
+        args.manifest_url,
+        "-m",
+        args.manifest_name,
+        "-b",
+        args.manifest_branch,
+    ]
+    if args.depth:
+        repo_args.extend(["--depth", str(args.depth)])
     exec(
-        [
-            "repo",
-            "init",
-            "--manifest-url",
-            args.manifest_url,
-            "--depth=1",
-            "--manifest-branch",
-            args.manifest_branch
-        ],
+        repo_args,
         cwd=repo_dir,
     )
     exec(["repo", "sync", "-j16"] + args.projects, cwd=repo_dir)
@@ -51,14 +58,27 @@ def main(argv):
         "--dir", type=Path, help="Repo dir", default=DEFAULT_SOURCES_DIR
     )
     parser.add_argument(
-        "--manifest-url", type=str, help="Manifest repository location of ROCm",
-        default="https://github.com/nod-ai/ROCm.git"
+        "--manifest-url",
+        type=str,
+        help="Manifest repository location of ROCm",
+        default="http://github.com/ROCm/ROCm.git",
+    )
+    parser.add_argument(
+        "--manifest-name",
+        type=str,
+        help="Repo manifest name",
+        default="tools/rocm-build/rocm-6.3.1.xml",
     )
     parser.add_argument(
-        "--manifest-branch", type=str, help="Branch to sync with repo tool",
-        default="the-rock-main"
+        "--manifest-branch",
+        type=str,
+        help="Branch to sync with repo tool",
+        default="roc-6.3.x",
     )
     parser.add_argument("--no-patch", action="store_true", help="Disable patching")
+    parser.add_argument(
+        "--depth", type=int, help="Git depth to pass to repo", default=None
+    )
     parser.add_argument(
         "--projects",
         nargs="+",
@@ -73,8 +93,8 @@ def main(argv):
             "rocm-cmake",
             "rocm-core",
             "rocminfo",
+            "rocprofiler-register",
             "ROCR-Runtime",
-            "ROCT-Thunk-Interface",
         ],
     )
     args = parser.parse_args(argv)

diff --git a/cmake/extensions/README.txt b/cmake/extensions/README.txt
@@ -0,0 +1,8 @@
+When the super-project build system configures a sub-project, it injects a
+hook that performs additional pre and post-processing. A primary thing that
+does is look for include files here:
+
+  ${CMAKE_PROJECT_NAME}_pre.cmake
+  ${CMAKE_PROJECT_NAME}_post.cmake
+
+If found, they are included.