From 4a3a5ff9052afb70bcacc3ac5e7ed557653bf11a Mon Sep 17 00:00:00 2001
From: Tim Law <tim@timrlaw.com>
Date: Mon, 25 Nov 2024 16:23:42 +0000
Subject: [PATCH] Add CMake option: AMReX_CUDA_OBJDIR_AS_TEMPDIR (#4246)

See issue #4245
---
 Docs/sphinx_documentation/source/GPU.rst | 2 ++
 Tools/CMake/AMReXCUDAOptions.cmake       | 3 +++
 Tools/CMake/AMReXParallelBackends.cmake  | 5 +++++
 Tools/CMake/AMReX_SetupCUDA.cmake        | 5 +++++
 4 files changed, 15 insertions(+)

diff --git a/Docs/sphinx_documentation/source/GPU.rst b/Docs/sphinx_documentation/source/GPU.rst
index c75440a27f..b4b7aba9b0 100644
--- a/Docs/sphinx_documentation/source/GPU.rst
+++ b/Docs/sphinx_documentation/source/GPU.rst
@@ -268,6 +268,8 @@ check the :ref:`table <tab:cmakecudavar>` below.
    +------------------------------+-------------------------------------------------+-------------+-----------------+
    | AMReX_CUDA_KEEP_FILES        |  Keep intermediately files (folder: nvcc_tmp)   | NO          | YES, NO         |
    +------------------------------+-------------------------------------------------+-------------+-----------------+
+   | AMReX_CUDA_OBJDIR_AS_TEMPDIR |  Place intermediate files in object file folder | NO          | YES, NO         |
+   +------------------------------+-------------------------------------------------+-------------+-----------------+
    | AMReX_CUDA_LTO               |  Enable CUDA link-time-optimization             | NO          | YES, NO         |
    +------------------------------+-------------------------------------------------+-------------+-----------------+
    | AMReX_CUDA_MAXREGCOUNT       |  Limits the number of CUDA registers available  | 255         | User-defined    |
diff --git a/Tools/CMake/AMReXCUDAOptions.cmake b/Tools/CMake/AMReXCUDAOptions.cmake
index 279b2d7250..71a6eb389f 100644
--- a/Tools/CMake/AMReXCUDAOptions.cmake
+++ b/Tools/CMake/AMReXCUDAOptions.cmake
@@ -102,3 +102,6 @@ cuda_print_option(AMReX_CUDA_BACKTRACE)
 
 option(AMReX_CUDA_KEEP_FILES "Keep intermediately generated files (folder: nvcc_tmp)" OFF)
 cuda_print_option(AMReX_CUDA_KEEP_FILES)
+
+option(AMReX_CUDA_OBJDIR_AS_TEMPDIR "Place intermediate files in object file folder" OFF)
+cuda_print_option(AMReX_CUDA_OBJDIR_AS_TEMPDIR)
diff --git a/Tools/CMake/AMReXParallelBackends.cmake b/Tools/CMake/AMReXParallelBackends.cmake
index cecf3fae5d..e7f631ca43 100644
--- a/Tools/CMake/AMReXParallelBackends.cmake
+++ b/Tools/CMake/AMReXParallelBackends.cmake
@@ -141,6 +141,11 @@ if (  AMReX_GPU_BACKEND STREQUAL "CUDA"
       list(APPEND _cuda_flags --keep "SHELL:--keep-dir ${PROJECT_BINARY_DIR}/nvcc_tmp")
    endif ()
 
+   # place intermediate files in object file folder
+   if (AMReX_CUDA_OBJDIR_AS_TEMPDIR)
+      list(APPEND _cuda_flags --objdir-as-tempdir)
+   endif ()
+
    # compilation timings
    if (AMReX_CUDA_COMPILATION_TIMER)
       file(REMOVE "${PROJECT_BINARY_DIR}/nvcc_timings.csv")
diff --git a/Tools/CMake/AMReX_SetupCUDA.cmake b/Tools/CMake/AMReX_SetupCUDA.cmake
index 4b73a66f85..d921f817b4 100644
--- a/Tools/CMake/AMReX_SetupCUDA.cmake
+++ b/Tools/CMake/AMReX_SetupCUDA.cmake
@@ -103,6 +103,11 @@ if (AMReX_CUDA_KEEP_FILES)
     string(APPEND CMAKE_CUDA_FLAGS " --keep --keep-dir ${PROJECT_BINARY_DIR}/nvcc_tmp")
 endif ()
 
+# place intermediate files in object file folder
+if (AMReX_CUDA_OBJDIR_AS_TEMPDIR)
+    string(APPEND CMAKE_CUDA_FLAGS " --objdir-as-tempdir")
+endif ()
+
 # compilation timings
 if (AMReX_CUDA_COMPILATION_TIMER)
     file(REMOVE "${PROJECT_BINARY_DIR}/nvcc_timings.csv")