diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt
index 9f3873fb50fa8..506b6dc7b7454 100644
--- a/ggml/src/CMakeLists.txt
+++ b/ggml/src/CMakeLists.txt
@@ -574,7 +574,7 @@ if (GGML_SYCL)
         if (GGML_SYCL_TARGET STREQUAL "INTEL")
             list(APPEND GGML_EXTRA_LIBS OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread)
         elseif (GGML_SYCL_TARGET STREQUAL "NVIDIA")
-            list(APPEND CMAKE_CXX_FLAGS -fsycl-targets=nvptx64-nvidia-cuda)
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=nvptx64-nvidia-cuda")
             list(APPEND GGML_EXTRA_LIBS pthread m dl onemkl)
         endif()
     endif()