diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9deab4bf0..0a41d27fd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -99,7 +99,11 @@ set(PORTBLAS_GENERATED_SRC ${CMAKE_CURRENT_BINARY_DIR}/generated_src)
 set(PORTBLAS_INCLUDE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   $<INSTALL_INTERFACE:include>)
 set(PORTBLAS_COMMON_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common/include)
-set(PORTBLAS_SRC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src> $<INSTALL_INTERFACE:src>)
+if(INSTALL_HEADER_ONLY)
+  set(PORTBLAS_SRC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src> $<INSTALL_INTERFACE:src>)
+else()
+  set(PORTBLAS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src)
+endif()
 set(PORTBLAS_SRC_GENERATOR ${CMAKE_CURRENT_SOURCE_DIR}/python_generator)
 list(APPEND THIRD_PARTIES_INCLUDE ${CBLAS_INCLUDE})