forked from ggerganov/llama.cpp
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introducing experimental OpenCL backend with support for Qualcomm Adr…
…eno GPUs (ggerganov#10693) * [cl][adreno] Add Adreno GPU support Add new OpenCL backend to support Adreno GPUs --------- Co-authored-by: Skyler Szot <[email protected]> Co-authored-by: Shangqing Gu <[email protected]> Co-authored-by: Alexander Angus <[email protected]> Co-authored-by: Hongqiang Wang <[email protected]> Co-authored-by: Max Krasnyansky <[email protected]> * [cl][ci] Add workflow for CL * [cl][adreno] Fix memory leak for non SMALL_ALLOC path * opencl: integrate backend dyn.load interface and fix compiler and format warnings * opencl: remove small-alloc support and fix build errors for non-opencl platforms * opencl: fixed merge conflict (MUSA added twice in cmake) * opencl-ci: use RUNNER_TEMP instead of github.workspace * opencl: fix embed tool invocation with python3 * opencl: CI workflow fixes * opencl: Clean up small-alloc in CMake files * opencl: cleanup ggml-opencl2 header file * opencl: use ulong for offsets and strides in ADD kernel * opencl: use cl_ulong for all offsets * opencl: use cl_ulong for sizes and strides * opencl: use `GGML_LOG_xxx` instead of `fprintf(stderr, ...)` * opencl: rename backend `opencl2` -> `opencl` * opencl: rename kernel files `ggml-opencl2` -> `ggml-opencl` * opencl: make OpenCL required, remove redundant lib and inc directories * `ggml-base`, `..` and `.` are added by `ggml_add_backend_library` * opencl: rename backend - funcs, structs, etc `opencl2` -> `opencl` * opencl: remove copyright marker since main license already covers * opencl: replace some more OPENCL2 leftovers * opencl: remove limits on `tensor_extra` * opencl: use pools for `tensor_extra` * opencl: fix compiler warnings with GCC and Clang Still getting the warning about clCreateCmdQueue being obsolete. Will fix that separately. * opencl: fail gracefully if opencl devices are not available Also for unsupported GPUs. * opencl: fix MSVC builds (string length error) * opencl: check for various requirements, allow deprecated API * opencl: update log message for unsupported GPUs --------- Co-authored-by: Skyler Szot <[email protected]> Co-authored-by: Shangqing Gu <[email protected]> Co-authored-by: Alexander Angus <[email protected]> Co-authored-by: Hongqiang Wang <[email protected]> Co-authored-by: Max Krasnyansky <[email protected]>
- Loading branch information
6 people
authored
Dec 13, 2024
1 parent
c27ac67
commit a76c56f
Showing
17 changed files
with
9,014 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| #ifndef GGML_OPENCL_H | ||
| #define GGML_OPENCL_H | ||
|
|
||
| #include "ggml.h" | ||
| #include "ggml-backend.h" | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| // | ||
| // backend API | ||
| // | ||
| GGML_BACKEND_API ggml_backend_t ggml_backend_opencl_init(void); | ||
| GGML_BACKEND_API bool ggml_backend_is_opencl(ggml_backend_t backend); | ||
|
|
||
| GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void); | ||
| GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void); | ||
|
|
||
| GGML_BACKEND_API ggml_backend_reg_t ggml_backend_opencl_reg(void); | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif // GGML_OPENCL_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| find_package(OpenCL REQUIRED) | ||
| find_package(Python3 REQUIRED) | ||
|
|
||
| set(TARGET_NAME ggml-opencl) | ||
|
|
||
| ggml_add_backend_library(${TARGET_NAME} | ||
| ggml-opencl.cpp | ||
| ../../include/ggml-opencl.h) | ||
| target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIBRARIES}) | ||
| target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_INCLUDE_DIRS}) | ||
|
|
||
| if (GGML_OPENCL_PROFILING) | ||
| message(STATUS "OpenCL profiling enabled (increases CPU overhead)") | ||
| add_compile_definitions(GGML_OPENCL_PROFILING) | ||
| endif () | ||
|
|
||
| add_compile_definitions(GGML_OPENCL_SOA_Q) | ||
|
|
||
| if (GGML_OPENCL_USE_ADRENO_KERNELS) | ||
| message(STATUS "OpenCL will use matmul kernels optimized for Adreno") | ||
| add_compile_definitions(GGML_OPENCL_USE_ADRENO_KERNELS) | ||
| endif () | ||
|
|
||
| if (GGML_OPENCL_EMBED_KERNELS) | ||
| add_compile_definitions(GGML_OPENCL_EMBED_KERNELS) | ||
|
|
||
| set(OPENCL_CL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl.cl.h") | ||
| set(OPENCL_MM_CL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_mm.cl.h") | ||
| set(OPENCL_CVT_CL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_cvt.cl.h") | ||
|
|
||
| set(OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_gemv_noshuffle.cl.h") | ||
| set(OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_gemv_noshuffle_general.cl.h") | ||
| set(OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_mul_mat_Ab_Bi_8x4.cl.h") | ||
| set(OPENCL_TRANSPOSE_16_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_transpose_16.cl.h") | ||
| set(OPENCL_TRANSPOSE_32_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_transpose_32.cl.h") | ||
| set(OPENCL_TRANSPOSE_32_16_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-opencl_transpose_32_16.cl.h") | ||
|
|
||
| set(EMBED_KERNEL_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/kernels/embed_kernel.py") | ||
| file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated") | ||
|
|
||
| include_directories("${CMAKE_BINARY_DIR}/autogenerated") | ||
|
|
||
| # Python must be accessible from command line | ||
| add_custom_command( | ||
| OUTPUT ${OPENCL_CL_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl.cl | ||
| ${OPENCL_CL_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_MM_CL_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_mm.cl | ||
| ${OPENCL_MM_CL_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_mm.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_mm.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_CVT_CL_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_cvt.cl | ||
| ${OPENCL_CVT_CL_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_cvt.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_cvt.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_gemv_noshuffle.cl | ||
| ${OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_gemv_noshuffle.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_gemv_noshuffle.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_gemv_noshuffle_general.cl | ||
| ${OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_gemv_noshuffle_general.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_gemv_noshuffle_general.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl | ||
| ${OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_mul_mat_Ab_Bi_8x4.cl.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_TRANSPOSE_16_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_transpose_16.cl | ||
| ${OPENCL_TRANSPOSE_16_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_transpose_16.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_transpose_16.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_TRANSPOSE_32_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_transpose_32.cl | ||
| ${OPENCL_TRANSPOSE_32_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_transpose_32.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_transpose_32.cl.h" | ||
| ) | ||
|
|
||
| add_custom_command( | ||
| OUTPUT ${OPENCL_TRANSPOSE_32_16_SOURCE_EMBED} | ||
| COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ggml-opencl_transpose_32_16.cl | ||
| ${OPENCL_TRANSPOSE_32_16_SOURCE_EMBED} | ||
| DEPENDS kernels/ggml-opencl_transpose_32_16.cl ${EMBED_KERNEL_SCRIPT} | ||
| COMMENT "Generate ggml-opencl_transpose_32_16.cl.h" | ||
| ) | ||
|
|
||
| target_sources(${TARGET_NAME} PRIVATE | ||
| ${OPENCL_CL_SOURCE_EMBED} | ||
| ${OPENCL_MM_CL_SOURCE_EMBED} | ||
| ${OPENCL_CVT_CL_SOURCE_EMBED} | ||
| ${OPENCL_GEMV_NOSHUFFLE_SOURCE_EMBED} | ||
| ${OPENCL_GEMV_NOSHUFFLE_GENERAL_SOURCE_EMBED} | ||
| ${OPENCL_MUL_MAT_Ab_Bi_8x4_SOURCE_EMBED} | ||
| ${OPENCL_TRANSPOSE_16_SOURCE_EMBED} | ||
| ${OPENCL_TRANSPOSE_32_SOURCE_EMBED} | ||
| ${OPENCL_TRANSPOSE_32_16_SOURCE_EMBED}) | ||
| else () | ||
| # copy ggml-opencl.cl to bin directory | ||
| configure_file(kernels/ggml-opencl.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_mm.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_mm.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_cvt.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_cvt.cl COPYONLY) | ||
|
|
||
| configure_file(kernels/ggml-opencl_gemv_noshuffle.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_gemv_noshuffle.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_gemv_noshuffle_general.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_gemv_noshuffle_general.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_mul_mat_Ab_Bi_8x4.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_mul_mat_Ab_Bi_8x4.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_transpose_16.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_transpose_16.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_transpose_32.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_transpose_32.cl COPYONLY) | ||
| configure_file(kernels/ggml-opencl_transpose_32_16.cl ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-opencl_transpose_32_16.cl COPYONLY) | ||
| endif () |
Oops, something went wrong.