From 877d91b531a23a77d81a334f4e92d7ff7f55018e Mon Sep 17 00:00:00 2001 From: "Meng, Hengyu" Date: Mon, 19 Aug 2024 05:42:49 +0000 Subject: [PATCH] mmvq in cuda path --- ggml/src/ggml-sycl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl.cpp b/ggml/src/ggml-sycl.cpp index a455188211d4e..48084e524c78f 100644 --- a/ggml/src/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl.cpp @@ -3581,7 +3581,8 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 - && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE && src1->ne[1] > MMVQ_MIN_BATCH_SIZE; + && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE + && ctx.stream()->get_backend() == sycl::backend::ext_oneapi_cuda ? true: src1->ne[1] > MMVQ_MIN_BATCH_SIZE; bool use_mul_mat_q = ggml_sycl_supports_mmq(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;