Merge branch 'ggerganov:master' into master

sealad886 · Jun 16, 2024 · 4bc518f · 4bc518f
2 parents 75a5324 + 19b7a83
commit 4bc518f
Show file tree

Hide file tree

Showing 3 changed files with 285 additions and 368 deletions.
diff --git a/flake.lock b/flake.lock
diff --git a/ggml-cuda/mmvq.cu b/ggml-cuda/mmvq.cu
@@ -117,7 +117,7 @@ static __global__ void mul_mat_vec_q(
             tmp[j][i] = warp_reduce_sum(tmp[j][i]);
         }
 
-        if (threadIdx.x < rows_per_cuda_block) {
+        if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || row0 + threadIdx.x < nrows_dst)) {
             dst[j*nrows_dst + row0 + threadIdx.x] = tmp[j][threadIdx.x];
         }
     }