diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index 55ce502dba1c7..66220a3a66eb7 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -1547,7 +1547,7 @@ def set_vocab(self):
 
     def _reverse_hf_permute(self, weights: Tensor, n_head: int, n_kv_head: int | None = None) -> Tensor:
         if n_kv_head is not None and n_head != n_kv_head:
-            n_head //= n_kv_head
+            n_head = n_kv_head
 
         return (
             weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])