disable sdp_causal in phi-3 to fix overflow (#11157)

intel-analytics · May 28, 2024 · bc5008f · bc5008f
1 parent 33852bd
commit bc5008f
Showing 1 changed file with 9 additions and 8 deletions.
diff --git a/python/llm/src/ipex_llm/transformers/models/phi3.py b/python/llm/src/ipex_llm/transformers/models/phi3.py
@@ -139,14 +139,15 @@ def attention_forward(
         else:
             attn_output = xe_addons.sdp(query_states, key_states, value_states,
                                         attention_mask)
-    elif use_sdp_causal(q_len, kv_seq_len, self.head_dim, query_states, self.training):
-        import xe_addons
-        if isinstance(past_key_value, DynamicFp8Cache):
-            attn_output = xe_addons.sdp_fp8_causal(query_states, key_states,
-                                                   value_states, attention_mask)
-        else:
-            attn_output = xe_addons.sdp_causal(query_states, key_states,
-                                               value_states, attention_mask)
+    # disable sdp_causal to avoid overflow for now
+    # elif use_sdp_causal(q_len, kv_seq_len, self.head_dim, query_states, self.training):
+    #     import xe_addons
+    #     if isinstance(past_key_value, DynamicFp8Cache):
+    #         attn_output = xe_addons.sdp_fp8_causal(query_states, key_states,
+    #                                                value_states, attention_mask)
+    #     else:
+    #         attn_output = xe_addons.sdp_causal(query_states, key_states,
+    #                                            value_states, attention_mask)
     else:
         if isinstance(past_key_value, DynamicFp8Cache):
             key_states, value_states = restore_fp8_kv_cache(key_states, value_states,