From 914ffdcabdd63919317ac5de94a652eede0a3f63 Mon Sep 17 00:00:00 2001 From: Jimin Ha Date: Wed, 25 Sep 2024 15:53:21 -0700 Subject: [PATCH] Fix gptj - bias error Transformer4.45 has the Dynamic cache updates removing self.bias causing the failure. Until we investigate further and update the code based on the new transformer, we are putting bias back to GaudiGPTJAttention.init() https://github.com/huggingface/transformers/pull/31421 --- optimum/habana/transformers/models/gptj/modeling_gptj.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/optimum/habana/transformers/models/gptj/modeling_gptj.py b/optimum/habana/transformers/models/gptj/modeling_gptj.py index b7f6951427..3927e1feb9 100644 --- a/optimum/habana/transformers/models/gptj/modeling_gptj.py +++ b/optimum/habana/transformers/models/gptj/modeling_gptj.py @@ -73,6 +73,14 @@ def __init__(self, config: GPTJConfig, layer_idx=None): super().__init__(config) self.config = config + max_positions = config.max_position_embeddings + self.register_buffer( + "bias", + torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view( + 1, 1, max_positions, max_positions + ), + persistent=False, + ) self.matmul_qk = Matmul() self.matmul_av = Matmul() self.k_cache = KVCache()