init

IBM · Feb 11, 2025 · ffbead5 · ffbead5
1 parent 5d9c551
commit ffbead5
Showing 1 changed file with 0 additions and 6 deletions.
diff --git a/dolomite_engine/hf_models/models/ladder_residual_TP/layer.py b/dolomite_engine/hf_models/models/ladder_residual_TP/layer.py
@@ -58,17 +58,11 @@ def forward(
             max_seqlen=max_seqlen,
         )
 
-        if self.m_residual is not None:
-            current_attention_out = current_attention_out * self.m_residual
-
         if current_mlp_out is not None:
             residual = residual + current_mlp_out
 
         # current_mlp_out = rmsnorm_cute_wrapper(residual, self.ln_2.weight, self.ln_2.eps, self.sequence_parallel)
         current_mlp_out = rmsnorm_cute_forward(residual, ln_2_weight, self.ln_2.eps, self.sequence_parallel)
         current_mlp_out = self.mlp_block(current_mlp_out)
 
-        if self.m_residual is not None:
-            current_mlp_out = current_mlp_out * self.m_residual
-
         return current_attention_out, current_mlp_out, residual