llama 2 hf convert yaml

stanford-crfm · Nov 18, 2024 · 0c4598d · 0c4598d
1 parent 558e5d6
commit 0c4598d
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/llama2_hf_convert.yaml b/llama2_hf_convert.yaml
@@ -0,0 +1,10 @@
+model:  # 7B class model
+  type: llama
+  seq_len: 2048
+  hidden_dim: 4096
+  intermediate_dim: 11008
+  num_layers: 32
+  num_heads: 32
+  num_kv_heads: 32
+  use_flash_attention: True
+  flash_attention_block_size: 1024