From 03aa29d1c04b2004ce285deb04f9650e30db49cc Mon Sep 17 00:00:00 2001
From: Ahmed Ahmed <ahmedah@stanford.edu>
Date: Sun, 17 Nov 2024 20:07:37 -0800
Subject: [PATCH] llama 2

---
 config/llama2_hf.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 config/llama2_hf.yaml

diff --git a/config/llama2_hf.yaml b/config/llama2_hf.yaml
new file mode 100644
index 000000000..a5742486c
--- /dev/null
+++ b/config/llama2_hf.yaml
@@ -0,0 +1,13 @@
+# Model configuration
+model:
+  type: llama
+  seq_len: 2048
+  hidden_dim: 4096
+  intermediate_dim: 11008
+  num_layers: 32
+  num_heads: 32
+  num_kv_heads: 32
+  use_flash_attention: true
+  flash_attention_block_size: 512
+  use_bias: false
+  use_layer_norm_weight: false