update llama2 configs

delphi-suite · Apr 24, 2024 · 669c91b · 669c91b
1 parent 2256391
commit 669c91b
Show file tree

Hide file tree

Showing 10 changed files with 20 additions and 20 deletions.
diff --git a/configs/stories/llama2/100k.json b/configs/stories/llama2/100k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 24,
-    "intermediate_size": 64,
+    "hidden_size": 12,
+    "intermediate_size": 48,
     "num_attention_heads": 2,
     "num_hidden_layers": 1,
     "num_key_value_heads": 1

diff --git a/configs/stories/llama2/10m.json b/configs/stories/llama2/10m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 362,
-    "intermediate_size": 965,
+    "hidden_size": 332,
+    "intermediate_size": 896,
     "num_attention_heads": 12,
     "num_hidden_layers": 6,
     "num_key_value_heads": 6

diff --git a/configs/stories/llama2/1m.json b/configs/stories/llama2/1m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 112,
-    "intermediate_size": 298,
+    "hidden_size": 84,
+    "intermediate_size": 256,
     "num_attention_heads": 8,
     "num_hidden_layers": 4,
     "num_key_value_heads": 4

diff --git a/configs/stories/llama2/2.5m.json b/configs/stories/llama2/2.5m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 196,
-    "intermediate_size": 522,
+    "hidden_size": 168,
+    "intermediate_size": 384,
     "num_attention_heads": 8,
     "num_hidden_layers": 4,
     "num_key_value_heads": 4

diff --git a/configs/stories/llama2/250k.json b/configs/stories/llama2/250k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 48,
-    "intermediate_size": 128,
+    "hidden_size": 28,
+    "intermediate_size": 96,
     "num_attention_heads": 4,
     "num_hidden_layers": 2,
     "num_key_value_heads": 2

diff --git a/configs/stories/llama2/25m.json b/configs/stories/llama2/25m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 512,
-    "intermediate_size": 1365,
+    "hidden_size": 484,
+    "intermediate_size": 1332,
     "num_attention_heads": 16,
     "num_hidden_layers": 8,
     "num_key_value_heads": 8

diff --git a/configs/stories/llama2/500k.json b/configs/stories/llama2/500k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 84,
-    "intermediate_size": 224,
+    "hidden_size": 52,
+    "intermediate_size": 184,
     "num_attention_heads": 4,
     "num_hidden_layers": 2,
     "num_key_value_heads": 2

diff --git a/configs/stories/llama2/50k.json b/configs/stories/llama2/50k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 12,
-    "intermediate_size": 32,
+    "hidden_size": 6,
+    "intermediate_size": 24,
     "num_attention_heads": 2,
     "num_hidden_layers": 1,
     "num_key_value_heads": 1

diff --git a/configs/stories/llama2/50m.json b/configs/stories/llama2/50m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 726,
-    "intermediate_size": 1952,
+    "hidden_size": 708,
+    "intermediate_size": 1896,
     "num_attention_heads": 16,
     "num_hidden_layers": 8,
     "num_key_value_heads": 8

diff --git a/configs/stories/llama2/5m.json b/configs/stories/llama2/5m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 246,
-    "intermediate_size": 656,
+    "hidden_size": 232,
+    "intermediate_size": 512,
     "num_attention_heads": 12,
     "num_hidden_layers": 6,
     "num_key_value_heads": 6