🔧 update settings to use CONTEXT_LENGTH instead

MAX_TOKENS
shroominic · Dec 3, 2023 · 793d2d8 · 793d2d8
1 parent dd19fb2
commit 793d2d8
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/src/funcchain/chain/creation.py b/src/funcchain/chain/creation.py
@@ -229,8 +229,8 @@ def _crop_large_inputs(
             from funcchain import settings
 
             content_tokens = count_tokens(v)
-            if base_tokens + content_tokens > settings.MAX_TOKENS:
-                input_kwargs[k] = v[: (settings.MAX_TOKENS - base_tokens) * 2 // 3]
+            if base_tokens + content_tokens > settings.CONTEXT_LENGTH:
+                input_kwargs[k] = v[: (settings.CONTEXT_LENGTH - base_tokens) * 2 // 3]
                 print("Truncated: ", len(input_kwargs[k]))
 
 

diff --git a/src/funcchain/settings.py b/src/funcchain/settings.py
@@ -25,7 +25,8 @@ class FuncchainSettings(BaseSettings):
     RETRY_PARSE_SLEEP: float = 0.1
 
     # Prompt
-    MAX_TOKENS: int = 4096
+    CONTEXT_LENGTH: int = 8196
+    MAX_TOKENS: int = 2048
     DEFAULT_SYSTEM_PROMPT: str = ""
 
     # KEYS