diff --git a/paperqa/litqa.py b/paperqa/litqa.py index 485b529d6..e9127590f 100644 --- a/paperqa/litqa.py +++ b/paperqa/litqa.py @@ -16,7 +16,7 @@ from paperqa.llms import LiteLLMModel, LLMModel from paperqa.prompts import EVAL_PROMPT_TEMPLATE, QA_PROMPT_TEMPLATE -from paperqa.settings import make_default_litellm_router_settings +from paperqa.settings import make_default_litellm_model_list_settings from paperqa.types import Answer if TYPE_CHECKING: @@ -139,7 +139,8 @@ def from_question( if isinstance(eval_model, str): eval_model = LiteLLMModel( - name=eval_model, config=make_default_litellm_router_settings(eval_model) + name=eval_model, + config=make_default_litellm_model_list_settings(eval_model), ) async def llm_from_answer(answer: Answer | str) -> LitQAEvaluation: diff --git a/paperqa/llms.py b/paperqa/llms.py index a0677eec9..cd07470e7 100644 --- a/paperqa/llms.py +++ b/paperqa/llms.py @@ -16,7 +16,7 @@ from enum import StrEnum from inspect import isasyncgenfunction, signature from sys import version_info -from typing import Any, TypeVar +from typing import Any, TypeVar, cast import litellm import numpy as np @@ -82,7 +82,10 @@ class EmbeddingModel(ABC, BaseModel): name: str config: dict[str, Any] = Field( default_factory=dict, - description="Optional `rate_limit` key, value must be a RateLimitItem or RateLimitItem string for parsing", + description=( + "Optional `rate_limit` key, value must be a RateLimitItem or RateLimitItem" + " string for parsing" + ), ) async def check_rate_limit(self, token_count: float, **kwargs) -> None: @@ -107,7 +110,12 @@ class LiteLLMEmbeddingModel(EmbeddingModel): name: str = Field(default="text-embedding-3-small") config: dict[str, Any] = Field( default_factory=dict, # See below field_validator for injection of kwargs - description="Optional `rate_limit` key, value must be a RateLimitItem or RateLimitItem string for parsing", + description=( + "The optional `rate_limit` key's value must be a RateLimitItem or" + " RateLimitItem string for parsing. The optional `kwargs` key is keyword" + " arguments to pass to the litellm.aembedding function. Note that LiteLLM's" + " Router is not used here." + ), ) @field_validator("config") @@ -510,21 +518,38 @@ def get_litellm_retrying_config(timeout: float = 60.0) -> dict[str, Any]: return {"num_retries": 3, "timeout": timeout} -class LiteLLMModel(LLMModel): - """A wrapper around the litellm library. +class PassThroughRouter(litellm.Router): + """Router that is just a wrapper on LiteLLM's normal free functions.""" - `config` should have two high level keys: - `model_list`: stores a list of all model configurations - (see https://docs.litellm.ai/docs/routing) - `router_kwargs`: kwargs for the Router class - `rate_limit`: (Optional) dictionary keyed by model group name - with values of type limits.RateLimitItem (in tokens / minute) - or valid limits.RateLimitItem string for parsing + def __init__(self, **kwargs): + self._default_kwargs = kwargs - This way users can specify routing strategies, retries, etc. - """ + async def atext_completion(self, *args, **kwargs): + return await litellm.atext_completion(*args, **(self._default_kwargs | kwargs)) - config: dict = Field(default_factory=dict) + async def acompletion(self, *args, **kwargs): + return await litellm.acompletion(*args, **(self._default_kwargs | kwargs)) + + +class LiteLLMModel(LLMModel): + """A wrapper around the litellm library.""" + + config: dict = Field( + default_factory=dict, + description=( + "Configuration of this model containing several important keys. The" + " optional `model_list` key stores a list of all model configurations" + " (SEE: https://docs.litellm.ai/docs/routing). The optional" + " `router_kwargs` key is keyword arguments to pass to the Router class." + " Inclusion of a key `pass_through_router` with a truthy value will lead" + " to using not using LiteLLM's Router, instead just LiteLLM's free" + f" functions (see {PassThroughRouter.__name__}). Rate limiting applies" + " regardless of `pass_through_router` being present. The optional" + " `rate_limit` key is a dictionary keyed by model group name with values" + " of type limits.RateLimitItem (in tokens / minute) or valid" + " limits.RateLimitItem string for parsing." + ), + ) name: str = "gpt-4o-mini" _router: litellm.Router | None = None @@ -548,9 +573,12 @@ def maybe_set_config_attribute(cls, data: dict[str, Any]) -> dict[str, Any]: } | data.get("config", {}) if "router_kwargs" not in data.get("config", {}): - data["config"]["router_kwargs"] = get_litellm_retrying_config() | { - "retry_after": 5 - } + if data.get("config", {}).get("pass_through_router"): + data["config"]["router_kwargs"] = get_litellm_retrying_config() + else: + data["config"]["router_kwargs"] = get_litellm_retrying_config() | { + "retry_after": 5 + } # we only support one "model name" for now, here we validate model_list = data["config"]["model_list"] @@ -562,7 +590,7 @@ def maybe_set_config_attribute(cls, data: dict[str, Any]) -> dict[str, Any]: # pylint: disable-next=possibly-used-before-assignment _DeploymentTypedDictValidator.validate_python(model_list) if "config" in data and len({m["model_name"] for m in model_list}) > 1: - raise ValueError("Only one model name per router is supported for now.") + raise ValueError("Only one model name per model list is supported for now.") return data def __getstate__(self): @@ -573,12 +601,15 @@ def __getstate__(self): return state @property - def router(self): + def router(self) -> litellm.Router: if self._router is None: - self._router = litellm.Router( - model_list=self.config["model_list"], - **self.config.get("router_kwargs", {}), - ) + router_kwargs: dict = self.config.get("router_kwargs", {}) + if self.config.get("pass_through_router"): + self._router = PassThroughRouter(**router_kwargs) + else: + self._router = litellm.Router( + model_list=self.config["model_list"], **router_kwargs + ) return self._router async def check_rate_limit(self, token_count: float, **kwargs) -> None: @@ -622,11 +653,11 @@ async def acomplete_iter( # type: ignore[override] async def achat( # type: ignore[override] self, messages: Iterable[dict[str, str]] ) -> Chunk: - response = await self.router.acompletion(self.name, messages) + response = await self.router.acompletion(self.name, list(messages)) return Chunk( - text=response.choices[0].message.content, - prompt_tokens=response.usage.prompt_tokens, - completion_tokens=response.usage.completion_tokens, + text=cast(litellm.Choices, response.choices[0]).message.content, + prompt_tokens=response.usage.prompt_tokens, # type: ignore[attr-defined] + completion_tokens=response.usage.completion_tokens, # type: ignore[attr-defined] ) @rate_limited @@ -634,7 +665,10 @@ async def achat_iter( # type: ignore[override] self, messages: Iterable[dict[str, str]] ) -> AsyncIterable[Chunk]: completion = await self.router.acompletion( - self.name, messages, stream=True, stream_options={"include_usage": True} + self.name, + list(messages), + stream=True, + stream_options={"include_usage": True}, ) async for chunk in completion: yield Chunk( @@ -797,5 +831,5 @@ def embedding_model_factory(embedding: str, **kwargs) -> EmbeddingModel: if embedding == "sparse": return SparseEmbeddingModel(**kwargs) if kwargs: # Only override the default config if there are actually kwargs - kwargs = {"config": kwargs} + return LiteLLMEmbeddingModel(name=embedding, config=kwargs) return LiteLLMEmbeddingModel(name=embedding, **kwargs) diff --git a/paperqa/settings.py b/paperqa/settings.py index 78ac9fe2e..f41adbc57 100644 --- a/paperqa/settings.py +++ b/paperqa/settings.py @@ -466,7 +466,9 @@ def _deprecated_field(self) -> Self: return self -def make_default_litellm_router_settings(llm: str, temperature: float = 0.0) -> dict: +def make_default_litellm_model_list_settings( + llm: str, temperature: float = 0.0 +) -> dict: """Settings matching "model_list" schema here: https://docs.litellm.ai/docs/routing.""" return { "model_list": [ @@ -674,21 +676,23 @@ def get_llm(self) -> LiteLLMModel: return LiteLLMModel( name=self.llm, config=self.llm_config - or make_default_litellm_router_settings(self.llm, self.temperature), + or make_default_litellm_model_list_settings(self.llm, self.temperature), ) def get_summary_llm(self) -> LiteLLMModel: return LiteLLMModel( name=self.summary_llm, config=self.summary_llm_config - or make_default_litellm_router_settings(self.summary_llm, self.temperature), + or make_default_litellm_model_list_settings( + self.summary_llm, self.temperature + ), ) def get_agent_llm(self) -> LiteLLMModel: return LiteLLMModel( name=self.agent.agent_llm, config=self.agent.agent_llm_config - or make_default_litellm_router_settings( + or make_default_litellm_model_list_settings( self.agent.agent_llm, self.temperature ), ) diff --git a/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[with-router].yaml b/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[with-router].yaml new file mode 100644 index 000000000..047e77230 --- /dev/null +++ b/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[with-router].yaml @@ -0,0 +1,103 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "Please tell me a story"}], "model": + "gpt-4o-mini", "max_tokens": 3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "110" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.46.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.46.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2yQzW6DMBCE7zyFtWeogPxV3NKoyrFKr1WFjFnAjbEt2yhporx7ZSAhVXLxYWbn + 8+yeA0KAl5ARYA11rNUiWm8PO24+F8edtVu6jk+b+Vu1PxTvxeakIfQJVfwgc9fUC1OtFui4koPN + DFKHnpqs0tdlnKTJsjdaVaLwsVq7aK6ilksepXE6j+JVlLyO6UZxhhYy8hUQQsi5f31PWeIRMhKH + V6VFa2mNkN2GCAGjhFeAWsuto9JBOJlMSYeyr/4hGZJOK0no/YTBqrPUt5SdEKN+uX0pVK2NKuzo + 3/SKS26b3CC1Snq8QFm7Bnr/EhDy3S/X/esL2qhWu9ypPUqPTNIBCNNJJ3M2ek45Ku4yi/AJLC/R + US7s3W2AUdZgOSXj4G63xy+fIYb9uKwfKMFIAvtrHbZ5xWWNRhs+XLvSOaZFiYsZLhGCS/AHAAD/ + /wMAUK1Q0HsCAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d0a39c48953f987-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 10 Oct 2024 23:00:17 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=.zPDYpJNWP9Qlhwa0l15L4150V_FFED3SUgP7YpB0Yw-1728601217-1.0.1.1-OgjF0rGnJj.tvPKAB74wcSxUg.L0fincQRx1CXXeYzDIBXj1HkyoQwp5scEGswx.wpY2_Kle7KXuUtgjYgVGng; + path=/; expires=Thu, 10-Oct-24 23:30:17 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=NKsjVntX6B0L.BW5e_HHH6wTxhiEfXXCNGNjWC3LbKY-1728601217051-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "128" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999989" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d3ac279f1203bdccf4ef91f84012a332 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestLiteLLMModel.test_max_token_truncation.yaml b/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[without-router].yaml similarity index 68% rename from tests/cassettes/TestLiteLLMModel.test_max_token_truncation.yaml rename to tests/cassettes/TestLiteLLMModel.test_max_token_truncation[without-router].yaml index caf455c3d..42e0bf960 100644 --- a/tests/cassettes/TestLiteLLMModel.test_max_token_truncation.yaml +++ b/tests/cassettes/TestLiteLLMModel.test_max_token_truncation[without-router].yaml @@ -33,24 +33,24 @@ interactions: x-stainless-runtime: - CPython x-stainless-runtime-version: - - 3.12.5 + - 3.12.7 method: POST uri: https://api.openai.com/v1/chat/completions response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//bJBRT4MwFIXf+RVNn8EwxoTwNhezaDR7Meo0hnTlAp2lbWhZpsv+uymw - gdle+nDOPV/PvQcHIcwynCBMS2Jopbg3X5rpS7SMXtcPu+374v7x7mm3mL/9rp/3Hyvs2oTcbIGa - U+qGykpxMEyKzqY1EAOWOomCeBbHQThtjUpmwG2sUMYLpVcxwbzAD0LPj7xJ3KdLyShonKBPByGE - Du1re4oM9jhBvntSKtCaFICT8xBCuJbcKphozbQhwmB3MKkUBkRbfSUooEZJgch4ooa80cS2FA3n - vX48f8lloWq50b1/1nMmmC7TGoiWwuI5iMKUuPWPDkJf7XLNv75Y1bJSJjXyG4RFToIOiIeTDua0 - 94w0hI8yM/cKLM3AEMb16DaYElpCNiR9Z7Tb5ZfXEN1+TBQXFKcnYf2jDVRpzkQBtapZd+1cpXk8 - 2wC5jeIQO0fnDwAA//8DAHuAwiJ7AgAA + H4sIAAAAAAAAAwAAAP//bJDNboMwEITvPIW1Z6iA/FXc2h5yrKKo6qGqkGMWcGJs1zZNqyjvXhlI + oEouPszsfJ7dU0AI8AIyAqymjjVaRE/r42a/MN9f8UFt38r3/fq5LDf0pTpsjzWEPqF2e2Tuknpg + qtECHVeyt5lB6tBTk1X6uIyTNFl1RqMKFD5WaRfNVdRwyaM0TudRvIqSxyFdK87QQkY+AkIIOXWv + 7ykL/IGMxOFFadBaWiFk1yFCwCjhFaDWcuuodBCOJlPSoeyqv0qGpNVKEjqdMFi2lvqWshVi0M/X + L4WqtFE7O/hXveSS2zo3SK2SHi9QVq6Gzj8HhHx2y7X/+oI2qtEud+qA0iOTtAfCeNLRnA2eU46K + SWYR3oHlBTrKhZ3cBhhlNRZjMg4mu91+eQ/R78dldUMJBhLYX+uwyUsuKzTa8P7apc4x3RW4mOES + ITgHfwAAAP//AwAe32pYewIAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d08fd0aaeaf67c4-SJC + - 8d0a39c719bbcf17-SJC Connection: - keep-alive Content-Encoding: @@ -58,14 +58,14 @@ interactions: Content-Type: - application/json Date: - - Thu, 10 Oct 2024 19:24:04 GMT + - Thu, 10 Oct 2024 23:00:17 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=BTdrirW_Guxpv.H.rPyazpVjfuD5kh2lWDDnQth1z4M-1728588244-1.0.1.1-W6yhhzCqXUjie6dbonmojEUTSHpKkrTd8f..HSkJUNVqCXeZNhD10LbAcC7HGtFW1gwFcDY36SlknE_MHO7ajQ; - path=/; expires=Thu, 10-Oct-24 19:54:04 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=LJeuVcZ7wwyb_ngFye2WH2ctmt4WlOoQ1WrdalQaqlk-1728601217-1.0.1.1-LXEA5NWU4Ph_F354G1wnqppvr8Ht0PASE2WJsQb9_JWtQG31oelTy6FaUKelPH73ULj8OT.IspeFntBRex.GWw; + path=/; expires=Thu, 10-Oct-24 23:30:17 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=LWX01ieVS4uvshESWXrymr6RrGTZy6Gj4GDJzGEDMG4-1728588244222-0.0.1.1-604800000; + - _cfuvid=HJ_TEmTsc0ArmcmSzLMwMcKvGqCuZknBnHIvIz7e32g-1728601217391-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -78,7 +78,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "401" + - "132" openai-version: - "2020-10-01" strict-transport-security: @@ -96,7 +96,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_d8aeedcb649baad21ab2c3be58fe48cf + - req_4d95169fedd215ff57ae897bb111314a status: code: 200 message: OK diff --git a/tests/cassettes/TestLiteLLMModel.test_run_prompt[with-router].yaml b/tests/cassettes/TestLiteLLMModel.test_run_prompt[with-router].yaml new file mode 100644 index 000000000..971d76d80 --- /dev/null +++ b/tests/cassettes/TestLiteLLMModel.test_run_prompt[with-router].yaml @@ -0,0 +1,527 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "The duck says"}], "model": "gpt-4o-mini", + "max_tokens": 56, "stream": true, "stream_options": {"include_usage": true}, + "temperature": 0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "179" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.46.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.46.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + duck"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + says"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + \""},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"qu"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"ack"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"!\""},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + It''s"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + classic"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + sound"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + associated"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + with"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + ducks"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + Is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + there"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + something"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + specific"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + you''d"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + like"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + to"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + know"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + or"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + discuss"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + about"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + ducks"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-AGx3IFOe2Nkgb3uc5s0eANKM2UQeR","object":"chat.completion.chunk","created":1728603608,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":28,"total_tokens":38,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"text_tokens":0,"reasoning_tokens":0,"audio_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d0a74268bec15ca-SJC + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 10 Oct 2024 23:40:08 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=2ii3axrdHPR_97FB7wT02DNTiSulGSvJmoCrS0Z19sU-1728603608-1.0.1.1-0CTIHcB25tnYHmwaW9JL9Z4tpOmTekU2xRoC1mCs._YeW6fCgNH.hROUHlBFBl.SAmGpxqpSMvPjy0GHBrxF.w; + path=/; expires=Fri, 11-Oct-24 00:10:08 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=xy0WAi9kOooj1SWTQu38LOxFYS_W5czVO8NKi1bYZKc-1728603608407-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "120" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999938" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3195d8534fad2b5e96ae8fb91b503799 + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": "The duck says"}], "model": "gpt-4o-mini", + "max_tokens": 56, "stream": false, "temperature": 0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "137" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.46.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.46.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//bFKxbttADN31FewtWaxAluNY9VK0S2C0QxOkRYqmMM4nSrrodFRFCrEb + +N8LybLlIFlueO/eIx/JlwBA2VQtQZlCi6lqF36+2c5WfHtX3t48/fzy8G/78dfi7sfX6PvDt/tK + TToFbZ7QyFF1aaiqHYolf6BNg1qwc50u4uQ6ml1HSU9UlKLrZHkt4RWFlfU2jKP4KowW4TQZ1AVZ + g6yW8DsAAHjp365Pn+JWLSGaHJEKmXWOann6BKAach2iNLNl0V7UZCQNeUHft35fIKStKYH1juFR + /W21KT88KljJBYMG4zoHA0ytT0Ezk7FdLHi2UvRKvoQVgxTYIDBVKIX1OXCNxmbWwI7aixScLRGE + oPT0DNRAatm0zKA31MrB5tN5hw1mLetuSr51bsD3p8iO8rqhDQ/8Cc+st1ysG9RMvovHQrXq2X0A + 8KcfbftqWqpuqKplLVSi7wynw2TVuNCRjJOBFBLtRnx2xF+5rVMUbR2frUYZbQpMR2UUnEV7W/M9 + C8HtWb/jIRxiW5+/R+o2tfSmbDCUVrxjwWqdWZ9jUzf2cB1ZvU7m8xjNfIZTFeyD/wAAAP//AwC3 + fipBKwMAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d0a742b2ecdf993-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 10 Oct 2024 23:40:09 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vA8GaQd4QdHB5TgVHvP8e7sIF6FIHlZbpxLjMjCqMjY-1728603609-1.0.1.1-3MWKf6WzZesR1kCewz6vTtJ3LO7NJbLSEeHTNqzR6GD9dIvaaNXwO7pYAkcnEosoDqN18rVnkmz9gqc_wMG9IA; + path=/; expires=Fri, 11-Oct-24 00:10:09 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=lWnn4On0tpGPJubUJg3g3_ixEgz.2YEyOpa6NpdThmM-1728603609679-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "662" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999938" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_583766eac4d75597e0fb5f85fe281a3a + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": "The duck says"}], "model": "gpt-4o-mini", + "max_tokens": 56, "stream": true, "stream_options": {"include_usage": true}, + "temperature": 0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "179" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.46.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.46.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: + 'data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + duck"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + says"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + \""},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":"qu"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":"ack"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":"!\""},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + It''s"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + classic"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + sound"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + associated"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + with"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + ducks"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + Is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + there"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + something"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + specific"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + you''d"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + like"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + to"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + know"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + about"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + ducks"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + or"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + their"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":" + sounds"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-AGx3J2UOjoyB2R4AvrnJgXi4CIGEf","object":"chat.completion.chunk","created":1728603609,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_8552ec53e1","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":29,"total_tokens":39,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"text_tokens":0,"reasoning_tokens":0,"audio_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8d0a74314da315ca-SJC + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Thu, 10 Oct 2024 23:40:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "162" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999938" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_027c1bff2e728df8203ed1df33097fcf + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestLiteLLMModel.test_run_prompt.yaml b/tests/cassettes/TestLiteLLMModel.test_run_prompt[without-router].yaml similarity index 57% rename from tests/cassettes/TestLiteLLMModel.test_run_prompt.yaml rename to tests/cassettes/TestLiteLLMModel.test_run_prompt[without-router].yaml index fdf33d912..9f008f6e7 100644 --- a/tests/cassettes/TestLiteLLMModel.test_run_prompt.yaml +++ b/tests/cassettes/TestLiteLLMModel.test_run_prompt[without-router].yaml @@ -34,94 +34,129 @@ interactions: x-stainless-runtime: - CPython x-stainless-runtime-version: - - 3.12.5 + - 3.12.7 method: POST uri: https://api.openai.com/v1/chat/completions response: body: string: - 'data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + 'data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" duck"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" says"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" \""},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"qu"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"qu"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"ack"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"ack"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"!\""},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"!\""},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" - What"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + It''s"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" - else"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" - would"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + classic"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" - you"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + sound"},"logprobs":null,"finish_reason":null}],"usage":null} + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + associated"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + with"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + ducks"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + Is"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + there"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + something"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + specific"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + you''d"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" like"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" know"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" about"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" ducks"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" or"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" their"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" sounds"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - data: {"id":"chatcmpl-AGs6afMvu8WUP8jGhqNmt8Pi0JKZ0","object":"chat.completion.chunk","created":1728584592,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":20,"total_tokens":30,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} + data: {"id":"chatcmpl-AGx3Ki9UELmsaOxOqoUcnJH5EjcG6","object":"chat.completion.chunk","created":1728603610,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":29,"total_tokens":39,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"text_tokens":0,"reasoning_tokens":0,"audio_tokens":0}}} data: [DONE] @@ -132,20 +167,20 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d08a3e428962544-SJC + - 8d0a7435dfedcf7a-SJC Connection: - keep-alive Content-Type: - text/event-stream; charset=utf-8 Date: - - Thu, 10 Oct 2024 18:23:12 GMT + - Thu, 10 Oct 2024 23:40:10 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=cBVoK0Axh6_.07rmaY4Zqwwq3ln3IQXpc3Rm53oSBVg-1728584592-1.0.1.1-Rl_cG2qPBGQFy_F5fhkmebc7HkIoqnNHDPDVNmWz.zQXc._ONagIvlg9T9SFdfl7ntWzA7dyQEhYVrlkR_yypQ; - path=/; expires=Thu, 10-Oct-24 18:53:12 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=7P56L2VMG1PCThImPfIUAaPNVzpqxmbmrtCtWEiLFdQ-1728603610-1.0.1.1-sy9Xzhak3B1RHPPhKcDai7IIdXhvz2XR64z_o4pLNnMsp8k3T3cJ1mqZ0bd5bX7ImnJh5sInc_1zG4.u8yaKzA; + path=/; expires=Fri, 11-Oct-24 00:10:10 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=Y5f3ErAod8E4mgiJ3EsCj4IUkfxT7z97cOARuxWbjKs-1728584592456-0.0.1.1-604800000; + - _cfuvid=UgWrZ5wECsYCUbBp6mrgpj_irTJG.6GksEClR_J.ewg-1728603610859-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -158,7 +193,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "227" + - "153" openai-version: - "2020-10-01" strict-transport-security: @@ -176,14 +211,14 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_bf90a543fda52141d8c5073a06df6dfe + - req_9b7e3d27cffee3c259fc4ea6223a3b43 status: code: 200 message: OK - request: body: '{"messages": [{"role": "user", "content": "The duck says"}], "model": "gpt-4o-mini", - "max_tokens": 56, "stream": false, "temperature": 0}' + "max_tokens": 56, "temperature": 0}' headers: accept: - application/json @@ -192,7 +227,7 @@ interactions: connection: - keep-alive content-length: - - "137" + - "120" content-type: - application/json host: @@ -214,25 +249,25 @@ interactions: x-stainless-runtime: - CPython x-stainless-runtime-version: - - 3.12.5 + - 3.12.7 method: POST uri: https://api.openai.com/v1/chat/completions response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//bFHLbtswELzrK7a85GIFtmvHji9BgLZGTgWatoc2hUFTK4kxxWW5q7pG - 4H8vKMuPIL3wMMOZ3Zl9yQCULdQClKm1mCa4/H7JN3oUaOu/DZePHx8/f4g/Au++f3n+s/ykBklB - 62c0clRdG2qCQ7HkD7SJqAWT62g2nk/nk+ntuCMaKtAlWRUkn1DeWG/z8XA8yYezfDTv1TVZg6wW - 8DMDAHjp3rSnL/CvWsBwcEQaZNYVqsXpE4CK5BKiNLNl0V7U4Ewa8oK+W/1rjVC0ZgOsdwxP6ner - zebdk4IHuWLQYFxyMMDU+gI0MxmbYsHWSt0p+RoeGKTGiMDUoNTWV8ABjS2tgR21VwU4u0EQgo2n - Leg1tXLQAsUktfEwgO8u14xYtqxTVb51rsf3p9yOqhBpzT1/wkvrLderiJrJp4wsFFTH7jOAX12/ - 7avKVIjUBFkJbdAnw1Ffrzpf9UyOb3tSSLQ74++P+Cu3VYGireOL+yijTY3FWTnMLqK9nfk/i0M8 - 66s3LlnvpHjHgs2qtL7CGKI9XLwMq3I+XaO+mc0nKttn/wAAAP//AwAfLI/k/wIAAA== + H4sIAAAAAAAAAwAAAP//bFHBjpswEL3zFVOfQwUk2US5rNpLL9tbpbbqVsgxA7gYD7XHhWiVf68g + JGS1e/HhPb837828RABCF+IAQtWSVduZ+NOXYf1U/Nv9cD934Wtdmn7bn/ZDPzyZzWexGhV0/IOK + r6qPitrOIGuyF1o5lIyja7rL9g/J+iFNJ6KlAs0oqzqONxS32uo4S7JNnOzidD+ra9IKvTjArwgA + 4GV6x5y2wEEcIFldkRa9lxWKw+0TgHBkRkRI77VnaVmsFlKRZbRT9G81QhFUA16ePDyLv0Gq5sOz + gO+1ZEDjEXoKpoATBTC6QWCCxlIP8kiBJ60HcsA1ageegi384/0wh2XwcixsgzEzfr6lN1R1jo5+ + 5m94qa32de5QerJjUs/UiYk9RwC/py2FV8VF56jtOGdq0I6G6bwksdxmIbMrycTSLPj6ir9yywtk + qY2/27JQUtVYLMokuqv2duZ7FozDXd7lppfa2lbvkTIUmt6MjebRwp88Y5uX2lboOqcvhy67fL/d + Zqi2a0xFdI7+AwAA//8DAPVofqz2AgAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d08a3e8fe71643a-SJC + - 8d0a743a8c49cf7a-SJC Connection: - keep-alive Content-Encoding: @@ -240,15 +275,9 @@ interactions: Content-Type: - application/json Date: - - Thu, 10 Oct 2024 18:23:13 GMT + - Thu, 10 Oct 2024 23:40:12 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=Kk_q3TEse_n46UNb02eeJspbya_WdtrsoDHV9W3TjNg-1728584593-1.0.1.1-bNwNBUMD0LW7_VQfRFlleID9JujljwEK0sagG9QGrU1hQsCqq93TOMSRJMXHniQP8dOH7tCd2hKeU489J0o1_w; - path=/; expires=Thu, 10-Oct-24 18:53:13 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=k8qsGpey9Ugw4nvdemc3bVV2.B6TQHgaq5gMtsLOQIU-1728584593445-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked X-Content-Type-Options: @@ -260,7 +289,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "449" + - "609" openai-version: - "2020-10-01" strict-transport-security: @@ -278,7 +307,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_25bada8803a41392edc5b4ffb82012b3 + - req_8d598421c7bf0a81c4a60e61578d269b status: code: 200 message: OK @@ -317,129 +346,129 @@ interactions: x-stainless-runtime: - CPython x-stainless-runtime-version: - - 3.12.5 + - 3.12.7 method: POST uri: https://api.openai.com/v1/chat/completions response: body: string: - 'data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + 'data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" duck"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" says"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" \""},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"qu"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"qu"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"ack"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"ack"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"!\""},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"!\""},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" It''s"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" a"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" classic"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" sound"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" associated"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" with"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" ducks"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" Is"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" there"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" something"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" specific"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" you''d"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" like"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" to"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" know"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" about"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" ducks"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" or"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" their"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":" sounds"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - data: {"id":"chatcmpl-AGs6bQ2ciKLlTAA94H9M9xYcdOOGM","object":"chat.completion.chunk","created":1728584593,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":29,"total_tokens":39,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} + data: {"id":"chatcmpl-AGx3MvJQjvEnIrCZ8yN4yKAvatZWB","object":"chat.completion.chunk","created":1728603612,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_e2bde53e6e","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":29,"total_tokens":39,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"text_tokens":0,"reasoning_tokens":0,"audio_tokens":0}}} data: [DONE] @@ -450,13 +479,13 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d08a3ede8a72544-SJC + - 8d0a7441cb92cf7a-SJC Connection: - keep-alive Content-Type: - text/event-stream; charset=utf-8 Date: - - Thu, 10 Oct 2024 18:23:13 GMT + - Thu, 10 Oct 2024 23:40:12 GMT Server: - cloudflare Transfer-Encoding: @@ -470,7 +499,7 @@ interactions: openai-organization: - future-house-xr4tdh openai-processing-ms: - - "151" + - "156" openai-version: - "2020-10-01" strict-transport-security: @@ -488,7 +517,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_21ba2a5bf6eba6b7e264e6f0da1862f6 + - req_b50c3dd1a1c802137b27d550955c4756 status: code: 200 message: OK diff --git a/tests/test_llms.py b/tests/test_llms.py index 021409e0d..e699cc5ae 100644 --- a/tests/test_llms.py +++ b/tests/test_llms.py @@ -1,6 +1,9 @@ import pathlib import pickle +from typing import Any +from unittest.mock import patch +import litellm import pytest from paperqa import LiteLLMModel @@ -10,23 +13,36 @@ class TestLiteLLMModel: @pytest.mark.vcr(match_on=[*VCR_DEFAULT_MATCH_ON, "body"]) + @pytest.mark.parametrize( + "config", + [ + pytest.param( + { + "model_list": [ + { + "model_name": "gpt-4o-mini", + "litellm_params": { + "model": "gpt-4o-mini", + "temperature": 0, + "max_tokens": 56, + }, + } + ] + }, + id="with-router", + ), + pytest.param( + { + "pass_through_router": True, + "router_kwargs": {"temperature": 0, "max_tokens": 56}, + }, + id="without-router", + ), + ], + ) @pytest.mark.asyncio - async def test_run_prompt(self) -> None: - llm = LiteLLMModel( - name="gpt-4o-mini", - config={ - "model_list": [ - { - "model_name": "gpt-4o-mini", - "litellm_params": { - "model": "gpt-4o-mini", - "temperature": 0, - "max_tokens": 56, - }, - } - ] - }, - ) + async def test_run_prompt(self, config: dict[str, Any]) -> None: + llm = LiteLLMModel(name="gpt-4o-mini", config=config) outputs = [] @@ -68,20 +84,43 @@ async def ac(x) -> None: assert completion.cost > 0 @pytest.mark.vcr + @pytest.mark.parametrize( + ("config", "bypassed_router"), + [ + pytest.param( + { + "model_list": [ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "gpt-4o-mini", "max_tokens": 3}, + } + ] + }, + False, + id="with-router", + ), + pytest.param( + {"pass_through_router": True, "router_kwargs": {"max_tokens": 3}}, + True, + id="without-router", + ), + ], + ) @pytest.mark.asyncio - async def test_max_token_truncation(self) -> None: - llm = LiteLLMModel( - name="gpt-4o-mini", - config={ - "model_list": [ - { - "model_name": "gpt-4o-mini", - "litellm_params": {"model": "gpt-4o-mini", "max_tokens": 3}, - } - ] - }, - ) - chunk = await llm.acomplete("Please tell me a story") # type: ignore[call-arg] + async def test_max_token_truncation( + self, config: dict[str, Any], bypassed_router: bool + ) -> None: + llm = LiteLLMModel(name="gpt-4o-mini", config=config) + with patch( + "litellm.Router.atext_completion", + side_effect=litellm.Router.atext_completion, + autospec=True, + ) as mock_atext_completion: + chunk = await llm.acomplete("Please tell me a story") # type: ignore[call-arg] + if bypassed_router: + mock_atext_completion.assert_not_awaited() + else: + mock_atext_completion.assert_awaited_once() assert isinstance(chunk, Chunk) assert chunk.completion_tokens == 3 assert chunk.text