From ed00688d36def3a24b45f7f0b9394750e124aba4 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Fri, 3 Jan 2025 10:28:55 +0800 Subject: [PATCH 1/4] Update Dockerfile --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b22f47..4323fa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,10 @@ WORKDIR /app # Install system dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libc6-dev \ + build-essential \ + python3-dev \ + gcc \ + g++ \ && rm -rf /var/lib/apt/lists/* # Copy only the requirements file first to leverage Docker cache @@ -44,4 +47,4 @@ ENV PYTHONUNBUFFERED=1 EXPOSE 8000 # Run the application -ENTRYPOINT ["python", "optillm.py"] +CMD ["optillm"] From d540ac2aaf9d57d9620875a55074d4e0c0b9ea69 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Fri, 3 Jan 2025 11:36:02 +0800 Subject: [PATCH 2/4] Fix streaming response - yield response to allow clients that expect streaming --- optillm.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/optillm.py b/optillm.py index 689fb28..3d9deec 100644 --- a/optillm.py +++ b/optillm.py @@ -246,11 +246,10 @@ def execute_single_approach(approach, system_prompt, initial_query, client, mode if hasattr(request, 'json'): data = request.get_json() messages = data.get('messages', []) - # Copy all parameters except 'model' and 'messages' + # Copy all parameters except 'stream', 'model' , 'n' and 'messages' kwargs = {k: v for k, v in data.items() - if k not in ['model', 'messages', 'optillm_approach']} + if k not in ['model', 'messages', 'stream', 'n', 'optillm_approach']} response = none_approach(original_messages=messages, client=client, model=model, **kwargs) - # For none approach, we return the response and a token count of 0 # since the full token count is already in the response return response, 0 @@ -369,6 +368,22 @@ def generate_streaming_response(final_response, model): # Yield the final message to indicate the stream has ended yield "data: [DONE]\n\n" +def extract_contents(response_obj): + contents = [] + # Handle both single response and list of responses + responses = response_obj if isinstance(response_obj, list) else [response_obj] + + for response in responses: + # Extract content from first choice if it exists + if (response.get('choices') and + len(response['choices']) > 0 and + response['choices'][0].get('message') and + response['choices'][0]['message'].get('content')): + contents.append(response['choices'][0]['message']['content']) + + # Return single string if only one content, otherwise return list + return contents[0] if len(contents) == 1 else contents + def parse_conversation(messages): system_prompt = "" conversation = [] @@ -523,8 +538,13 @@ def proxy(): result = responses else: result, completion_tokens = execute_single_approach(approaches[0], system_prompt, initial_query, client, model) + logger.debug(f'Direct proxy response: {result}') - return jsonify(result), 200 + + if stream: + return Response(generate_streaming_response(extract_contents(result), model), content_type='text/event-stream') + else : + return jsonify(result), 200 elif operation == 'AND' or operation == 'OR': if contains_none: @@ -545,7 +565,7 @@ def proxy(): messages = tagged_conversation_to_messages(response) if messages: # Only take the last message if we have any response = messages[-1]['content'] - + if stream: return Response(generate_streaming_response(response, model), content_type='text/event-stream') else: From ad11b754dc10cafe3aad673ed3d0306a0e01cae2 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Fri, 3 Jan 2025 11:38:10 +0800 Subject: [PATCH 3/4] return a list --- optillm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/optillm.py b/optillm.py index 3d9deec..005e866 100644 --- a/optillm.py +++ b/optillm.py @@ -381,8 +381,7 @@ def extract_contents(response_obj): response['choices'][0]['message'].get('content')): contents.append(response['choices'][0]['message']['content']) - # Return single string if only one content, otherwise return list - return contents[0] if len(contents) == 1 else contents + return contents def parse_conversation(messages): system_prompt = "" From 60527e27732f2d0c852406c2926a33a75cfc16f9 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Fri, 3 Jan 2025 11:39:37 +0800 Subject: [PATCH 4/4] prep for new release --- optillm/plugins/readurls_plugin.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/optillm/plugins/readurls_plugin.py b/optillm/plugins/readurls_plugin.py index 8d6e9d1..0c65d92 100644 --- a/optillm/plugins/readurls_plugin.py +++ b/optillm/plugins/readurls_plugin.py @@ -25,7 +25,7 @@ def extract_urls(text: str) -> List[str]: def fetch_webpage_content(url: str, max_length: int = 100000) -> str: try: headers = { - 'User-Agent': 'optillm/0.0.19 (https://github.com/codelion/optillm)' + 'User-Agent': 'optillm/0.0.20 (https://github.com/codelion/optillm)' } response = requests.get(url, headers=headers, timeout=10) diff --git a/setup.py b/setup.py index c2693f6..ce51d3a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="optillm", - version="0.0.19", + version="0.0.20", packages=find_packages(), py_modules=['optillm'], package_data={