Skip to content

Commit

Permalink
Merge pull request #47 from pvs-hd-tea/46-feature-use-second-gpt-as-a…
Browse files Browse the repository at this point in the history 
…-code-review-for-analysis

Add GPTReviewAnalyzer
  • Loading branch information
@psaegert
psaegert authored Dec 6, 2023
2 parents fa5f667 + bd3c25d commit a04102c
Show file tree
Hide file tree
Showing 14 changed files with 1,022 additions and 19 deletions.
486 changes: 486 additions & 0 deletions experimental/gptreviewanalyzer_example.txt
View file

Large diffs are not rendered by default.

374 changes: 374 additions & 0 deletions experimental/test_gpt_review_analyzer.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,374 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from llmcoder.analyze import GPTReviewAnalyzer_v1"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"input = ''' model_first : str, optional\n",
" The model to use for the first completion, by default \"ft:gpt-3.5-turbo-1106:personal::8LCi9Q0d\"\n",
" model_feedback : str, optional\n",
" The model to use for the feedback loop, by default \"gpt-3.5-turbo\"\n",
" feedback_variant : str, optional\n",
" The feedback variant to use, by default \"separate\"\n",
" system_prompt : str, optional\n",
" The system prompt to use, by default the one used for preprocessing and fine-tuning\n",
" max_iter : int, optional\n",
" The maximum number of iterations to run the feedback loop, by default 10\n",
" log_conversation : bool, optional\n",
" Whether to log the conversation, by default False\n",
" \"\"\"\n",
" if analyzers is None:\n",
" self.analyzers = []\n",
" else:\n",
" self.analyzers = [AnalyzerFactory.create_analyzer(analyzer) for analyzer in analyzers]\n",
"\n",
" self.model_first = model_first\n",
" self.model_feedback = model_feedback\n",
"\n",
" self.client = openai.OpenAI(api_key=get_openai_key())\n",
" if feedback_variant not in [\"separate\", \"coworker\"]:\n",
" raise ValueError(\"Inavlid feedback method\")\n",
"\n",
" self.iterations = 0\n",
" self.analyzer_pass_history: list[list[dict]] = []\n",
" self.max_iter = max_iter\n",
" self.feedback_variant = feedback_variant\n",
"\n",
" if log_conversation:\n",
" self.conversation_file = self._create_conversation_file()\n",
" else:\n",
" self.conversation_file = None # type: ignore\n",
"\n",
" self.messages: list = []\n",
"\n",
" if system_prompt is None:\n",
" self.system_prompt = get_system_prompt()\n",
" elif system_prompt in os.listdir(get_system_prompt_dir()):\n",
" self.system_prompt = get_system_prompt(system_prompt)\n",
" else:\n",
" self.system_prompt = system_prompt\n",
"\n",
" self._add_message(\"system\", message=self.system_prompt)\n",
"\n",
" def complete(self, code: str) -> str:\n",
" \"\"\"\n",
" Complete the provided code with the LLMCoder feedback loop\n",
"\n",
" Parameters\n",
" ----------\n",
" code : str\n",
" The code to complete\n",
"\n",
" Returns\n",
" -------\n",
" str\n",
" The completed code\n",
" \"\"\"\n",
" # Get the first completion with\n",
" self.complete_first(code)\n",
"\n",
" if len(self.analyzers) > 0:\n",
" # Run the feedback loop until the code is correct or the max_iter is reached\n",
" for _ in range(self.max_iter):\n",
" if self.feedback_step():\n",
" # If the feedback is correct, break the loop and return the code\n",
" break\n",
"\n",
" # Return the last message\n",
" return self.messages[-1][\"content\"]\n",
"\n",
" @staticmethod\n",
" def _create_conversation_file() -> str:\n",
" \"\"\"\n",
" Create the conversation file\n",
"\n",
" Returns\n",
" -------\n",
" str\n",
" The path of the conversation file\n",
" \"\"\"\n",
" return os.path.join(get_conversations_dir(create=True), f\"{datetime.now()}.jsonl\")\n",
"\n",
" def _add_message(self, role: str, model: str = 'gpt-3.5-turbo', message: str | None = None) -> None:\n",
" \"\"\"\n",
" Add a message to the messages list\n",
"\n",
" Parameters\n",
" ----------\n",
" role : str\n",
" The role of the message\n",
" model : str, optional\n",
" The model to use for the completion, by default 'gpt-3.5-turbo'\n",
" message : str, optional\n",
" The message to add, by default None\n",
" \"\"\"\n",
" # If the user is the assistant, generate a response\n",
" if role == \"assistant\" and message is None:\n",
" chat_completion = self.client.chat.completions.create(messages=self.messages, model=model) # type: ignore\n",
"\n",
" message = chat_completion.choices[0].message.content\n",
"\n",
" self.messages.append({\n",
" \"role\": role,\n",
" \"content\": message,\n",
" })\n",
"\n",
" if self.conversation_file is not None:\n",
" # If the conversation file already exists, only append the last message as a single line\n",
" if os.path.isfile(self.conversation_file):\n",
" with open(self.conversation_file, \"a\") as f:\n",
" f.write(json.dumps(self.messages[-1], ensure_ascii=False) + \"\\n\")\n",
" # Otherwise, write the whole conversation\n",
" else:\n",
" '''"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"completion = '''open(self.conversation_file, \"w\") as f:\n",
" for message in self.messages:\n",
" f.write(json.dumps(message, ensure_ascii=False) + \"\\n\")'''"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# print(analyzer._prompt_template(input, completion))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"analyzer = GPTReviewAnalyzer_v1(system_prompt=\"2023-12-02_GPTReviewAnalyzer_v3.txt\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"True\n",
"\n",
"The added code in the [COMPLETE] section looks reasonable and addresses the missing part in the [INCOMPLETE] section. However, there are a few improvements that can be made:\n",
"\n",
"1. Syntax Validity: The line `open(self.conversation_file, \"w\") as f:` is missing the `with` statement. It should be written as `with open(self.conversation_file, \"w\") as f:` to properly open the file in a context manager.\n",
"\n",
"2. Logical Consistency: It seems that the intention is to write the entire conversation to the file if there is no previous content present. However, the current implementation only writes the last message in the conversation. To fix this, you can modify the loop to write all messages instead of just the last one.\n",
"\n",
"3. Performance Efficiency: Since the file is being opened for writing, it is better to use the mode \"w\" instead of \"a\" (append). This will ensure that the file is cleared before writing the new content. If the file contains a large conversation history, this can improve performance by avoiding unnecessary writes to the file.\n",
"\n",
"Here's the modified code with the suggested improvements:\n",
"\n",
"```python\n",
"# Write the last message if the file already exists\n",
"if os.path.exists(self.conversation_file):\n",
" with open(self.conversation_file, \"a\") as f:\n",
" f.write(json.dumps(self.messages[-1], ensure_ascii=False) + \"\\n\")\n",
"# Otherwise, write the whole conversation\n",
"else:\n",
" with open(self.conversation_file, \"w\") as f:\n",
" for message in self.messages:\n",
" f.write(json.dumps(message, ensure_ascii=False) + \"\\n\")\n",
"```\n",
"\n",
"SCORE: 7\n"
]
}
],
"source": [
"result = analyzer.analyze('...' + input[-200:], completion)\n",
"print(result['pass'])\n",
"print()\n",
"print(result['message'])"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"from llmcoder import LLMCoder"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"llmcoder = LLMCoder(analyzers=[\"gpt_review_analyzer_v1\"], temperature=0.7)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"input = '''def _check_hashability(df):\n",
" \"\"\"This is a static method that checks if a given pandas DataFrame is hashable or not.\n",
"\n",
" It checks if all the columns containing object types in the input DataFrame are hashable or not.\n",
" If any column is not hashable, it raises a TypeError indicating which columns are not hashable.\n",
"\n",
" Parameters\n",
" ----------\n",
" df : pandas.DataFrame\n",
" The DataFrame to be checked for hashability.\n",
"\n",
" Raises\n",
" ------\n",
" TypeError\n",
" If any column containing object types in the input DataFrame is not hashable.\n",
" \"\"'''"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Creating first completion...\n",
"Starting feedback loop...\n",
"Starting feedback iteration 1...\n",
"Analyzing code...\n",
"Running GPTReviewAnalyzer_v1...\n",
"1 / 1 analyzers passed\n"
]
}
],
"source": [
"completion = llmcoder.complete(input)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"SYSTEM\n",
"You are AutocompleteGPT, a useful AI autocomplete tool that provides code completions based on the user's code.\n",
"You are a precision-focused tool for code autocompletion, adept in languages like Python, JavaScript, C++, and SQL.\n",
"Precisely continue the code from the point of interruption and do not repeat or modify the original code, even if it is incorrect or the code interrupts in the middle of a line.\n",
"Your code is well documented with comments and annotations, and you should provide a clear explanation of the code's purpose in your code completion.\n",
"Your unique capability is to provide completions without altering, repeating, or commenting on the original code.\n",
"You offer only the necessary code to complete the snippet, ensuring the response is exclusively code, with no additional comments, explanations, or annotations.\n",
"This approach makes you an ideal assistant for users seeking straightforward and efficient code extensions, enhancing their work with accurate, logic-driven completions while maintaining the integrity and simplicity of the original input.\n",
"Your response begins with the next characters of the line if the last line of the user'S code is incomplete, or the next line if the last line of the user's code is complete.\n",
"Your application is a VSCode extension like GitHub Copilot, which provides seamless code completions based on the user's code at the point of interruption.\n",
"--------------------------------------------------\n",
"USER\n",
"def _check_hashability(df):\n",
" \"\"\"This is a static method that checks if a given pandas DataFrame is hashable or not.\n",
"\n",
" It checks if all the columns containing object types in the input DataFrame are hashable or not.\n",
" If any column is not hashable, it raises a TypeError indicating which columns are not hashable.\n",
"\n",
" Parameters\n",
" ----------\n",
" df : pandas.DataFrame\n",
" The DataFrame to be checked for hashability.\n",
"\n",
" Raises\n",
" ------\n",
" TypeError\n",
" If any column containing object types in the input DataFrame is not hashable.\n",
" \"\"\n",
"--------------------------------------------------\n",
"ASSISTANT\n",
"\"\n",
"--------------------------------------------------\n"
]
}
],
"source": [
"for message in llmcoder.messages:\n",
" print(message['role'].upper())\n",
" print(message['content'])\n",
" print('-' * 50)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The completion is identical to the user's code, so there are no additions or changes to review. The completion is correct and aligned with the user's intent. \n",
"\n",
"SCORE: 10\n"
]
}
],
"source": [
"for analyzer_message in llmcoder.analyzer_message_history:\n",
" print(analyzer_message[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "llmcoder",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Loading

0 comments on commit a04102c

Please sign in to comment.