Skip to content

Commit

Permalink
is_dialogue_valid
Browse files Browse the repository at this point in the history
  • Loading branch information
Yuri Peshkichev committed Jan 15, 2025
1 parent d38163d commit d015ab3
Show file tree
Hide file tree
Showing 11 changed files with 3,122 additions and 245 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import logging
import json
from chatsky_llm_autoconfig.graph import BaseGraph, Graph
from chatsky_llm_autoconfig.dialogue import Dialogue
from langchain_core.language_models.chat_models import BaseChatModel
from langchain.prompts import PromptTemplate
from pydantic import BaseModel, Field
Expand Down Expand Up @@ -111,6 +112,66 @@ class TransitionValidationResult(BaseModel):

return result

def is_dialogue_valid(dialogue: Dialogue, model: BaseChatModel) -> dict[str]:
"""
Validates dialogue graph structure and logical transitions between nodes.
Parameters:
G (BaseGraph): The dialogue graph to validate
model (BaseChatModel): The LLM model to use for validation
Returns:
dict: {'value': bool, 'description': str}
"""

# Define validation result model
class DialogueValidationResult(BaseModel):
isValid: bool = Field(description="Whether the dialogue is valid or not")
description: str = Field(description="Explanation of why it's valid or invalid")

# Create prompt template
dialogue_validate_prompt_template = """
You are evaluating if beginning and ending of dialogue are logical and consistent.
Given this dialogue in JSON:
{dialogue}
EVALUATE: Are the first and final dialogue messages logical in the conversation?
Consider:
1. Does the first message naturally start the dialogue?
2. Does the final message logically connect to the previous dialogue?
3. Does the dialogue look logically finished?
Reply in JSON format:
{{"isValid": true/false, "description": "Brief explanation of why it's valid or invalid"}}
"""

dialogue_validate_prompt = PromptTemplate(
input_variables=["json_dialogue"], template=dialogue_validate_prompt_template
)

parser = PydanticOutputParser(pydantic_object=DialogueValidationResult)

# Convert graph to JSON string
dialogue_json = json.dumps(dialogue)

# Prepare input for validation
input_data = {
"dialogue": dialogue_json,
}

# print(triplet_validate_prompt.format(**input_data))

# Run validation
dialogue_check_chain = dialogue_validate_prompt | model | parser
response = dialogue_check_chain.invoke(input_data)

result = {"value": response.isValid, "description": response.description}

return result



def is_theme_valid(G: BaseGraph, model: BaseChatModel, topic: str) -> dict[str]:
"""
Expand Down
Loading

0 comments on commit d015ab3

Please sign in to comment.