checking data

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/base.py

@@ -3,6 +3,7 @@
 import abc
 from chatsky_llm_autoconfig.graph import BaseGraph
 from chatsky_llm_autoconfig.dialogue import Dialogue
+from langchain_core.language_models.chat_models import BaseChatModel
 
 
 class BaseAlgorithm(BaseModel, abc.ABC):
@@ -71,7 +72,7 @@ async def ainvoke(self, topic: str, graph: BaseGraph) -> BaseGraph:
 class TopicGraphGenerator(BaseAlgorithm):
     """Graph generator that works only with topics."""
 
-    def invoke(self, topic: str) -> BaseGraph:
+    def invoke(self, topic: str, model: BaseChatModel) -> BaseGraph:
         raise NotImplementedError
 
     async def ainvoke(self, topic: str) -> BaseGraph:

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/dialogue_augmentation.py

@@ -31,7 +31,7 @@
 2. The output must be a list of dictionaries, where each dictionary has:
    - 'text': string
    - 'participant': either 'user' or 'assistant'
-   
+
 3. Ensure all utterance variations:
    - Are appropriate for the theme
    - Maintain consistency in tone and style

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/dialogue_generation.py

@@ -1,4 +1,5 @@
 import random
+import networkx as nx
 from chatsky_llm_autoconfig.graph import BaseGraph
 from chatsky_llm_autoconfig.algorithms.base import DialogueGenerator
 from chatsky_llm_autoconfig.dialogue import Dialogue
@@ -69,5 +70,51 @@ def invoke(self, graph: BaseGraph, start_node: int = 1, end_node: int = -1, topi
 
         return all_dialogues
 
-    async def ainvoke(self, *args, **kwargs):
+    async def ainvoke(self, *args, **kwargs): 
         return self.invoke(*args, **kwargs)
+
+
+@AlgorithmRegistry.register(input_type=BaseGraph, output_type=Dialogue)
+class DialoguePathSampler(DialogueGenerator):
+    def invoke(self, graph: BaseGraph, start_node: int = 1, end_node: int = -1, topic="") -> list[Dialogue]:
+        nx_graph = graph.graph
+
+        # Find all nodes with no outgoing edges (end nodes)
+        end_nodes = [node for node in nx_graph.nodes() if nx_graph.out_degree(node) == 0]
+        dialogues = []
+        # If no end nodes found, return empty list
+        if not end_nodes:
+            return []
+
+        all_paths = []
+        # Get paths from start node to each end node
+        for end in end_nodes:
+            paths = list(nx.all_simple_paths(nx_graph, source=start_node, target=end))
+            all_paths.extend(paths)
+
+        for path in all_paths:
+            dialogue_turns = []
+            # Process each node and edge in the path
+            for i in range(len(path)):
+                # Add assistant utterance from current node
+                current_node = path[i]
+                assistant_utterance = random.choice(nx_graph.nodes[current_node]["utterances"])
+                dialogue_turns.append({"text": assistant_utterance, "participant": "assistant"})
+
+                # Add user utterance from edge (if not at last node)
+                if i < len(path) - 1:
+                    next_node = path[i + 1]
+                    edge_data = nx_graph.edges[current_node, next_node]
+                    user_utterance = (
+                        random.choice(edge_data["utterances"])
+                        if isinstance(edge_data["utterances"], list)
+                        else edge_data["utterances"]
+                    )
+                    dialogue_turns.append({"text": user_utterance, "participant": "user"})
+
+            dialogues.append(Dialogue().from_list(dialogue_turns))
+
+        return dialogues
+
+    async def ainvoke(self, *args, **kwargs):
+        return self.invoke(*args, **kwargs)

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/topic_graph_generation.py

@@ -1,29 +1,22 @@
+from typing import Optional
 from chatsky_llm_autoconfig.algorithms.base import TopicGraphGenerator
 from chatsky_llm_autoconfig.autometrics.registry import AlgorithmRegistry
 from chatsky_llm_autoconfig.schemas import DialogueGraph
-from langchain_openai import ChatOpenAI
-
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
-
 from chatsky_llm_autoconfig.graph import BaseGraph, Graph
-import os
+from langchain_core.language_models.chat_models import BaseChatModel
 
-from pydantic import SecretStr
+from pydantic import Field
+from typing import ClassVar
 
 
 @AlgorithmRegistry.register(input_type=str, output_type=BaseGraph)
 class CycleGraphGenerator(TopicGraphGenerator):
     """Generator specifically for topic-based cyclic graphs"""
 
-    prompt: str = ""
-    cycle_graph_generation_prompt: str = ""
-
-    def __init__(self):
-        super().__init__()
-        self.cycle_graph_generation_prompt = PromptTemplate.from_template(
-            """
-    Create a cyclic dialogue graph where the conversation MUST return to an existing node.
+    DEFAULT_TEMPLATE: ClassVar[str] = """
+    Create a complex dialogue graph where the conversation MUST return to an existing node.
 
     **CRITICAL: Response Specificity**
     Responses must acknowledge and build upon what the user has already specified:
@@ -63,26 +56,36 @@ def __init__(self):
 
     **Your task is to create a cyclic dialogue graph about the following topic:** {topic}.
     """
-        )
 
-    def invoke(self, topic: str) -> BaseGraph:
+    cycle_graph_generation_prompt: PromptTemplate = Field(
+        default_factory=lambda: PromptTemplate.from_template(CycleGraphGenerator.DEFAULT_TEMPLATE)
+    )
+
+    def __init__(self, prompt: Optional[PromptTemplate] = None):
+        super().__init__()
+        if prompt is not None:
+            self.cycle_graph_generation_prompt = prompt
+
+    def invoke(self, topic: str, model: BaseChatModel) -> BaseGraph:
         """
         Generate a cyclic dialogue graph based on the topic input.
 
-        :param input_data: TopicInput containing the topic
-        :return: Generated Graph object with cyclic structure
+        Args:
+            topic (str): The topic for the dialogue graph
+            model_name (str): The name of the model to use
+
+        Returns:
+            BaseGraph: Generated Graph object with cyclic structure
         """
         parser = JsonOutputParser(pydantic_object=DialogueGraph)
-        model = ChatOpenAI(model="gpt-4o", api_key=SecretStr(os.getenv("OPENAI_API_KEY") or ""), base_url=os.getenv("OPENAI_BASE_URL"), temperature=0)
+
         chain = self.cycle_graph_generation_prompt | model | parser
 
         generated_graph = chain.invoke({"topic": topic})
-
         return Graph(generated_graph)
 
     async def ainvoke(self, *args, **kwargs):
+        """
+        Async version of invoke - to be implemented
+        """
         pass
-
-
-if __name__ == "__main__":
-    cycle_graph_generator = CycleGraphGenerator()

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/autometrics/registry.py

@@ -11,10 +11,10 @@ class AlgorithmRegistry:
     _algorithms = {}
 
     @classmethod
-    def register(cls, name=None, input_type=None, path_to_result=None, output_type=None):
+    def register(cls, name=None, input_type=None, output_type=None):
         def decorator(func):
             algorithm_name = name or func.__name__
-            cls._algorithms[algorithm_name] = {"type": func, "input_type": input_type, "path_to_result": path_to_result, "output_type": output_type}
+            cls._algorithms[algorithm_name] = {"type": func, "input_type": input_type, "output_type": output_type}
             return func
 
         return decorator

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/autometrics/test_data/complex_graphs_original.json

@@ -251,7 +251,7 @@
         "are_theme_valid": true,
         "are_triplets_valid": [
         "Invalid transition from ['Do you want to add anything else?'] to ['Do you want to order something else?'] via edge '['No']': The transition is invalid because the target utterance repeats the same question as the source utterance, which is unnecessary after the user has already responded with 'No'. The conversation should logically progress or conclude after the user's response.",
-        "Invalid transition from ['We have a vacant room. Do you need anything else?'] to ['Okay, now I need your ID card.'] via edge '['No, thanks']': The transition is invalid because the user's response 'No, thanks' indicates that they do not need anything else, which should logically conclude the interaction. However, the assistant's follow-up request for an ID card does not logically follow from the user's statement, as it introduces a new requirement without a clear connection to the user's response.",
+        "Invalid transition from ['We have a vacant room. Do you need anything else?'] to ['Okay, now I need your ID card.'] via edge '['No, thanks']': The transition is invalid because the user's response 'No, thanks' indicates that they do not need anything else, which should logically conclude the interaction. However, the assistant's follow-up request for an ID card does not logically follow from the user's statement, as it introduces a new requirement without a clear connection to the user's response."
         ],
         "graph": {
             "edges": [

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/metrics/llm_metrics.py

@@ -5,121 +5,110 @@
 This module contains functions that checks Graphs and Dialogues for various metrics using LLM calls.
 """
 
-from chatsky_llm_autoconfig.graph import BaseGraph
-from typing import List, Tuple
+import logging
+import json
+from chatsky_llm_autoconfig.graph import BaseGraph, Graph
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain.prompts import PromptTemplate
 from pydantic import BaseModel, Field
-from typing import List
 from langchain_core.output_parsers import PydanticOutputParser
-import logging
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 
 
-def are_triplets_valid(G: BaseGraph, model: BaseChatModel, topic: str) -> dict[str]:
+def are_triplets_valid(G: Graph, model: BaseChatModel) -> dict[str]:
     """
-    Validates the dialog graph structure and logical transitions between nodes.
+    Validates dialogue graph structure and logical transitions between nodes.
 
     Parameters:
-        G (BaseGraph): The dialog graph to validate
+        G (BaseGraph): The dialogue graph to validate
         model (BaseChatModel): The LLM model to use for validation
-        topic (str): The topic of the dialog
 
     Returns:
         dict: {'value': bool, 'description': str}
     """
-    # Define prompt template and parser inside the function since they're only used here
-    triplet_validate_prompt_template = """
-    You are given a dialog between assistant and a user.
-    source_utterances, edge_utterances, target_utterances are dialog parts and each contains an array with exactly one utterance.
-    They should be read left to right.
-
-    - source_utterances are assistant phrases 
-    - edge_utterances are user phrases
-    - target_utterances are assistant phrases 
-
-    TASK. Evaluate if the transition makes a logical connection when reading from Source utterances to Target utterances through Edge utterances
-
-    this is an invalid transition:
-    {{
-        'source_utterances': ['Welcome to our online bookstore. How can I assist you today?'],
-        'edge_utterances': ['Hello! Are you looking for any book recommendations?'],
-        'target_utterances': ['We have a wide selection of genres. Which do you prefer?'],
-        'topic': 'Dialog about purchasing books between assistant and customer'
-    }}
-
-    Provide your answer in the following JSON format:
-    {{"isValid": true or false, "description": "Explanation of why it's valid or invalid."}}
 
-    Dialog topic: {topic}
-
-    (source_utterances) {source_utterances} -> (edge_utterances) {edge_utterances} -> (target_utterances) {target_utterances}
+    # Define validation result model
+    class TransitionValidationResult(BaseModel):
+        isValid: bool = Field(description="Whether the transition is valid or not")
+        description: str = Field(description="Explanation of why it's valid or invalid")
 
-    Your answer:"""
+    # Create prompt template
+    triplet_validate_prompt_template = """
+    You are evaluating if dialog transitions make logical sense.
+    
+    Given this conversation graph in JSON:
+    {json_graph}
+    
+    For the current transition:
+    Source (Assistant): {source_utterances}
+    User Response: {edge_utterances}
+    Target (Assistant): {target_utterances}
+
+    EVALUATE: Do these three messages form a logical sequence in the conversation?
+    Consider:
+    1. Does the assistant's first response naturally lead to the user's response?
+    2. Does the user's response logically connect to the assistant's next message?
+    3. Is the overall flow natural and coherent?
+
+
+    Reply in JSON format:
+    {{"isValid": true/false, "description": "Brief explanation of why it's valid or invalid"}}
+    """
 
     triplet_validate_prompt = PromptTemplate(
-        input_variables=["source_utterances", "edge_utterances", "target_utterances", "topic"],
-        template=triplet_validate_prompt_template,
+        input_variables=["json_graph", "source_utterances", "edge_utterances", "target_utterances"], template=triplet_validate_prompt_template
     )
 
-    class TransitionValidationResult(BaseModel):
-        isValid: bool = Field(description="Whether the transition is valid or not.")
-        description: str = Field(description="Explanation of why it's valid or invalid.")
-
     parser = PydanticOutputParser(pydantic_object=TransitionValidationResult)
 
-    graph = G.graph_dict
-    # Create a mapping from node IDs to node data for quick access
-    node_map = {node["id"]: node for node in graph["nodes"]}
+    # Convert graph to JSON string
+    graph_json = json.dumps(G.graph_dict)
+
+    # Create node mapping
+    node_map = {node["id"]: node for node in G.graph_dict["nodes"]}
+
     overall_valid = True
     descriptions = []
 
-    for edge in graph["edges"]:
+    for edge in G.graph_dict["edges"]:
         source_id = edge["source"]
         target_id = edge["target"]
-        edge_utterances = edge["utterances"]
 
-        # Check if source and target nodes exist
-        if source_id not in node_map:
-            description = f"Invalid edge: source node {source_id} does not exist."
-            logging.info(description)
-            overall_valid = False
-            descriptions.append(description)
-            continue
-        if target_id not in node_map:
-            description = f"Invalid edge: target node {target_id} does not exist."
-            logging.info(description)
+        if source_id not in node_map or target_id not in node_map:
+            description = f"Invalid edge: missing node reference {source_id} -> {target_id}"
             overall_valid = False
             descriptions.append(description)
             continue
 
-        source_node = node_map[source_id]
-        target_node = node_map[target_id]
-
-        # Get utterances from nodes
-        source_utterances = source_node.get("utterances", [])
-        target_utterances = target_node.get("utterances", [])
+        # Get utterances
+        source_utterances = node_map[source_id]["utterances"]
+        target_utterances = node_map[target_id]["utterances"]
+        edge_utterances = edge["utterances"]
 
-        # Prepare input data for the chain
+        # Prepare input for validation
         input_data = {
+            "json_graph": graph_json,
             "source_utterances": source_utterances,
             "edge_utterances": edge_utterances,
             "target_utterances": target_utterances,
-            "topic": topic,
         }
 
+        # print(triplet_validate_prompt.format(**input_data))
+
+        # Run validation
         triplet_check_chain = triplet_validate_prompt | model | parser
         response = triplet_check_chain.invoke(input_data)
 
         if not response.isValid:
             overall_valid = False
-            description = f"Invalid transition from {source_utterances} to {target_utterances} via edge '{edge_utterances}': {response.description}"
+            description = f"Invalid transition: {response.description}"
             logging.info(description)
             descriptions.append(description)
 
     result = {"value": overall_valid, "description": " ".join(descriptions) if descriptions else "All transitions are valid."}
+
     return result
 
 

experiments/2024.11.14_dialogue2graph/comp_mini_results.csv

@@ -0,0 +1,6 @@
+,topic,are_triplets_valid,are_triplets_valid_details,is_theme_valid,is_theme_valid_details,all_utterances_present
+0,booking a hotel room,False,"Invalid transition: The assistant's first response offers an alternative (two separate single rooms) after stating that there are no vacant rooms. The user's response indicates that this alternative does not suit them, which logically leads to a conclusion of the conversation rather than a request for further assistance. Therefore, the assistant's next message asking if it can help with anything else does not follow logically.",True,"The dialog consistently revolves around the topic of booking a hotel room, including inquiries about the duration of stay, room availability, and the process of checking in and providing identification. All exchanges are relevant to the hotel booking process.",True
+1,abstract purchase,False,"Invalid transition: The assistant's first response asks what the user would like to add, and the user's response indicates they want to add Y. However, the assistant's next message asks if the user wants to order something else, which does not logically follow from the user's intent to add Y. The flow is not coherent as it skips the necessary confirmation or follow-up regarding the addition of Y. Invalid transition: The assistant's first message asks if the user wants to add anything else, and the user responds 'No', which is a logical response. However, the assistant's next message asks if the user wants to order something else, which is not a natural follow-up to the user's 'No' response. The flow is not coherent as it suggests the user has the option to order something else after declining to add anything. Invalid transition: The assistant's message 'Do you want to order something else?' suggests that the user has already made an order, and the user responds 'Yes', indicating they want to order something else. However, the next message from the assistant 'What would you like to order?' implies that the user is starting a new order, which is inconsistent with the context of adding to an existing order.",True,"The dialog stays on the expected topic of abstract purchase as it revolves around making an order, adding items, and confirming the order details. All utterances are related to the process of purchasing.",True
+2,chatting with a smart assistant,True,All transitions are valid.,True,"The dialog remains on the expected topic of chatting with a smart assistant. The user engages in conversation and requests various skills (weather, music, timer) that a smart assistant can perform, which aligns with the purpose of interacting with such an assistant.",True
+3,taking a loan in a bank,False,"Invalid transition: The assistant's question about proceeding with the loan application does not logically lead to the user's question about becoming a client. The user seems to be asking for information unrelated to the loan application process, indicating a disconnect in the conversation flow.",True,"The dialog stays on the expected topic of taking a loan in a bank. The conversation includes questions about becoming a client, filling out a loan application form, and the next steps in the loan process, which are all relevant to the topic of obtaining a loan.",True
+4,coffee shop,False,"Invalid transition: The assistant's question 'Which tea would you like?' does not logically lead to the user's response 'Latte tea' because 'Latte' is a type of coffee, not tea. Therefore, the user's response is not appropriate for the context of the assistant's question.",True,"The dialog stays on the expected topic of a coffee shop, as it revolves around ordering coffee and tea, discussing options, and addressing customer preferences.",True