Merge pull request #160 from P3ngLiu/feature/v0.2.2/PoT

Feature/v0.2.2/PoT

examples/PoT/README.md

@@ -0,0 +1,125 @@
+# Program of Thought (PoT) Example
+
+Program of Thought (PoT) is a novel approach that combines language models with code execution to solve complex reasoning tasks. It leverages the model's ability to generate executable code that breaks down problems into computational steps, providing a more structured and verifiable solution path compared to traditional methods.
+
+This example demonstrates how to use the framework for Program of Thought tasks. The example code can be found in the `examples/PoT` directory.
+
+```bash
+   cd examples/PoT
+```
+
+## Overview
+
+This example implements a Program of Thought (PoT) workflow that consists of the following components:
+
+1. **Input Interface**
+   - Handles user input containing questions
+   - Manages example prompts for few-shot learning
+   - Processes additional options for the workflow
+
+2. **PoT Workflow**
+   - Takes the user's question and example prompts
+   - Generates executable Python code to solve the problem
+   - Executes the code to obtain numerical answers
+   - Provides step-by-step reasoning through code
+
+3. **PoT Executor**
+   - Executes the generated Python code in a safe environment
+   - Returns the computed answer
+
+4. **Choice Extractor**
+   - Compares the executed result with available choices
+   - Returns the choice that most closely matches the result
+   - Handles numerical and textual comparison to find best match
+
+![DnC Workflow](./docs/images/pot_workflow.jpg)
+
+## Prerequisites
+
+- Python 3.10+
+- Required packages installed (see requirements.txt)
+- Access to OpenAI API or compatible endpoint (see configs/llms/gpt.yml)
+- Redis server running locally or remotely
+- Conductor server running locally or remotely
+
+## Configuration
+
+The container.yaml file is a configuration file that manages dependencies and settings for different components of the system, including Conductor connections, Redis connections, and other service configurations. To set up your configuration:
+
+1. Generate the container.yaml file:
+   ```bash
+   python compile_container.py
+   ```
+   This will create a container.yaml file with default settings under `examples/PoT`.
+
+
+2. Configure your LLM settings in `configs/llms/gpt.yml`:
+   - Set your OpenAI API key or compatible endpoint through environment variable or by directly modifying the yml file
+   ```bash
+   export custom_openai_key="your_openai_api_key"
+   export custom_openai_endpoint="your_openai_endpoint"
+   export model_id="your_model_id"  # e.g. gpt-4, gpt-3.5-turbo
+   ```
+   - Configure other model settings like temperature as needed through environment variable or by directly modifying the yml file
+
+3. Update settings in the generated `container.yaml`:
+   - Modify Redis connection settings:
+     - Set the host, port and credentials for your Redis instance
+     - Configure `redis_stream_client` sections
+   - Update the Conductor server URL under conductor_config section
+   - Adjust any other component settings as needed
+
+4. [Optional] Prepare example prompts:
+   - Create a text file containing example math problems and their Python solutions
+   - Use the format shown in eval_gsm8k_fewshot.py
+   - Pass the file path using --examples when running evaluation
+
+## Running the Example
+
+3. Run the Program of Thought (PoT) example:
+
+   For terminal/CLI usage:
+   ```bash
+   python run_cli.py
+   ```
+
+   For web interface usage:
+   ```bash
+   python run_webpage.py
+   ```
+   For evaluating on GSM8K dataset with few-shot examples:
+   ```bash
+   python eval_gsm8k_fewshot.py \
+     --endpoint "https://api.openai.com/v1" \
+     --api_key "your_openai_api_key" \
+     --model_id "gpt-3.5-turbo" \
+     --dataset_path "gsm8k_test.jsonl" \
+     --examples "examples.txt" \  # Optional: provide custom examples
+     --output_path "output"
+   ```
+
+   For evaluating on AQUA dataset with zero-shot learning:
+   ```bash
+   python eval_aqua_zeroshot.py \
+     --endpoint "https://api.openai.com/v1" \
+     --api_key "your_openai_api_key" \
+     --model_id "gpt-3.5-turbo" \
+     --dataset_path "aqua_test.jsonl" \
+     --output_path "output"
+   ```
+
+   The evaluation scripts will:
+   - Process questions from the dataset using Program of Thought approach
+   - Generate Python code solutions for each question
+   - Save results to JSON files in the specified output directory
+   - Include metrics like prompt tokens and completion tokens
+
+
+## Troubleshooting
+
+If you encounter issues:
+- Verify Redis is running and accessible
+- Check your OpenAI API key is valid
+- Ensure all dependencies are installed correctly
+- Review logs for any error messages
+- **Open an issue on GitHub if you can't find a solution, we will do our best to help you out!**

examples/PoT/agent/input_interface/input_interface.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+
+from omagent_core.utils.registry import registry
+from omagent_core.utils.general import read_image
+from omagent_core.engine.worker.base import BaseWorker
+from omagent_core.utils.logger import logging
+
+
+CURRENT_PATH = Path(__file__).parents[0]
+
+
+@registry.register_worker()
+class PoTInputInterface(BaseWorker):
+    """Input interface processor that handles user questions and example inputs.
+    
+    This processor manages the interactive input collection process for math problem solving:
+    1. Collects a math word problem from the user
+    2. Optionally collects example problems/solutions for few-shot learning
+    3. Optionally collects multiple choice options if applicable
+    4. Validates and formats all inputs appropriately
+    5. Returns a structured dictionary containing the processed inputs
+    
+    The interface is designed to be flexible, allowing both basic question-only
+    inputs as well as more complex scenarios with examples and multiple choice options.
+    """
+
+    def _run(self, *args, **kwargs):
+        # Prompt user for the main math question and extract text content
+        input = self.input.read_input(workflow_instance_id=self.workflow_instance_id, input_prompt='Please input a math related question:')
+        content = input['messages'][-1]['content']
+        for content_item in content:
+            if content_item['type'] == 'text':
+                query = content_item['data']
+
+        # Collect optional example problems/solutions for few-shot learning
+        # User can input "None" to skip this step
+        input = self.input.read_input(workflow_instance_id=self.workflow_instance_id, input_prompt='Please input examples if you have, input "None" if you do not have:')
+        content = input['messages'][-1]['content']
+        for content_item in content:
+            if content_item['type'] == 'text':
+                examples = content_item['data']
+        if examples == 'None':
+            examples = None
+
+        # Collect optional multiple choice options if this is a multiple choice question
+        # User can input "None" for standard numerical answer questions
+        input = self.input.read_input(workflow_instance_id=self.workflow_instance_id, input_prompt='Please input options if you are doing a multiple choice question, input "None" if you do not have:')
+        content = input['messages'][-1]['content']
+        for content_item in content:
+            if content_item['type'] == 'text':
+                options = content_item['data']
+        if options == 'None':
+            options = None
+
+        # Return all collected inputs in a structured format
+        inputs = {'query': query, 'examples': examples, 'options': options}
+        logging.info(inputs)
+        return inputs

examples/PoT/compile_container.py

@@ -0,0 +1,19 @@
+# Import core modules and components
+from omagent_core.utils.container import container
+
+# Import workflow related modules
+from pathlib import Path
+from omagent_core.utils.registry import registry
+
+# Set up path and import modules
+CURRENT_PATH = root_path = Path(__file__).parents[0]
+registry.import_module()
+
+# Register required components
+container.register_callback(callback='AppCallback')
+container.register_input(input='AppInput')
+# Compile container config
+container.compile_config(CURRENT_PATH)
+
+
+

examples/PoT/configs/llms/gpt.yml

@@ -0,0 +1,7 @@
+name: OpenaiGPTLLM
+model_id: ${env| model_id, gpt-3.5-turbo}
+api_key: ${env| custom_openai_key, openai_api_key}
+endpoint: ${env| custom_openai_endpoint, https://api.openai.com/v1}
+temperature: 0
+use_default_sys_prompt: false
+vision: false

examples/PoT/configs/workers/PoT_workflow.yml

@@ -0,0 +1,5 @@
+- name: PoTExecutor
+  llm: ${sub| gpt}
+- name: ChoiceExtractor
+  llm: ${sub| gpt}
+- name: PoTInputInterface

examples/PoT/eval_aqua_zeroshot.py

@@ -0,0 +1,111 @@
+# Import required modules and components
+from omagent_core.utils.container import container
+from omagent_core.engine.workflow.conductor_workflow import ConductorWorkflow
+from omagent_core.advanced_components.workflow.pot.workflow import PoTWorkflow
+from pathlib import Path
+from omagent_core.utils.registry import registry
+from omagent_core.clients.devices.programmatic.client import ProgrammaticClient
+from omagent_core.utils.logger import logging
+import argparse
+import json
+import os
+
+
+def parse_args():
+    """Parse command line arguments for AQUA evaluation"""
+    parser = argparse.ArgumentParser(description='Evaluate AQUA dataset using Program of Thought')
+    parser.add_argument('--endpoint', type=str, default="https://api.openai.com/v1",
+                        help='OpenAI API endpoint')
+    parser.add_argument('--api_key', type=str, default=None,
+                        help='OpenAI API key')
+    parser.add_argument('--model_id', type=str, default="gpt-3.5-turbo",
+                        help='Model ID to use')
+    parser.add_argument('--dataset_path', type=str, default="aqua_test.jsonl",
+                        help='Path to dataset')
+    parser.add_argument('--output_path', type=str, default='output',
+                        help='Path to output file. If not provided, will use default')
+    return parser.parse_args()
+
+def main():
+    """Main function to run AQUA evaluation"""
+    # Parse command line arguments
+    args = parse_args()
+
+    # Set environment variables for API
+    os.environ["custom_openai_endpoint"] = args.endpoint
+    os.environ["custom_openai_key"] = args.api_key
+    os.environ["model_id"] = args.model_id
+
+    # Load dataset and setup variables
+    dataset_path = args.dataset_path
+    model_id = args.model_id
+    dataset_name = 'aqua'
+
+    # Read dataset from JSONL file
+    datasets = []
+    with open(dataset_path, 'r') as f:
+        for line in f:
+            datasets.append(json.loads(line))
+
+    # Setup logging and paths
+    logging.init_logger("omagent", "omagent", level="INFO")
+    CURRENT_PATH = Path(__file__).parents[0]
+
+    # Initialize agent modules and configuration
+    registry.import_module(project_path=CURRENT_PATH.joinpath('agent'))
+    container.from_config(CURRENT_PATH.joinpath('container.yaml'))
+
+    # Setup Program of Thought workflow
+    workflow = ConductorWorkflow(name='PoT')
+    pot_workflow = PoTWorkflow()
+    pot_workflow.set_input(query=workflow.input('query'), examples=workflow.input('examples'), options=workflow.input('options'))
+    workflow >> pot_workflow
+    workflow.register(overwrite=True)
+
+    # Initialize programmatic client
+    config_path = CURRENT_PATH.joinpath('configs')
+    programmatic_client = ProgrammaticClient(processor=workflow, config_path=config_path)
+
+    # Prepare batch processing inputs
+    output_json = []
+    workflow_input_list = []
+    for question in datasets:
+        workflow_input_list.append({
+            "id": question['id'],
+            "query": question['question'],
+            "examples": None,
+            "options": str(question['options'])
+        })
+
+    # Process questions in batches
+    res = programmatic_client.start_batch_processor(workflow_input_list=workflow_input_list, max_tasks=5)
+
+    # Collect results
+    for r, w in zip(res, workflow_input_list):
+        output_json.append({
+            "id": w['id'],
+            "question": w['query'],
+            "last_output": r['last_output'],
+            "prompt_tokens": r['prompt_tokens'],
+            "completion_tokens": r['completion_tokens']
+        })
+
+    # Prepare final output
+    final_output = {
+        "dataset_name": dataset_name,
+        "model_id": model_id,
+        "alg": "POT",
+        "model_result": output_json
+    }
+
+    # Save results to output file
+    if not os.path.exists(args.output_path):
+        os.makedirs(args.output_path)
+    with open(f'{args.output_path}/{dataset_name}_{model_id}_POT_output.json', 'w') as f:
+        json.dump(final_output, f, indent=4)
+
+    # Cleanup
+    programmatic_client.stop_processor()
+
+if __name__ == "__main__":
+    main()