diff --git a/agents-api/agents_api/activities/utils.py b/agents-api/agents_api/activities/utils.py index 5c8dd1e75..28af694cc 100644 --- a/agents-api/agents_api/activities/utils.py +++ b/agents-api/agents_api/activities/utils.py @@ -397,8 +397,8 @@ def get_handler(system: SystemDef) -> Callable: from ..queries.agents.update_agent import update_agent as update_agent_query from ..queries.docs.delete_doc import delete_doc as delete_doc_query from ..queries.docs.list_docs import list_docs as list_docs_query + from ..queries.entries.get_history import get_history as get_history_query from ..queries.sessions.create_session import create_session as create_session_query - from ..queries.sessions.delete_session import delete_session as delete_session_query from ..queries.sessions.get_session import get_session as get_session_query from ..queries.sessions.list_sessions import list_sessions as list_sessions_query from ..queries.sessions.update_session import update_session as update_session_query diff --git a/agents-api/agents_api/app.py b/agents-api/agents_api/app.py index c977491bc..99c7814ef 100644 --- a/agents-api/agents_api/app.py +++ b/agents-api/agents_api/app.py @@ -26,7 +26,7 @@ async def lifespan(*containers: list[FastAPI | ObjectWithState]): pg_dsn = os.environ.get("PG_DSN") for container in containers: - if not getattr(container.state, "postgres_pool", None): + if hasattr(container, "state") and not getattr(container.state, "postgres_pool", None): container.state.postgres_pool = await create_db_pool(pg_dsn) # INIT S3 # @@ -35,7 +35,7 @@ async def lifespan(*containers: list[FastAPI | ObjectWithState]): s3_endpoint = os.environ.get("S3_ENDPOINT") for container in containers: - if not getattr(container.state, "s3_client", None): + if hasattr(container, "state") and not getattr(container.state, "s3_client", None): session = get_session() container.state.s3_client = await session.create_client( "s3", @@ -49,13 +49,13 @@ async def lifespan(*containers: list[FastAPI | ObjectWithState]): finally: # CLOSE POSTGRES # for container in containers: - if getattr(container.state, "postgres_pool", None): + if hasattr(container, "state") and getattr(container.state, "postgres_pool", None): await container.state.postgres_pool.close() container.state.postgres_pool = None # CLOSE S3 # for container in containers: - if getattr(container.state, "s3_client", None): + if hasattr(container, "state") and getattr(container.state, "s3_client", None): await container.state.s3_client.close() container.state.s3_client = None diff --git a/agents-api/agents_api/clients/litellm.py b/agents-api/agents_api/clients/litellm.py index 782bf52f6..25249d42b 100644 --- a/agents-api/agents_api/clients/litellm.py +++ b/agents-api/agents_api/clients/litellm.py @@ -105,10 +105,8 @@ async def aembedding( embedding_list: list[dict[Literal["embedding"], list[float]]] = response.data # Truncate the embedding to the specified dimensions - embedding_list = [ + return [ item["embedding"][:dimensions] for item in embedding_list if len(item["embedding"]) >= dimensions ] - - return embedding_list \ No newline at end of file diff --git a/agents-api/agents_api/queries/chat/gather_messages.py b/agents-api/agents_api/queries/chat/gather_messages.py index 1ed96bf0d..0a644a7bb 100644 --- a/agents-api/agents_api/queries/chat/gather_messages.py +++ b/agents-api/agents_api/queries/chat/gather_messages.py @@ -1,6 +1,7 @@ from typing import TypeVar from uuid import UUID +import numpy as np from beartype import beartype from fastapi import HTTPException from pydantic import ValidationError @@ -10,14 +11,13 @@ from ...common.protocol.developers import Developer from ...common.protocol.sessions import ChatContext from ...common.utils.db_exceptions import common_db_exceptions, partialclass +from ..docs.mmr import maximal_marginal_relevance from ..docs.search_docs_by_embedding import search_docs_by_embedding from ..docs.search_docs_by_text import search_docs_by_text from ..docs.search_docs_hybrid import search_docs_hybrid from ..entries.get_history import get_history from ..sessions.get_session import get_session from ..utils import rewrap_exceptions -from ..docs.mmr import maximal_marginal_relevance -import numpy as np T = TypeVar("T") @@ -77,86 +77,90 @@ async def gather_messages( ) recall_options = session.recall_options - # search the last `search_threshold` messages - search_messages = [ - msg - for msg in (past_messages + new_raw_messages)[-(recall_options.num_search_messages) :] - if isinstance(msg["content"], str) and msg["role"] in ["user", "assistant"] - ] - - if len(search_messages) == 0: - return past_messages, [] + # Ensure recall_options is not None and has the necessary attributes + if recall and recall_options: + # search the last `search_threshold` messages + search_messages = [ + msg + for msg in (past_messages + new_raw_messages)[ + -(recall_options.num_search_messages) : + ] + if isinstance(msg["content"], str) and msg["role"] in ["user", "assistant"] + ] - # Search matching docs - embed_text = "\n\n".join([ - f"{msg.get('name') or msg['role']}: {msg['content']}" for msg in search_messages - ]).strip() - - # Don't embed if search mode is text only - if recall_options.mode != "text": - [query_embedding, *_] = await litellm.aembedding( - # Truncate on the left to keep the last `search_query_chars` characters - inputs=embed_text[-(recall_options.max_query_length) :], - # TODO: Make this configurable once it's added to the ChatInput model - embed_instruction="Represent the query for retrieving supporting documents: ", - ) - - # Truncate on the right to take only the first `search_query_chars` characters - query_text = search_messages[-1]["content"].strip()[: recall_options.max_query_length] - - # List all the applicable owners to search docs from - active_agent_id = chat_context.get_active_agent().id - user_ids = [user.id for user in chat_context.users] - owners = [("user", user_id) for user_id in user_ids] + [("agent", active_agent_id)] - - # Search for doc references - doc_references: list[DocReference] = [] - match recall_options.mode: - case "vector": - doc_references: list[DocReference] = await search_docs_by_embedding( - developer_id=developer.id, - owners=owners, - query_embedding=query_embedding, - connection_pool=connection_pool, - ) - case "hybrid": - doc_references: list[DocReference] = await search_docs_hybrid( - developer_id=developer.id, - owners=owners, - text_query=query_text, - embedding=query_embedding, - connection_pool=connection_pool, - ) - case "text": - doc_references: list[DocReference] = await search_docs_by_text( - developer_id=developer.id, - owners=owners, - query=query_text, - connection_pool=connection_pool, + if len(search_messages) == 0: + return past_messages, [] + + # Search matching docs + embed_text = "\n\n".join([ + f"{msg.get('name') or msg['role']}: {msg['content']}" for msg in search_messages + ]).strip() + + # Don't embed if search mode is text only + if recall_options.mode != "text": + [query_embedding, *_] = await litellm.aembedding( + # Truncate on the left to keep the last `search_query_chars` characters + inputs=embed_text[-(recall_options.max_query_length) :], + # TODO: Make this configurable once it's added to the ChatInput model + embed_instruction="Represent the query for retrieving supporting documents: ", ) - # Apply MMR if enabled - if ( - recall_options.mmr_strength > 0 - and len(doc_references) > recall_options.limit - and recall_options.mode != "text" - and len([doc for doc in doc_references if doc.snippet.embedding is not None]) >= 2 - ): - # FIXME: This is a temporary fix to ensure that the MMR algorithm works. - # We shouldn't be having references without embeddings. - doc_references = [ - doc for doc in doc_references if doc.snippet.embedding is not None - ] + # Truncate on the right to take only the first `search_query_chars` characters + query_text = search_messages[-1]["content"].strip()[: recall_options.max_query_length] + + # List all the applicable owners to search docs from + active_agent_id = chat_context.get_active_agent().id + user_ids = [user.id for user in chat_context.users] + owners = [("user", user_id) for user_id in user_ids] + [("agent", active_agent_id)] + + # Search for doc references + doc_references: list[DocReference] = [] + match recall_options.mode: + case "vector": + doc_references = await search_docs_by_embedding( + developer_id=developer.id, + owners=owners, + query_embedding=query_embedding, + connection_pool=connection_pool, + ) + case "hybrid": + doc_references = await search_docs_hybrid( + developer_id=developer.id, + owners=owners, + text_query=query_text, + embedding=query_embedding, + connection_pool=connection_pool, + ) + case "text": + doc_references = await search_docs_by_text( + developer_id=developer.id, + owners=owners, + query=query_text, + connection_pool=connection_pool, + ) + + # Apply MMR if enabled + if ( + recall_options.mmr_strength > 0 + and len(doc_references) > recall_options.limit + and recall_options.mode != "text" + and len([doc for doc in doc_references if doc.snippet.embedding is not None]) >= 2 + ): + # FIXME: This is a temporary fix to ensure that the MMR algorithm works. + # We shouldn't be having references without embeddings. + doc_references = [ + doc for doc in doc_references if doc.snippet.embedding is not None + ] + + # Apply MMR + indices = maximal_marginal_relevance( + np.asarray(query_embedding), + [doc.snippet.embedding for doc in doc_references], + k=recall_options.limit, + ) + doc_references = [doc for i, doc in enumerate(doc_references) if i in set(indices)] - # Apply MMR - indices = maximal_marginal_relevance( - np.asarray(query_embedding), - [doc.snippet.embedding for doc in doc_references], - k=recall_options.limit, - ) - # Apply MMR - doc_references = [ - doc for i, doc in enumerate(doc_references) if i in set(indices) - ] + return past_messages, doc_references - return past_messages, doc_references + # If recall is False or recall_options is None, return past messages with no doc references + return past_messages, [] diff --git a/agents-api/scripts/agents_api.py b/agents-api/scripts/agents_api.py deleted file mode 100644 index 8ab7d2e0c..000000000 --- a/agents-api/scripts/agents_api.py +++ /dev/null @@ -1,6 +0,0 @@ -import fire -from agents_api.web import main - - -def run(): - fire.Fire(main) diff --git a/agents-api/tests/test_chat_routes.py b/agents-api/tests/test_chat_routes.py index 02758b2e8..990893ade 100644 --- a/agents-api/tests/test_chat_routes.py +++ b/agents-api/tests/test_chat_routes.py @@ -97,7 +97,7 @@ async def _( connection_pool=pool, ) - (embed, _) = mocks + (_embed, _) = mocks chat_context = await prepare_chat_context( developer_id=developer_id, diff --git a/cookbooks/01-website-crawler.py b/cookbooks/01-website-crawler.py index d8f712bee..f7effcd74 100644 --- a/cookbooks/01-website-crawler.py +++ b/cookbooks/01-website-crawler.py @@ -1,5 +1,6 @@ import os import uuid + import yaml from julep import Client @@ -10,7 +11,8 @@ # Creating Julep Client with the API Key api_key = os.getenv("JULEP_API_KEY") if not api_key: - raise ValueError("JULEP_API_KEY not found in environment variables") + msg = "JULEP_API_KEY not found in environment variables" + raise ValueError(msg) client = Client(api_key=api_key, environment="dev") @@ -26,6 +28,11 @@ model="gpt-4o", ) +spider_api_key = os.getenv("SPIDER_API_KEY") +if not spider_api_key: + msg = "SPIDER_API_KEY not found in environment variables" + raise ValueError(msg) + # Defining a Task task_def = yaml.safe_load(f""" name: Crawling Task @@ -63,7 +70,7 @@ page['content'] for page in _['result'] ) ) - + # Prompt step to create a summary of the results - prompt: | You are {{{{agent.about}}}} @@ -90,6 +97,7 @@ # Waiting for the execution to complete import time + time.sleep(5) # Getting the execution details @@ -104,4 +112,4 @@ # Stream the steps of the defined task print("Streaming execution transitions:") -print(client.executions.transitions.stream(execution_id=execution.id)) \ No newline at end of file +print(client.executions.transitions.stream(execution_id=execution.id)) diff --git a/cookbooks/02-sarcastic-news-headline-generator.py b/cookbooks/02-sarcastic-news-headline-generator.py index cf305c15e..e5dbe7f48 100644 --- a/cookbooks/02-sarcastic-news-headline-generator.py +++ b/cookbooks/02-sarcastic-news-headline-generator.py @@ -1,5 +1,6 @@ import os import uuid + import yaml from julep import Client @@ -10,7 +11,8 @@ # Create Julep Client with the API Key api_key = os.getenv("JULEP_API_KEY") if not api_key: - raise ValueError("JULEP_API_KEY not found in environment variables") + msg = "JULEP_API_KEY not found in environment variables" + raise ValueError(msg) client = Client(api_key=api_key, environment="dev") @@ -76,7 +78,8 @@ ) # Waiting for the execution to complete -import time +import time + time.sleep(5) # Getting the execution details @@ -92,4 +95,3 @@ # Stream the steps of the defined task print("Streaming execution transitions:") print(client.executions.transitions.stream(execution_id=execution.id)) - diff --git a/cookbooks/03-trip-planning-assistant.py b/cookbooks/03-trip-planning-assistant.py index e3375a728..dab1e2b7c 100644 --- a/cookbooks/03-trip-planning-assistant.py +++ b/cookbooks/03-trip-planning-assistant.py @@ -1,7 +1,8 @@ +import os import uuid + import yaml from julep import Client -import os openweathermap_api_key = os.getenv("OPENWEATHERMAP_API_KEY") brave_api_key = os.getenv("BRAVE_API_KEY") @@ -139,6 +140,7 @@ # Wait for the execution to complete import time + time.sleep(200) # Getting the execution details @@ -146,10 +148,10 @@ execution = client.executions.get(execution.id) # Print the output print(execution.output) -print("-"*50) +print("-" * 50) -if 'final_plan' in execution.output: - print(execution.output['final_plan']) +if "final_plan" in execution.output: + print(execution.output["final_plan"]) # Lists all the task steps that have been executed up to this point in time transitions = client.executions.transitions.list(execution_id=execution.id).items @@ -158,4 +160,4 @@ for transition in reversed(transitions): print("Transition type: ", transition.type) print("Transition output: ", transition.output) - print("-"*50) \ No newline at end of file + print("-" * 50) diff --git a/cookbooks/04-hook-generator-trending-reels.py b/cookbooks/04-hook-generator-trending-reels.py index b3aa8ebbb..0de1d9bbb 100644 --- a/cookbooks/04-hook-generator-trending-reels.py +++ b/cookbooks/04-hook-generator-trending-reels.py @@ -2,30 +2,34 @@ # %% # Global UUID is generated for agent and task -import time, yaml -from dotenv import load_dotenv import os +import time import uuid + +import yaml +from dotenv import load_dotenv + load_dotenv(override=True) -AGENT_UUID = uuid.uuid4() +AGENT_UUID = uuid.uuid4() TASK_UUID = uuid.uuid4() -RAPID_API_KEY = os.getenv('RAPID_API_KEY') -RAPID_API_HOST = os.getenv('RAPID_API_HOST') -JULEP_API_KEY = os.getenv('JULEP_API_KEY') or os.getenv('JULEP_API_KEY_LOCAL') +RAPID_API_KEY = os.getenv("RAPID_API_KEY") +RAPID_API_HOST = os.getenv("RAPID_API_HOST") +JULEP_API_KEY = os.getenv("JULEP_API_KEY") or os.getenv("JULEP_API_KEY_LOCAL") -print(f'AGENT_UUID: {AGENT_UUID}') -print(f'TASK_UUID: {TASK_UUID}') -print(f'JULEP_API_KEY: {JULEP_API_KEY}') -print(f'RAPID_API_KEY: {RAPID_API_KEY}') -print(f'RAPID_API_HOST: {RAPID_API_HOST}') +print(f"AGENT_UUID: {AGENT_UUID}") +print(f"TASK_UUID: {TASK_UUID}") +print(f"JULEP_API_KEY: {JULEP_API_KEY}") +print(f"RAPID_API_KEY: {RAPID_API_KEY}") +print(f"RAPID_API_HOST: {RAPID_API_HOST}") # ### Creating Julep Client with the API Key from julep import Client + # # Create a client -client = Client(api_key=JULEP_API_KEY,environment="dev") +client = Client(api_key=JULEP_API_KEY, environment="dev") # Creating an agent for handling persistent sessions agent = client.agents.create_or_update( @@ -180,7 +184,7 @@ "Take [discount] off when you try [product].", "I didn’t know X could be related to X.", "Why is it important to [do product-related task]?", - "99\% of your [target audience] don't. To be the 1% you need to [X].", + r"99\% of your [target audience] don't. To be the 1% you need to [X].", "This [product] is the secret to [X]." ]}, {"categories": "Curiosity & Engagement", @@ -220,7 +224,7 @@ hooks_doc_content = [] for category in hooks_data: - hooks_doc_content.extend(category['content']) + hooks_doc_content.extend(category["content"]) doc = client.agents.docs.create( agent_id=AGENT_UUID, title="hooks_doc", content=hooks_doc_content) @@ -231,7 +235,6 @@ print(doc.content) # Task Definition -import yaml task_def = yaml.safe_load(f""" name: Agent Crawler @@ -241,7 +244,7 @@ api_call: method: GET url: "https://instagram-scraper-api2.p.rapidapi.com/v1/hashtag" - headers: + headers: x-rapidapi-key: "{RAPID_API_KEY}" x-rapidapi-host: "{RAPID_API_HOST}" follow_redirects: true @@ -279,7 +282,7 @@ 'virality_score': (((response.get('reshare_count') or 0) / (response.get('play_count') or 1)) if (response.get('play_count') or 0) > 0 else 0), 'engagement_score': (((response.get('like_count') or 0) / (response.get('play_count') or 1)) if (response.get('play_count') or 0) > 0 else 0), 'video_duration': (response.get('video_duration') or 0) - }} for response in _['json']['data']['items'])" + }} for response in _['json']['data']['items'])" - over: _['summary'] parallelism: 4 @@ -295,7 +298,7 @@ Engagement Score: {{{{_['engagement_score']}}}} Provide a json repsonse containing the caption, virality score, enagement score and one-liner description for the reel. - unwrap: true + unwrap: true - evaluate: summary: outputs[1]['summary'] @@ -304,7 +307,7 @@ - tool: get_hooks_doc arguments: agent_id: "'{AGENT_UUID}'" - + - evaluate: hooks_doc: _[0]['content'] @@ -315,9 +318,9 @@ - role: system content: >- You are a skilled content creator tasked with generating 3 engaging video hooks for each reel having its description and caption. Use the following document containing hook templates to create effective hooks: - + {{{{_.hooks_doc}}}} - + Here are the caption and description to create hooks for: Caption: {{{{_['caption']}}}} @@ -326,7 +329,7 @@ Engagement Score: {{{{_['engagement_score']}}}} Your task is to generate 3 hooks (for the reel) by adapting the most suitable templates from the document. Each hook should be no more than 1 sentence long and directly relate to its corresponding idea. - + Basically, all the ideas are taken from a search about this topic, which is {{{{inputs[0].topic}}}}. You should focus on this while writing the hooks. Ensure that each hook is creative, engaging, and relevant to its idea while following the structure of the chosen template. @@ -355,7 +358,6 @@ ) # Waiting for the execution to complete -import time time.sleep(120) # Lists all the task steps that have been executed up to this point in time @@ -365,11 +367,9 @@ for transition in reversed(transitions): print("Transition type: ", transition.type) print("Transition output: ", transition.output) - print("-"*50) + print("-" * 50) import json + response = client.executions.transitions.list(execution_id=execution.id).items[0].output print(json.dumps(response, indent=4)) - - - diff --git a/agents-api/scripts/__init__.py b/cookbooks/__init__.py similarity index 100% rename from agents-api/scripts/__init__.py rename to cookbooks/__init__.py diff --git a/example.py b/example.py index cc7982523..d8e093b5b 100644 --- a/example.py +++ b/example.py @@ -1,13 +1,14 @@ -### Step 0: Setup +# Step 0: Setup import os import time + import yaml -from julep import Julep # or AsyncJulep +from julep import Julep # or AsyncJulep client = Julep(api_key=os.environ["JULEP_API_KEY"]) -### Step 1: Create an Agent +# Step 1: Create an Agent agent = client.agents.create( name="Storytelling Agent", @@ -15,7 +16,7 @@ about="You are a creative storyteller that crafts engaging stories on a myriad of topics.", ) -### Step 2: Create a Task that generates a story and comic strip +# Step 2: Create a Task that generates a story and comic strip task_yaml = """ name: Storyteller @@ -83,7 +84,7 @@ {% for idea in outputs[1].plot_ideas %} - {{idea}} {% endfor %} - + Here are the results from researching the plot ideas on Wikipedia: {{_.wikipedia_results}} @@ -118,7 +119,7 @@ **yaml.safe_load(task_yaml) ) -### Step 3: Execute the Task +# Step 3: Execute the Task execution = client.executions.create( task_id=task.id, @@ -126,7 +127,7 @@ ) # 🎉 Watch as the story and comic panels are generated -while (result := client.executions.get(execution.id)).status not in ['succeeded', 'failed']: +while (result := client.executions.get(execution.id)).status not in ["succeeded", "failed"]: print(result.status, result.output) time.sleep(1) @@ -134,4 +135,4 @@ if result.status == "succeeded": print(result.output) else: - raise Exception(result.error) \ No newline at end of file + raise Exception(result.error) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/generate_changelog.py b/scripts/generate_changelog.py index 425d05210..dbfb3118f 100644 --- a/scripts/generate_changelog.py +++ b/scripts/generate_changelog.py @@ -1,142 +1,150 @@ # Standard library imports -import sys import json -import re import logging -from pathlib import Path import os +import re +import sys import time -from typing import List, Dict, Any +from pathlib import Path +from typing import Any + +import yaml # Third-party imports from julep import Client -import yaml # Configure logging with timestamp, level, and message format -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") # Constants and configurations HTML_TAGS_PATTERN = re.compile(r"(<[^>]+>)") # Regex pattern to match HTML tags -REQUIRED_ENV_VARS = ['AGENT_UUID', 'TASK_UUID', 'JULEP_API_KEY'] # List of required environment variables +REQUIRED_ENV_VARS = ["AGENT_UUID", "TASK_UUID", "JULEP_API_KEY"] # List of required environment variables + def load_template(filename: str) -> str: """Load template content from file""" - return Path(f'./scripts/templates/{filename}').read_text(encoding='utf-8') + return Path(f"./scripts/templates/{filename}").read_text(encoding="utf-8") + def run_task(pr_data: str) -> str: """ Execute the changelog generation task using Julep API. - + Args: pr_data (str): Formatted PR data to process - + Returns: str: Generated changelog content - + Raises: ValueError: If required environment variables are missing Exception: If task execution fails """ # Validate env vars with list comprehension if missing_vars := [var for var in REQUIRED_ENV_VARS if not os.environ.get(var)]: - raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}") - - client = Client(api_key=os.environ['JULEP_API_KEY'], environment="dev") - + msg = f"Missing required environment variables: {', '.join(missing_vars)}" + raise ValueError(msg) + + client = Client(api_key=os.environ["JULEP_API_KEY"], environment="dev") + # Use context manager for file operations - with Path('./scripts/templates/changelog.yaml').open(encoding='utf-8') as f: + with Path("./scripts/templates/changelog.yaml").open(encoding="utf-8") as f: task_description = yaml.safe_load(f) - + # Create or update the AI agent - agent = client.agents.create_or_update( - agent_id=os.environ['AGENT_UUID'], + client.agents.create_or_update( + agent_id=os.environ["AGENT_UUID"], name="Changelog Generator", about="An AI assistant that can generate a changelog from a list of PRs.", model="gpt-4o", ) # Create or update the task configuration - task = client.tasks.create_or_update( - task_id=os.environ['TASK_UUID'], - agent_id=os.environ['AGENT_UUID'], + client.tasks.create_or_update( + task_id=os.environ["TASK_UUID"], + agent_id=os.environ["AGENT_UUID"], **task_description ) # Create a new execution instance execution = client.executions.create( - task_id=os.environ['TASK_UUID'], + task_id=os.environ["TASK_UUID"], input={"pr_data": str(pr_data)} ) # Wait for task completion using context manager for proper resource cleanup with client: - while (result := client.executions.get(execution.id)).status not in ['succeeded', 'failed']: + while (result := client.executions.get(execution.id)).status not in ["succeeded", "failed"]: time.sleep(3) - + if result.status != "succeeded": raise Exception(result.error) return result.output -def preserve_and_update_changelog(new_changelog: str, source: str = './CHANGELOG.md') -> None: + +def preserve_and_update_changelog(new_changelog: str, source: str = "./CHANGELOG.md") -> None: """ Save the generated changelog while preserving HTML content. - + Args: new_changelog (str): The new changelog content to save source (str): Path to the changelog file (default: 'CHANGELOG.md') """ path = Path(source) path.parent.mkdir(parents=True, exist_ok=True) - + # Load templates at runtime - html_content = load_template('header.html') - author_list = load_template('authors.md') - + html_content = load_template("header.html") + author_list = load_template("authors.md") + content = f"{html_content}\n\n{new_changelog}\n\n{author_list}" - path.write_text(content, encoding='utf-8') + path.write_text(content, encoding="utf-8") + def is_html_tag(segment: str) -> bool: """ Check if a given string segment is an HTML tag. - + Args: segment (str): String to check - + Returns: bool: True if segment is an HTML tag, False otherwise """ return re.fullmatch(HTML_TAGS_PATTERN, segment) is not None + def process_body(body: str) -> str: """ Process PR body text by removing HTML tags and special markers. - + Args: body (str): PR description body text - + Returns: str: Cleaned and processed body text """ if not body: return "" - + # Remove HTML tags and clean up the text segments = [seg for seg in re.split(HTML_TAGS_PATTERN, body) if not is_html_tag(seg)] processed_body = "".join(segments) return processed_body.replace(">", "").replace("[!IMPORTANT]", "").strip() + def process_pr_data(pr_data: str) -> str: """ Generate changelog entries from PR data. - + Args: pr_data (str): JSON string containing PR information - + Returns: str: Formatted changelog entries """ - prs: List[Dict[str, Any]] = json.loads(pr_data) - + prs: list[dict[str, Any]] = json.loads(pr_data) + # Use list comprehension with f-strings entries = [ f"""- PR #{pr['number']}: {pr['title']} @@ -148,42 +156,44 @@ def process_pr_data(pr_data: str) -> str: ] return "\n".join(entries) + def main(pr_data: str) -> None: """ Main function to orchestrate changelog generation process. - + Args: pr_data (str): JSON string containing PR information - + Raises: Exception: If any step in the process fails """ try: logging.info("Processing PR data...") processed_pr_data = process_pr_data(pr_data) - + logging.info("Running task...") final_changelog = run_task(processed_pr_data) - + logging.info("Saving changelog...") preserve_and_update_changelog(final_changelog) logging.info("Successfully saved changelog to CHANGELOG.md") # delete the pr_data.json file - os.remove('pr_data.json') + os.remove("pr_data.json") logging.info("Deleted pr_data.json file") except Exception as e: - logging.error(f"Failed to generate changelog: {str(e)}") + logging.error(f"Failed to generate changelog: {e!s}") raise + # Script entry point if __name__ == "__main__": try: # Read PR data from JSON file - with open('pr_data.json', 'r') as file: + with open("pr_data.json") as file: pr_data = file.read() main(pr_data) except Exception as e: - logging.error(f"Script failed: {str(e)}") - sys.exit(1) \ No newline at end of file + logging.error(f"Script failed: {e!s}") + sys.exit(1) diff --git a/scripts/generate_jwt.py b/scripts/generate_jwt.py index 7953f08d8..23f5fddb5 100644 --- a/scripts/generate_jwt.py +++ b/scripts/generate_jwt.py @@ -1,5 +1,7 @@ +from datetime import UTC, datetime, timedelta + import jwt -from datetime import datetime, timedelta + dev_secret = "" prod_secret = "" test_secret = "your-supersupersecret-jwt-token-with-at-least-32-characters-long" @@ -7,10 +9,10 @@ { "sub": "097720c5-ab84-438c-b8b0-68e0eabd31ff", "email": "e@mail.com", - "iat": datetime.now(), - "exp": datetime.now() + timedelta(days=100), + "iat": datetime.now(UTC), # Specify timezone + "exp": datetime.now(UTC) + timedelta(days=100), # Specify timezone }, test_secret, algorithm="HS512", ) -print(encoded_jwt) \ No newline at end of file +print(encoded_jwt) diff --git a/scripts/readme_translator.py b/scripts/readme_translator.py index 180c69d9a..bd210d80b 100644 --- a/scripts/readme_translator.py +++ b/scripts/readme_translator.py @@ -1,31 +1,35 @@ -import re import logging -from typing import List -from pathlib import Path +import re from functools import partial -from deep_translator import GoogleTranslator +from pathlib import Path + import parmapper +from deep_translator import GoogleTranslator # Configure logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") HTML_TAGS_PATTERN = r"(<[^>]+>)" CODEBLOCK_PATTERN = r"(```[\s\S]*?```|\n)" + def create_translator(target: str) -> GoogleTranslator: """ Create a translator for a given target language. """ return GoogleTranslator(source="en", target=target) + def is_html_tag(segment: str) -> bool: """Check if the segment is an HTML tag.""" return re.fullmatch(HTML_TAGS_PATTERN, segment) is not None + def is_special_character(segment: str) -> bool: """Check if the segment consists of special characters only.""" return re.fullmatch(r'^[!"#$%&\'()*+,\-./:;<=>?@[\]^_`{|}~]+$', segment) is not None + def translate_sub_segment(translator: GoogleTranslator, sub_segment: str) -> str: """Translate a single sub-segment.""" try: @@ -35,45 +39,47 @@ def translate_sub_segment(translator: GoogleTranslator, sub_segment: str) -> str logging.error(f"Error translating segment '{sub_segment}': {e}") return sub_segment + def translate_segment(translator: GoogleTranslator, segment: str) -> str: """ Translate a given raw HTML content using the provided translator, preserving HTML tags and newlines. """ - if re.fullmatch(CODEBLOCK_PATTERN, segment) or segment == '\n': + if re.fullmatch(CODEBLOCK_PATTERN, segment) or segment == "\n": return segment segments = re.split(HTML_TAGS_PATTERN, segment) translated_segments = [] for sub_segment in segments: - if is_html_tag(sub_segment): - translated_segments.append(sub_segment) - elif is_special_character(sub_segment): + if is_html_tag(sub_segment) or is_special_character(sub_segment): translated_segments.append(sub_segment) else: translated_segments.append(translate_sub_segment(translator, sub_segment)) return "".join(translated_segments) + def translate_readme(source: str, target: str) -> str: """ Translate a README file from source to target language, preserving code blocks and newlines. """ - file_content = Path(source).read_text(encoding='utf-8') + file_content = Path(source).read_text(encoding="utf-8") translator = create_translator(target) segments = re.split(CODEBLOCK_PATTERN, file_content) segment_translation = partial(translate_segment, translator) translated_segments = list(parmapper.parmap(segment_translation, segments)) - return ''.join(translated_segments) + return "".join(translated_segments) + def save_translated_readme(translated_content: str, lang: str) -> None: """ Save the translated README content to a file. """ filename = f"README-{lang.split('-')[-1].upper()}.md" - with open(filename, "w", encoding='utf-8') as file: + with open(filename, "w", encoding="utf-8") as file: file.write(translated_content) + def main() -> None: """ Main function to translate README.md to multiple languages. @@ -87,5 +93,6 @@ def main() -> None: save_translated_readme(translated_readme, lang) logging.info(f"Saved translated README for {lang}.") + if __name__ == "__main__": main()