diff --git a/docs/index.md b/docs/index.md index 97d9b14..f606721 100644 --- a/docs/index.md +++ b/docs/index.md @@ -150,7 +150,7 @@ You can then store your prompt template locally or share it on the HF Hub. >>> prompt_template.save_to_local("./tests/test_data/code_teacher_test.yaml") >>> # or save it on the HF Hub >>> prompt_template.save_to_hub(repo_id="MoritzLaurer/example_prompts_test", filename="code_teacher_test.yaml", create_repo=True) -CommitInfo(commit_url='https://huggingface.co/MoritzLaurer/example_prompts_test/commit/4cefd2c94f684f9bf419382f96b36692cd175e84', commit_message='Upload prompt template code_teacher_test.yaml', commit_description='', oid='4cefd2c94f684f9bf419382f96b36692cd175e84', pr_url=None, repo_url=RepoUrl('https://huggingface.co/MoritzLaurer/example_prompts_test', endpoint='https://huggingface.co', repo_type='model', repo_id='MoritzLaurer/example_prompts_test'), pr_revision=None, pr_num=None) +CommitInfo(commit_url='https://huggingface.co/MoritzLaurer/example_prompts_test/commit/4cefd2c94f684f9bf419382f96b36692cd175e84', commit_message='Upload prompt template code_teacher_test.yaml', commit_description='', oid='4cefd2c94f684f9bf419382f96b36692cd175e84', pr_url=None, repo_url=RepoUrl('https://huggingface.co/MoritzLaurer/example_prompts_test', endpoint='https://huggingface.co', repo_type='dataset', repo_id='MoritzLaurer/example_prompts_test'), pr_revision=None, pr_num=None) ``` diff --git a/docs/repo_types_examples.md b/docs/repo_types_examples.md index a9df2d5..04b5fb9 100644 --- a/docs/repo_types_examples.md +++ b/docs/repo_types_examples.md @@ -7,18 +7,18 @@ The HF Hub is currently organized around three main repository types: - Dataset repositories: Repos with tabular datasets (mostly in parquet format). - Spaces repositories: Repos with hosted applications (often with code and data, which is then visualized in the Space). -Prompt templates can be integrated into any of these repository types as .yaml or .json files. [TODO: add JSON support, currently only YAML is supported.] +Prompt templates can be saved into any of these repository types as .yaml or .json files. We recommend saving prompt templates in dataset repos by default. -## 1. Prompt templates as independent artifacts in model repos -Many prompt templates can be reused with various models and are not linked to specific model weights. These prompt templates can be shared in an HF model repo, where the model card provides a description and usage instructions, and prompt templates are shared via .yaml or .json files in the same repository. +## 1. Saving collections of prompt templates as independent artifacts in dataset repos +Many prompt templates can be reused in different projects and with different models. We recommend sharing collections of reusable prompt templates in HF dataset repos, where the dataset card provides a description and usage instructions and the templates are shared as .yaml or .json files in the same repository.
1. Example: using the leaked Claude Artifacts prompt -#### List all prompt templates stored in a HF model repo +#### List all prompt templates stored in a HF dataset repo This [example HF repository](https://huggingface.co/MoritzLaurer/closed_system_prompts) contains leaked or released prompts from Anthropic and OpenAI. @@ -45,7 +45,7 @@ print(prompt_template) Prompt templates are downloaded as either `ChatPromptTemplate` or `TextPromptTemplate` classes. This class makes it easy to populate a prompt template and convert it into a format that's compatible with different LLM clients. The type is automatically determined based on whether the YAML contains a simple string (TextPromptTemplate) or a list of dictionaries following the OpenAI messages format (ChatPromptTemplate). #### Populate and use the prompt template -With the `create_messages` method, we can then populate the prompt template for a specific use-case. +We can then populate the prompt template for a specific use-case. ```python # Check which variables the prompt template requires @@ -53,14 +53,17 @@ print(prompt_template.template_variables) # ['current_date', 'user_message'] user_message = "Create a simple calculator web application" -messages_anthropic = prompt_template.create_messages( +messages = prompt_template.populate_template( user_message=user_message, current_date="Monday 21st October 2024", - client="anthropic" ) + +# The default output is in the OpenAI messages format. We can easily reformat it for another client. +messages_anthropic = messages.format_for_client(client="anthropic") + ``` -The output is a list or a dictionary in the format expected by the specified LLM client. For example, OpenAI expects a list of message dictionaries, while Anthropic expects a dictionary with "system" and "messages" keys. +The output is a PopulatedPrompt instance which contains a list or a dictionary in the format expected by the specified LLM client. For example, OpenAI expects a list of message dictionaries, while Anthropic expects a dictionary with "system" and "messages" keys. ```python #!pip install anthropic @@ -80,13 +83,13 @@ response = client_anthropic.messages.create(
2. Example: JudgeBench paper prompts -The paper "JudgeBench: A Benchmark for Evaluating LLM-Based Judges" (paper) collects several prompts for using LLMs to evaluate unstructured LLM outputs. After copying them into a HF Hub model repo in the standardized YAML format, they can be directly loaded and populated. +The paper "JudgeBench: A Benchmark for Evaluating LLM-Based Judges" (paper) collects several prompts for using LLMs to evaluate unstructured LLM outputs. After copying them into a HF Hub dataset repo in the standardized YAML format, they can be directly loaded and populated. ```python from prompt_templates import PromptTemplateLoader prompt_template = PromptTemplateLoader.from_hub( - repo_id="MoritzLaurer/judgebench-prompts", - filename="vanilla-prompt.yaml" + repo_id="MoritzLaurer/prompts_from_papers", + filename="judgebench-vanilla-prompt.yaml" ) ``` @@ -95,7 +98,7 @@ prompt_template = PromptTemplateLoader.from_hub(
3. Example: Sharing closed system prompts -The community has extracted system prompts from closed API providers like OpenAI or Anthropic and these prompts are unsystematically shared via GitHub, Reddit etc. (e.g. Anthropic Artifacts prompt). Some API providers have also started sharing their system prompts on their websites in non-standardized HTML (Anthropic, OpenAI). To simplify to use of these prompts, they can be shared in a HF Hub model repo as standardized YAML files. +The community has extracted system prompts from closed API providers like OpenAI or Anthropic and these prompts are unsystematically shared via GitHub, Reddit etc. (e.g. Anthropic Artifacts prompt). Some API providers have also started sharing their system prompts on their websites in non-standardized HTML (Anthropic, OpenAI). To simplify to use of these prompts, they can be shared in a HF Hub dataset repo as standardized YAML files. ```python @@ -112,7 +115,7 @@ prompt_template = PromptTemplateLoader.from_hub( -## 2. Sharing prompts together with model weights +## 2. Attaching prompt templates to model weights Some open-weight LLMs have been trained to exhibit specific behaviours with specific prompt templates. The vision language model [InternVL2](https://huggingface.co/collections/OpenGVLab/internvl-20-667d3961ab5eb12c7ed1463e) was trained to predict bounding boxes for manually specified areas with a special prompt template; the VLM [Molmo](https://huggingface.co/collections/allenai/molmo-66f379e6fe3b8ef090a8ca19) was trained to predict point coordinates of objects of images with a special prompt template; etc. @@ -135,7 +138,7 @@ prompt_template = PromptTemplateLoader.from_hub( # populate prompt image_url = "https://unsplash.com/photos/ZVw3HmHRhv0/download?ixid=M3wxMjA3fDB8MXxhbGx8NHx8fHx8fDJ8fDE3MjQ1NjAzNjl8&force=true&w=1920" region_to_detect = "the bird" -messages = prompt_template.create_messages(image_url=image_url, region_to_detect=region_to_detect, client="openai") +messages = prompt_template.populate_template(image_url=image_url, region_to_detect=region_to_detect) print(messages) #[{'role': 'user', @@ -171,7 +174,7 @@ response.choices[0].message.content -## 3. Attaching prompts to datasets +## 3. Attaching prompt templates to datasets LLMs are increasingly used to help create datasets, for example for quality filtering or synthetic text generation. The prompt templates used for creating a dataset are currently unsystematically shared on GitHub ([example](https://github.com/huggingface/cosmopedia/tree/main/prompts)), referenced in dataset cards ([example](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu#annotation)), or stored in .txt files ([example](https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier/blob/main/utils/prompt.txt)), @@ -186,7 +189,7 @@ To facilitate reproduction, these dataset prompt templates can be shared in YAML The FineWeb-Edu dataset was created by prompting `Meta-Llama-3-70B-Instruct` to score the educational value of web texts. The authors provide the prompt template in a .txt file. -When provided in a YAML/JSON file in the dataset repo, the prompt template can easily be loaded and supplemented with metadata like the model_id or generation parameters for easy reproducibility. +When provided in a YAML/JSON file in the dataset repo, the prompt template can easily be loaded and supplemented with client_parameters like the model_id or generation parameters for easy reproducibility. See this example dataset repository @@ -198,12 +201,11 @@ from transformers import pipeline prompt_template = PromptTemplateLoader.from_hub( repo_id="MoritzLaurer/dataset_prompts", filename="fineweb-edu-prompt.yaml", - repo_type="dataset" ) # populate the prompt text_to_score = "The quick brown fox jumps over the lazy dog" -messages = prompt_template.create_messages(text_to_score=text_to_score) +messages = prompt_template.populate_template(text_to_score=text_to_score) # test prompt with local llama model_id = "meta-llama/Llama-3.2-1B-Instruct" # prompt was original created for meta-llama/Meta-Llama-3-70B-Instruct @@ -239,11 +241,12 @@ The prompts could be directly added to the dataset repository in the standardize -## 4. Attaching prompts to HF Spaces +## 4. Attaching prompt templates to HF Spaces + +[TODO: create example] See also the [Agents](agents.md) and [Tools](standard_tool_format.md) page for using HF Spaces for hosting prompts and tools as part of agents. -[TODO: create example] diff --git a/docs/standard_prompt_format.md b/docs/standard_prompt_format.md index d71ded5..ec25df2 100644 --- a/docs/standard_prompt_format.md +++ b/docs/standard_prompt_format.md @@ -1,12 +1,12 @@ # Standardizing prompt templates -The library expects prompt templates to be stored as modular YAML or JSON files. They can be stored locally or in an HF repository, see for example the `Files` tab in these repos for [open-weight model prompts](https://huggingface.co/MoritzLaurer/open_models_special_prompts), [closed-model prompts](https://huggingface.co/MoritzLaurer/closed_system_prompts), or [dataset prompts](https://huggingface.co/datasets/MoritzLaurer/dataset_prompts). +The library expects prompt templates to be stored as modular YAML or JSON files. They can be stored locally or in a HF repository. A prompt template YAML or JSON file must follow the following standardized structure: - Top-level key (required): `prompt`. This top-level key signals to the parser that the content of the file is a prompt template. - Second-level key (required): `template`. This can be either a simple _string_, or a _list of dictionaries_ following the OpenAI messages format. The messages format is recommended for use with LLM APIs or inference containers. Variable placeholders for populating the prompt template string are denoted with double curly brackets _{{...}}_. -- Second-level keys (optional): (1) `template_variables` (_list_): variables for populating the prompt template. This is used for input validation and to make the required variables for long templates easily accessible; (2) `metadata` (_dict_): information about the template such as the source, date, author etc.; (3) `client_parameters` (_dict_): parameters for the inference client (e.g. temperature, model). +- Second-level keys (optional): (1) `template_variables` (_list_): variables for populating the prompt template. This is used for input validation and to make the required variables for long templates easily accessible; (2) `metadata` (_dict_): information about the template such as the source, date, author etc.; (3) `client_parameters` (_dict_): parameters for the inference client (e.g. temperature, model_id). Example prompt template following the standard in YAML: ```yaml @@ -29,6 +29,8 @@ prompt: author: "Karl Marx" ``` +**Repository types on the HF Hub:** Prompt template files can be shared in any HF repo type (dataset/model/space repo). We recommend sharing collections of prompt templates in dataset repos by default. See details [here](repo_types_examples.md). + **Naming convention:** We call a file a *"prompt template"*, when it has placeholders ({{...}}) for dynamically populating the template similar to an f-string. This makes files more useful and reusable by others for different use-cases. Once the placeholders in the template are populated with specific variables, we call it a *"prompt"*. The following example illustrates how the prompt template becomes a prompt. @@ -66,7 +68,7 @@ The following example illustrates how the prompt template becomes a prompt. - YAML (or JSON) is the standard for working with prompts in production settings in my experience with practitioners. See also [this discussion](https://github.com/langchain-ai/langchain/discussions/21672). - Managing individual prompt templates in separate YAML files makes each prompt template an independent modular unit. - This makes it e.g. easier to add metadata and production-relevant information in the respective prompt YAML file. - - Prompt templates in individual YAML files also enables users to add individual prompts into any HF repo abstraction (Model, Space, Dataset), while datasets always have to be their own abstraction. + - Prompt templates in individual YAML files also enables users to add individual prompts into any HF repo abstraction (Dataset, Model, Space repos), while tabular dataset file types are only compatible with one specific repo type. ### Pro/Con JSON files - The same pro arguments of YAML also apply to JSON. @@ -79,8 +81,8 @@ The following example illustrates how the prompt template becomes a prompt. - Issue: allows arbitrary code execution and is less safe - Harder to read for beginners -### Pro/Con prompts as datasets -- Some prompt datasets like [awesome-chatgpt-prompts](https://huggingface.co/datasets/fka/awesome-chatgpt-prompts) have received many likes on HF +### Pro/Con tabular file formats (e.g. parquet) +- Some tabular prompt datasets like [awesome-chatgpt-prompts](https://huggingface.co/datasets/fka/awesome-chatgpt-prompts) have received many likes on HF - The dataset viewer allows for easy and quick visualization - Main cons: the tabular data format is not well suited for reusing prompt templates and is not standard among practitioners @@ -94,8 +96,7 @@ and is not standard among practitioners ### Compatibility with LangChain LangChain is a great library for creating interoperability between different LLM clients. -This library is inspired by LangChain's [PromptTemplate](https://python.langchain.com/api_reference/core/prompts/langchain_core.prompts.prompt.PromptTemplate.html) -and [ChatPromptTemplate](https://python.langchain.com/api_reference/core/prompts/langchain_core.prompts.chat.ChatPromptTemplate.html) classes. One difference is that the LangChain ChatPromptTemplate expects a "messages" key instead of a "template" key for the prompt template in the messages format. This HF library uses the "template" key both for HF [TextPromptTemplate][prompt_templates.prompt_templates.TextPromptTemplate] and for HF [ChatPromptTemplate][prompt_templates.prompt_templates.ChatPromptTemplate] for simplicity. If you still load a YAML/JSON file with a "messages" key, it will be automatically renamed to "template". You can also always convert a HF PromptTemplate to a LangChain template with [.to_langchain_template()][prompt_templates.prompt_templates.ChatPromptTemplate.to_langchain_template]. The objective of this library is not to reproduce the full functionality of a library like LangChain, but to enable the community to share prompts on the HF Hub and load and reuse them with any of their favourite libraries. +This library is inspired by LangChain's [PromptTemplate](https://python.langchain.com/api_reference/core/prompts/langchain_core.prompts.prompt.PromptTemplate.html) and [ChatPromptTemplate](https://python.langchain.com/api_reference/core/prompts/langchain_core.prompts.chat.ChatPromptTemplate.html) classes. One difference is that the LangChain ChatPromptTemplate expects a "messages" key instead of a "template" key for the prompt template in the messages format. This HF library uses the "template" key both for HF [TextPromptTemplate][prompt_templates.prompt_templates.TextPromptTemplate] and for HF [ChatPromptTemplate][prompt_templates.prompt_templates.ChatPromptTemplate] for simplicity. If you still load a YAML/JSON file with a "messages" key, it will be automatically renamed to "template". You can also always convert a HF PromptTemplate to a LangChain template with [.to_langchain_template()][prompt_templates.prompt_templates.ChatPromptTemplate.to_langchain_template]. The objective of this library is not to reproduce the full functionality of a library like LangChain, but to enable the community to share prompts on the HF Hub and load and reuse them with any of their favourite libraries. A `PromptTemplate` from `prompt_templates` can be easily converted to a langchain template: diff --git a/examples/example-usage.ipynb b/examples/example-usage.ipynb index fd206e0..63c3296 100644 --- a/examples/example-usage.ipynb +++ b/examples/example-usage.ipynb @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "0b4d0b9b", "metadata": {}, "outputs": [ @@ -89,7 +89,7 @@ " 'openai-metaprompt-text.yaml']" ] }, - "execution_count": 2, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -102,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "379532fe", "metadata": {}, "outputs": [ @@ -112,7 +112,7 @@ "{'source': 'https://gist.github.com/dedlim/6bf6d81f77c19e20cd40594aa09e3ecd'}" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -279,21 +279,29 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 1, "id": "f8ad2dde", "metadata": {}, "outputs": [], "source": [ "import os\n", - "os.chdir(\"/Users/moritzlaurer/huggingface/projects/hf-hub-prompts\")" + "os.chdir(\"/Users/moritzlaurer/huggingface/projects/prompt-templates\")" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 2, "id": "0e10ad0f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/Users/moritzlaurer/huggingface/projects/prompt-templates\n" + ] + } + ], "source": [ "from prompt_templates import ChatPromptTemplate\n", "messages_template = [\n", @@ -314,6 +322,8 @@ " metadata=metadata,\n", ")\n", "\n", + "print(os.getcwd())\n", + "\n", "prompt_template\n", "\n", "prompt_template.save_to_local(\"./tests/test_data/code_teacher_test.yaml\")\n" @@ -321,23 +331,28 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "id": "eac98b0b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "CommitInfo(commit_url='https://huggingface.co/MoritzLaurer/example_prompts_test/commit/4cefd2c94f684f9bf419382f96b36692cd175e84', commit_message='Upload prompt template code_teacher_test.yaml', commit_description='', oid='4cefd2c94f684f9bf419382f96b36692cd175e84', pr_url=None, repo_url=RepoUrl('https://huggingface.co/MoritzLaurer/example_prompts_test', endpoint='https://huggingface.co', repo_type='model', repo_id='MoritzLaurer/example_prompts_test'), pr_revision=None, pr_num=None)" + "CommitInfo(commit_url='https://huggingface.co/datasets/MoritzLaurer/example_prompts_test_priv/commit/b60063f879c7d64fd9023740bbd716061f74e653', commit_message='Upload prompt template code_teacher_test.yaml', commit_description='', oid='b60063f879c7d64fd9023740bbd716061f74e653', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/MoritzLaurer/example_prompts_test_priv', endpoint='https://huggingface.co', repo_type='dataset', repo_id='MoritzLaurer/example_prompts_test_priv'), pr_revision=None, pr_num=None)" ] }, - "execution_count": 16, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "prompt_template.save_to_hub(repo_id=\"MoritzLaurer/example_prompts_test\", filename=\"code_teacher_test.yaml\", create_repo=True)" + "prompt_template.save_to_hub(\n", + " repo_id=\"MoritzLaurer/example_prompts_test_priv\", \n", + " filename=\"code_teacher_test.yaml\", \n", + " create_repo=True,\n", + " private=True\n", + ")" ] }, { @@ -596,7 +611,7 @@ "outputs": [], "source": [ "import os\n", - "os.chdir(\"/Users/moritzlaurer/huggingface/projects/hf-hub-prompts\")" + "os.chdir(\"/Users/moritzlaurer/huggingface/projects/prompt-templates\")" ] }, { @@ -792,7 +807,7 @@ "outputs": [], "source": [ "import os\n", - "os.chdir(\"/Users/moritzlaurer/huggingface/projects/hf-hub-prompts\")" + "os.chdir(\"/Users/moritzlaurer/huggingface/projects/prompt-templates\")" ] }, { @@ -895,7 +910,7 @@ "outputs": [], "source": [ "import os\n", - "os.chdir(\"/Users/moritzlaurer/huggingface/projects/hf-hub-prompts\")" + "os.chdir(\"/Users/moritzlaurer/huggingface/projects/prompt-templates\")" ] }, { diff --git a/prompt_templates/loaders.py b/prompt_templates/loaders.py index d2d2b98..5087f6e 100644 --- a/prompt_templates/loaders.py +++ b/prompt_templates/loaders.py @@ -120,7 +120,7 @@ def from_hub( cls, repo_id: str, filename: str, - repo_type: str = "model", + repo_type: str = "dataset", revision: Optional[str] = None, populator: Optional[PopulatorType] = None, jinja2_security_level: Literal["strict", "standard", "relaxed"] = "standard", @@ -134,7 +134,7 @@ def from_hub( repo_id (str): The repository ID on Hugging Face Hub (e.g., 'username/repo') filename (str): Name of the YAML file containing the template repo_type (str, optional): Type of repository. Must be one of - ['model', 'dataset', 'space']. Defaults to "model" + ['dataset', 'model', 'space']. Defaults to "dataset" revision (Optional[str], optional): Git revision to download from. Can be a branch name, tag, or commit hash. Defaults to None @@ -183,8 +183,8 @@ def from_hub( except ValueError as e: raise ValueError(f"Invalid repo_id format: {str(e)}") from e - if repo_type not in ["model", "dataset", "space"]: - raise ValueError(f"repo_type must be one of ['model', 'dataset', 'space'], got {repo_type}") + if repo_type not in ["dataset", "model", "space"]: + raise ValueError(f"repo_type must be one of ['dataset', 'model', 'space'], got {repo_type}") # Ensure .yaml extension if not filename.endswith(VALID_PROMPT_EXTENSIONS): @@ -349,7 +349,7 @@ def from_local(cls, path: Union[str, Path]) -> Tool: return tool @classmethod - def from_hub(cls, repo_id: str, filename: str, repo_type: str = "model", revision: Optional[str] = None) -> Tool: + def from_hub(cls, repo_id: str, filename: str, repo_type: str = "dataset", revision: Optional[str] = None) -> Tool: """Load a tool from the Hugging Face Hub. Downloads and loads a tool function from a repository on the Hugging Face Hub. @@ -359,7 +359,7 @@ def from_hub(cls, repo_id: str, filename: str, repo_type: str = "model", revisio repo_id (str): The repository ID on Hugging Face Hub (e.g., 'username/repo') filename (str): Name of the Python file containing the tool repo_type (str, optional): Type of repository. Must be one of - ['model', 'dataset', 'space']. Defaults to "model" + ['dataset', 'model', 'space']. Defaults to "dataset" revision (Optional[str], optional): Git revision to download from. Can be a branch name, tag, or commit hash. Defaults to None @@ -387,8 +387,8 @@ def from_hub(cls, repo_id: str, filename: str, repo_type: str = "model", revisio except ValueError as e: raise ValueError(f"Invalid repo_id format: {str(e)}") from e - if repo_type not in ["model", "dataset", "space"]: - raise ValueError(f"repo_type must be one of ['model', 'dataset', 'space'], got {repo_type}") + if repo_type not in ["dataset", "model", "space"]: + raise ValueError(f"repo_type must be one of ['dataset', 'model', 'space'], got {repo_type}") # Ensure .py extension if not filename.endswith(".py"): @@ -550,12 +550,14 @@ def _extract_tool_dependencies(file_path: Path) -> Set[str]: return dependencies -def list_prompt_templates(repo_id: str, repo_type: Optional[str] = "model", token: Optional[str] = None) -> List[str]: +def list_prompt_templates( + repo_id: str, repo_type: Optional[str] = "dataset", token: Optional[str] = None +) -> List[str]: """List available prompt template YAML files in a Hugging Face Hub repository. Args: repo_id (str): The repository ID on Hugging Face Hub. - repo_type (Optional[str]): The type of repository. Defaults to "model". + repo_type (Optional[str]): The type of repository. Defaults to "dataset". token (Optional[str]): An optional authentication token. Defaults to None. Returns: @@ -583,12 +585,12 @@ def list_prompt_templates(repo_id: str, repo_type: Optional[str] = "model", toke return sorted(yaml_files) -def list_tools(repo_id: str, repo_type: str = "model", token: Optional[str] = None) -> List[str]: +def list_tools(repo_id: str, repo_type: str = "dataset", token: Optional[str] = None) -> List[str]: """List available tool Python files in a Hugging Face Hub repository. Args: repo_id (str): The repository ID on Hugging Face Hub - repo_type (str, optional): The type of repository. Defaults to "model" + repo_type (str, optional): The type of repository. Defaults to "dataset" token (Optional[str], optional): An optional authentication token. Defaults to None Returns: diff --git a/prompt_templates/prompt_templates.py b/prompt_templates/prompt_templates.py index 8a07641..3f12a1f 100644 --- a/prompt_templates/prompt_templates.py +++ b/prompt_templates/prompt_templates.py @@ -8,6 +8,8 @@ import jinja2 import yaml from huggingface_hub import HfApi +from huggingface_hub.hf_api import CommitInfo +from huggingface_hub.utils import RepositoryNotFoundError from jinja2 import Environment, meta from .constants import Jinja2SecurityLevel, PopulatorType @@ -101,25 +103,39 @@ def save_to_hub( self, repo_id: str, filename: str, - repo_type: str = "model", + repo_type: str = "dataset", + format: Optional[Literal["yaml", "json"]] = None, token: Optional[str] = None, - commit_message: Optional[str] = None, create_repo: bool = False, - format: Optional[Literal["yaml", "json"]] = None, - ) -> Any: + private: bool = False, + exist_ok: bool = True, + resource_group_id: Optional[str] = None, + revision: Optional[str] = None, + create_pr: bool = False, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + parent_commit: Optional[str] = None, + ) -> CommitInfo: """Save the prompt template to the Hugging Face Hub as a YAML or JSON file. Args: repo_id: The repository ID on the Hugging Face Hub (e.g., "username/repo-name") filename: Name of the file to save (e.g., "prompt.yaml" or "prompt.json") - repo_type: Type of repository ("model", "dataset", or "space"). Defaults to "model" + repo_type: Type of repository ("dataset", "model", or "space"). Defaults to "dataset" token: Hugging Face API token. If None, will use token from environment commit_message: Custom commit message. If None, uses default message create_repo: Whether to create the repository if it doesn't exist. Defaults to False format: Output format ("yaml" or "json"). If None, inferred from filename extension + private: Whether to create a private repository. Defaults to False + exist_ok: Don't error if repo already exists. Defaults to True + resource_group_id: Optional resource group ID to associate with the repository + revision: Optional branch/revision to push to. Defaults to main branch + create_pr: Whether to create a Pull Request instead of pushing directly. Defaults to False + commit_description: Optional commit description + parent_commit: Optional parent commit to create PR from Returns: - str: URL of the uploaded file on the Hugging Face Hub + CommitInfo: Information about the commit/PR Examples: >>> from prompt_templates import ChatPromptTemplate @@ -144,6 +160,7 @@ def save_to_hub( ... repo_id="MoritzLaurer/example_prompts_test", ... filename="code_teacher_test.yaml", ... #create_repo=True, # if the repo does not exist, create it + ... #private=True, # if you want to create a private repo ... #token="hf_..." ... ) 'https://huggingface.co/MoritzLaurer/example_prompts_test/blob/main/code_teacher_test.yaml' @@ -184,15 +201,36 @@ def save_to_hub( # Upload to Hub api = HfApi(token=token) + + # Create repository if requested if create_repo: - api.create_repo(repo_id=repo_id, repo_type=repo_type, exist_ok=True) + api.create_repo( + repo_id=repo_id, + repo_type=repo_type, + token=token, + private=private, + exist_ok=exist_ok, + resource_group_id=resource_group_id, + ) + else: + # Check if repo exists to provide better error message + try: + api.repo_info(repo_id=repo_id, repo_type=repo_type) + except RepositoryNotFoundError as e: + raise ValueError(f"Repository {repo_id} does not exist. Set create_repo=True to create it.") from e + # Upload file return api.upload_file( path_or_fileobj=file_content.encode(), path_in_repo=filename, repo_id=repo_id, repo_type=repo_type, + token=token, commit_message=commit_message or f"Upload prompt template {filename}", + commit_description=commit_description, + revision=revision, + create_pr=create_pr, + parent_commit=parent_commit, ) def save_to_local(self, path: Union[str, Path], format: Optional[Literal["yaml", "json"]] = None) -> None: diff --git a/tests/test_data/closed_system_prompts/README.md b/tests/test_data/closed_system_prompts/README.md index a4528ef..1a68203 100644 --- a/tests/test_data/closed_system_prompts/README.md +++ b/tests/test_data/closed_system_prompts/README.md @@ -1,7 +1,8 @@ --- license: mit +library_name: prompt-templates tags: -- prompt +- prompts --- # System prompts of prominent closed LLM providers diff --git a/tests/test_data/dataset_prompts/README.md b/tests/test_data/dataset_prompts/README.md new file mode 100644 index 0000000..c7cd9e2 --- /dev/null +++ b/tests/test_data/dataset_prompts/README.md @@ -0,0 +1,56 @@ +--- +license: mit +library_name: prompt-templates +tags: +- prompts +--- + +## Sharing prompts linked to datasets +This repo illustrates how you can use the `prompt_templates` library to load prompts from YAML files in dataset repositories. + +LLMs are increasingly used to help create datasets, for example for quality filtering or synthetic text generation. +The prompts used for creating a dataset are currently unsystematically shared on GitHub ([example](https://github.com/huggingface/cosmopedia/tree/main/prompts)), +referenced in dataset cards ([example](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu#annotation)), or stored in .txt files ([example](https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier/blob/main/utils/prompt.txt)), +hidden in paper appendices or not shared at all. +This makes reproducibility unnecessarily difficult. + +To facilitate reproduction, these dataset prompts can be shared in YAML files in HF dataset repositories together with metadata on generation parameters, model_ids etc. + + +### Example: FineWeb-Edu + +The FineWeb-Edu dataset was created by prompting `Meta-Llama-3-70B-Instruct` to score the educational value of web texts. The authors provide the prompt in a .txt file. + +When provided in a YAML file in the dataset repo, the prompt can easily be loaded and supplemented with metadata like the model_id or generation parameters for easy reproducibility. + +```python +#!pip install hf_hub_prompts +from hf_hub_prompts import download_prompt +import torch +from transformers import pipeline + +prompt_template = download_prompt(repo_id="MoritzLaurer/dataset_prompts", filename="fineweb-edu-prompt.yaml", repo_type="dataset") + +# populate the prompt +text_to_score = "The quick brown fox jumps over the lazy dog" +messages = prompt_template.populate_template(text_to_score=text_to_score) + +# test prompt with local llama +model_id = "meta-llama/Llama-3.2-1B-Instruct" # prompt was original created for meta-llama/Meta-Llama-3-70B-Instruct + +pipe = pipeline( + "text-generation", + model=model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +outputs = pipe( + messages, + max_new_tokens=512, +) + +print(outputs[0]["generated_text"][-1]) +``` + + diff --git a/tests/test_data/dataset_prompts/fineweb-edu-prompt.yaml b/tests/test_data/dataset_prompts/fineweb-edu-prompt.yaml new file mode 100644 index 0000000..2f2c2b6 --- /dev/null +++ b/tests/test_data/dataset_prompts/fineweb-edu-prompt.yaml @@ -0,0 +1,26 @@ +prompt: + template: + - role: user + content: |- + Below is an extract from a web page. Evaluate whether the page has a high educational value and could be useful in an educational setting for teaching from primary school to grade school levels using the additive 5-point scoring system described below. Points are accumulated based on the satisfaction of each criterion: + + - Add 1 point if the extract provides some basic information relevant to educational topics, even if it includes some irrelevant or non-academic content like advertisements and promotional material. + - Add another point if the extract addresses certain elements pertinent to education but does not align closely with educational standards. It might mix educational content with non-educational material, offering a superficial overview of potentially useful topics, or presenting information in a disorganized manner and incoherent writing style. + - Award a third point if the extract is appropriate for educational use and introduces key concepts relevant to school curricula. It is coherent though it may not be comprehensive or could include some extraneous information. It may resemble an introductory section of a textbook or a basic tutorial that is suitable for learning but has notable limitations like treating concepts that are too complex for grade school students. + - Grant a fourth point if the extract highly relevant and beneficial for educational purposes for a level not higher than grade school, exhibiting a clear and consistent writing style. It could be similar to a chapter from a textbook or a tutorial, offering substantial educational content, including exercises and solutions, with minimal irrelevant information, and the concepts aren't too advanced for grade school students. The content is coherent, focused, and valuable for structured learning. + - Bestow a fifth point if the extract is outstanding in its educational value, perfectly suited for teaching either at primary school or grade school. It follows detailed reasoning, the writing style is easy to follow and offers profound and thorough insights into the subject matter, devoid of any non-educational or complex content. + + The extract: + {{text_to_score}}. + + After examining the extract: + - Briefly justify your total score, up to 100 words. + - Conclude with the score using the format: "Educational score: " + + template_variables: + - text_to_score + + metadata: + source: https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier/blob/main/utils/prompt.txt + original-model-used: meta-llama/Meta-Llama-3-70B-Instruct + implementation-details: https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu#annotation diff --git a/tests/test_data/example_prompts/README.md b/tests/test_data/example_prompts/README.md index aedd2f5..2d8c2f9 100644 --- a/tests/test_data/example_prompts/README.md +++ b/tests/test_data/example_prompts/README.md @@ -1,5 +1,6 @@ --- license: apache-2.0 +library_name: prompt-templates tags: - prompts --- diff --git a/tests/test_data/example_tools/README.md b/tests/test_data/example_tools/README.md index 7b95401..cfe389c 100644 --- a/tests/test_data/example_tools/README.md +++ b/tests/test_data/example_tools/README.md @@ -1,3 +1,6 @@ --- license: apache-2.0 +library_name: prompt-templates +tags: +- tools --- diff --git a/tests/test_data/open_model_special_prompts/README.md b/tests/test_data/open_models_special_prompts/README.md similarity index 92% rename from tests/test_data/open_model_special_prompts/README.md rename to tests/test_data/open_models_special_prompts/README.md index d9b06f5..7e9b28e 100644 --- a/tests/test_data/open_model_special_prompts/README.md +++ b/tests/test_data/open_models_special_prompts/README.md @@ -1,7 +1,9 @@ --- license: mit +library_name: prompt-templates tags: -- prompt +- prompts +- model-prompts --- ## Sharing special prompts of open-weight models @@ -14,7 +16,7 @@ To elicit this capability, users need to use this special prompt: `Please provid These these kinds of task-specific special prompts are currently unsystematically reported in model cards, github repos, .txt files etc. The prompt_templates library standardises the sharing of prompts in YAML files. -I recommend sharing these these special prompts directly in the model repository of the respective. +I recommend sharing these special prompts directly in the model repository of the respective model. Below is an example for the InternVL2 model. @@ -31,7 +33,7 @@ prompt_template = PromptTemplateLoader.from_hub(repo_id="MoritzLaurer/open_model # populate prompt image_url = "https://unsplash.com/photos/ZVw3HmHRhv0/download?ixid=M3wxMjA3fDB8MXxhbGx8NHx8fHx8fDJ8fDE3MjQ1NjAzNjl8&force=true&w=1920" region_to_detect = "the bird" -messages = prompt_template.format_messages(image_url=image_url, region_to_detect=region_to_detect, client="openai") +messages = prompt_template.populate_template(image_url=image_url, region_to_detect=region_to_detect) print(messages) # out: [{'role': 'user' @@ -49,7 +51,7 @@ prompt_template = PromptTemplateLoader.from_hub(repo_id="MoritzLaurer/open_model # populate prompt image_url = "https://unsplash.com/photos/ZVw3HmHRhv0/download?ixid=M3wxMjA3fDB8MXxhbGx8NHx8fHx8fDJ8fDE3MjQ1NjAzNjl8&force=true&w=1920" -messages = prompt_template.format_messages(image_url=image_url, client="openai") +messages = prompt_template.populate_template(image_url=image_url) print(messages) # [{'role': 'user', diff --git a/tests/test_data/open_model_special_prompts/internvl2-bbox-prompt.yaml b/tests/test_data/open_models_special_prompts/internvl2-bbox-prompt.yaml similarity index 100% rename from tests/test_data/open_model_special_prompts/internvl2-bbox-prompt.yaml rename to tests/test_data/open_models_special_prompts/internvl2-bbox-prompt.yaml diff --git a/tests/test_data/open_model_special_prompts/internvl2-objectdetection-prompt.yaml b/tests/test_data/open_models_special_prompts/internvl2-objectdetection-prompt.yaml similarity index 100% rename from tests/test_data/open_model_special_prompts/internvl2-objectdetection-prompt.yaml rename to tests/test_data/open_models_special_prompts/internvl2-objectdetection-prompt.yaml diff --git a/tests/test_data/prompts_from_papers/README.md b/tests/test_data/prompts_from_papers/README.md new file mode 100644 index 0000000..e1a8b0e --- /dev/null +++ b/tests/test_data/prompts_from_papers/README.md @@ -0,0 +1,5 @@ +--- +library_name: prompt-templates +tags: +- prompts +--- diff --git a/tests/test_data/prompts_from_papers/judgebench-vanilla-prompt.yaml b/tests/test_data/prompts_from_papers/judgebench-vanilla-prompt.yaml new file mode 100644 index 0000000..26b0459 --- /dev/null +++ b/tests/test_data/prompts_from_papers/judgebench-vanilla-prompt.yaml @@ -0,0 +1,29 @@ +prompt: + template: + - role: user + content: |- + You are a helpful assistant in evaluating the quality of the outputs for a given instruction. + Your goal is to select the best output for the given instruction. + Select the Output (a) or Output (b) that is better for the given instruction. The two outputs are generated by two different AI chatbots respectively. + Do NOT provide any explanation for your choice. + Do NOT say both / neither are good. + You should answer using ONLY “Output (a)” or “Output (b)”. Do NOT output any other words. + # Instruction: + {{question}} + # Output (a): + {{response_a}} + # Output (b): + {{response_b}} + # Which is better, Output (a) or Output (b)? Your response should be either “Output (a)” or “Output (b)”: + + template_variables: + - question + - response_a + - response_b + + metadata: + source: Appendix A.5, https://arxiv.org/pdf/2410.12784 + description: This prompt is used in the JudgeBench paper to evaluate the quality of the outputs for a given instruction. + + client_parameters: + temperature: 0 diff --git a/tests/test_data/sync_test_data.py b/tests/test_data/sync_test_data.py index 41b2f49..4bbd975 100644 --- a/tests/test_data/sync_test_data.py +++ b/tests/test_data/sync_test_data.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Set +import yaml from dotenv import load_dotenv from huggingface_hub import HfApi @@ -44,11 +45,14 @@ def sync_test_files(): dir_name = directory.name repo_id = f"MoritzLaurer/{dir_name}" - print(f"\nProcessing directory: {dir_name}") + # Determine repo_type based on README.md tags + repo_type = "model" if has_model_prompts_tag(directory) else "dataset" + + print(f"\nProcessing directory: {dir_name} as {repo_type}") # Ensure repository exists try: - api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True, token=token) + api.create_repo(repo_id=repo_id, repo_type=repo_type, exist_ok=True, token=token) except Exception as e: print(f"Note about repo creation (can usually be ignored): {e}") continue @@ -66,7 +70,7 @@ def sync_test_files(): # Get existing hub files to check for deletions hub_files: Set[str] = { f - for f in api.list_repo_files(repo_id=repo_id, repo_type="model", token=token) + for f in api.list_repo_files(repo_id=repo_id, repo_type=repo_type, token=token) if f.endswith((".yaml", ".yml", ".json", ".md", ".py")) } @@ -78,7 +82,7 @@ def sync_test_files(): path_or_fileobj=str(local_file), path_in_repo=local_file.name, repo_id=repo_id, - repo_type="model", + repo_type=repo_type, token=token, ) hub_files.discard(local_file.name) @@ -86,14 +90,14 @@ def sync_test_files(): print(f"Error uploading {local_file.name}: {e}") continue - # Delete files that exist on hub but not locally + # Delete files on hub that exist on hub but not locally for obsolete_file in hub_files: print(f"Deleting {obsolete_file} from hub (no longer exists locally)...") try: api.delete_file( path_in_repo=obsolete_file, repo_id=repo_id, - repo_type="model", + repo_type=repo_type, token=token, ) except Exception as e: @@ -110,5 +114,30 @@ def sync_test_files(): continue +def has_model_prompts_tag(directory: Path) -> bool: + """Check if README.md in directory has model-prompts tag. + This enables us to sync the test data to the Hub as a model repo as opposed to the default dataset repo if the README.md contains the tag.""" + readme_path = directory / "README.md" + if not readme_path.exists(): + return False + + try: + # Read the README file + content = readme_path.read_text() + + # Check if there's a YAML front matter + if not content.startswith("---"): + return False + + # Extract YAML front matter + yaml_content = content.split("---")[1] + metadata = yaml.safe_load(yaml_content) + + # Check if tags exist and if model-prompts is in tags + return isinstance(metadata.get("tags"), list) and "model-prompts" in metadata["tags"] + except Exception: + return False + + if __name__ == "__main__": sync_test_files()