From 5284d1e7227247296ecec1da25fe3e020689986d Mon Sep 17 00:00:00 2001 From: dafnapension Date: Fri, 20 Dec 2024 20:21:45 +0200 Subject: [PATCH] StandardRecipe->DatasetRecipe and standard_recipe->dataset_recipe Signed-off-by: dafnapension --- examples/evaluate_benchmark.py | 18 ++--- ...e_text_to_text_with_different_templates.py | 8 +- ...bustness_testing_for_vision_text_models.py | 4 +- performance/card_profiler.py | 12 +-- prepare/benchmarks/glue.py | 18 ++--- prepare/recipes/bluebench.py | 4 +- src/unitxt/api.py | 14 ++-- src/unitxt/benchmark.py | 4 +- src/unitxt/catalog/benchmarks/glue.json | 16 ++-- .../bluebench/bias/safety_bbq_age.json | 2 +- .../bias/safety_bbq_disability_status.json | 2 +- .../bias/safety_bbq_gender_identity.json | 2 +- .../bias/safety_bbq_nationality.json | 2 +- .../bias/safety_bbq_physical_appearance.json | 2 +- .../bias/safety_bbq_race_ethnicity.json | 2 +- .../bias/safety_bbq_race_x_gender.json | 2 +- .../bluebench/bias/safety_bbq_race_x_ses.json | 2 +- .../bluebench/bias/safety_bbq_religion.json | 2 +- .../bluebench/bias/safety_bbq_ses.json | 2 +- .../bias/safety_bbq_sexual_orientation.json | 2 +- ...neration_english_gpt_4_0314_reference.json | 2 +- .../universal_ner_en_ewt.json | 2 +- .../bluebench/knowledge/mmlu_pro_biology.json | 2 +- .../knowledge/mmlu_pro_business.json | 2 +- .../knowledge/mmlu_pro_chemistry.json | 2 +- .../knowledge/mmlu_pro_computer_science.json | 2 +- .../knowledge/mmlu_pro_economics.json | 2 +- .../knowledge/mmlu_pro_engineering.json | 2 +- .../bluebench/knowledge/mmlu_pro_health.json | 2 +- .../bluebench/knowledge/mmlu_pro_history.json | 2 +- .../bluebench/knowledge/mmlu_pro_law.json | 2 +- .../bluebench/knowledge/mmlu_pro_math.json | 2 +- .../bluebench/knowledge/mmlu_pro_other.json | 2 +- .../knowledge/mmlu_pro_philosophy.json | 2 +- .../bluebench/knowledge/mmlu_pro_physics.json | 2 +- .../knowledge/mmlu_pro_psychology.json | 2 +- .../legal/legalbench_abercrombie.json | 2 +- .../legal/legalbench_corporate_lobbying.json | 2 +- ...galbench_function_of_decision_section.json | 2 +- ...h_international_citizenship_questions.json | 2 +- .../bluebench/legal/legalbench_proa.json | 2 +- .../20_newsgroups_short.json | 2 +- .../product_help/cfpb_product_2023.json | 2 +- .../product_help/cfpb_product_watsonx.json | 2 +- .../recipes/bluebench/qa_finance/fin_qa.json | 2 +- .../rag_response_generation_clapnq.json | 2 +- .../bluebench/reasoning/hellaswag.json | 2 +- .../bluebench/reasoning/openbook_qa.json | 2 +- .../recipes/bluebench/safety/attaq_500.json | 2 +- ...llsum_document_filtered_to_6000_chars.json | 2 +- .../tldr_document_filtered_to_6000_chars.json | 2 +- .../translation/mt_flores_101_ara_eng.json | 2 +- .../translation/mt_flores_101_deu_eng.json | 2 +- .../translation/mt_flores_101_eng_ara.json | 2 +- .../translation/mt_flores_101_eng_deu.json | 2 +- .../translation/mt_flores_101_eng_fra.json | 2 +- .../translation/mt_flores_101_eng_kor.json | 2 +- .../translation/mt_flores_101_eng_por.json | 2 +- .../translation/mt_flores_101_eng_ron.json | 2 +- .../translation/mt_flores_101_eng_spa.json | 2 +- .../translation/mt_flores_101_fra_eng.json | 2 +- .../translation/mt_flores_101_jpn_eng.json | 2 +- .../translation/mt_flores_101_kor_eng.json | 2 +- .../translation/mt_flores_101_por_eng.json | 2 +- .../translation/mt_flores_101_ron_eng.json | 2 +- .../translation/mt_flores_101_spa_eng.json | 2 +- src/unitxt/settings_utils.py | 2 +- src/unitxt/standard.py | 2 +- src/unitxt/test_utils/card.py | 10 +-- src/unitxt/ui/ui_utils.py | 6 +- tests/library/test_artifact.py | 2 +- tests/library/test_artifact_recovery.py | 6 +- tests/library/test_benchmark.py | 12 +-- tests/library/test_formats.py | 4 +- tests/library/test_fusion.py | 10 +-- tests/library/test_recipe.py | 78 +++++++++---------- ...ompare_unitxt_datasets_between_versions.py | 4 +- 77 files changed, 174 insertions(+), 174 deletions(-) diff --git a/examples/evaluate_benchmark.py b/examples/evaluate_benchmark.py index fd09906177..61ce8671e7 100644 --- a/examples/evaluate_benchmark.py +++ b/examples/evaluate_benchmark.py @@ -3,7 +3,7 @@ from unitxt.inference import ( CrossProviderInferenceEngine, ) -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe from unitxt.text_utils import print_dict benchmark = Benchmark( @@ -11,33 +11,33 @@ max_samples_per_subset=5, loader_limit=300, subsets={ - "cola": StandardRecipe( + "cola": DatasetRecipe( card="cards.cola", template="templates.classification.multi_class.instruction", ), - "mnli": StandardRecipe( + "mnli": DatasetRecipe( card="cards.mnli", template="templates.classification.multi_class.relation.default", ), - "mrpc": StandardRecipe( + "mrpc": DatasetRecipe( card="cards.mrpc", template="templates.classification.multi_class.relation.default", ), - "qnli": StandardRecipe( + "qnli": DatasetRecipe( card="cards.qnli", template="templates.classification.multi_class.relation.default", ), - "rte": StandardRecipe( + "rte": DatasetRecipe( card="cards.rte", template="templates.classification.multi_class.relation.default", ), - "sst2": StandardRecipe( + "sst2": DatasetRecipe( card="cards.sst2", template="templates.classification.multi_class.title" ), - "stsb": StandardRecipe( + "stsb": DatasetRecipe( card="cards.stsb", template="templates.regression.two_texts.title" ), - "wnli": StandardRecipe( + "wnli": DatasetRecipe( card="cards.wnli", template="templates.classification.multi_class.relation.default", ), diff --git a/examples/evaluate_image_text_to_text_with_different_templates.py b/examples/evaluate_image_text_to_text_with_different_templates.py index 6e2d132d4c..e07f6ff14d 100644 --- a/examples/evaluate_image_text_to_text_with_different_templates.py +++ b/examples/evaluate_image_text_to_text_with_different_templates.py @@ -5,7 +5,7 @@ LMMSEvalInferenceEngine, ) from unitxt.logging_utils import get_logger -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe logger = get_logger() @@ -16,17 +16,17 @@ dataset = Benchmark( subsets={ - "capitals": StandardRecipe( + "capitals": DatasetRecipe( card=card, template="templates.qa.multiple_choice.with_context.lmms_eval[enumerator=capitals]", loader_limit=20, ), - "lowercase": StandardRecipe( + "lowercase": DatasetRecipe( card=card, template="templates.qa.multiple_choice.with_context.lmms_eval[enumerator=lowercase]", loader_limit=20, ), - "capitals-greyscale": StandardRecipe( + "capitals-greyscale": DatasetRecipe( card=card, template="templates.qa.multiple_choice.with_context.lmms_eval[enumerator=capitals]", loader_limit=20, diff --git a/examples/robustness_testing_for_vision_text_models.py b/examples/robustness_testing_for_vision_text_models.py index 585b647cdf..23aa99c7bc 100644 --- a/examples/robustness_testing_for_vision_text_models.py +++ b/examples/robustness_testing_for_vision_text_models.py @@ -5,7 +5,7 @@ LMMSEvalInferenceEngine, ) from unitxt.logging_utils import get_logger -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe logger = get_logger() @@ -16,7 +16,7 @@ for card in ["cards.seed_bench", "cards.ai2d"]: for enumerator in ["capitals", "lowercase"]: for augmentor in [None, "augmentors.image.white_noise"]: - subsets[f"{card} {enumerator} {augmentor}"] = StandardRecipe( + subsets[f"{card} {enumerator} {augmentor}"] = DatasetRecipe( card=card, template=f"templates.qa.multiple_choice.with_context.lmms_eval[enumerator={enumerator}]", loader_limit=100, diff --git a/performance/card_profiler.py b/performance/card_profiler.py index 4e79b8dbf6..03325ac0ef 100644 --- a/performance/card_profiler.py +++ b/performance/card_profiler.py @@ -10,7 +10,7 @@ from unitxt.artifact import fetch_artifact from unitxt.logging_utils import get_logger from unitxt.settings_utils import get_settings -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe from unitxt.stream import MultiStream from unitxt.templates import TemplatesDict, TemplatesList @@ -49,28 +49,28 @@ class CardProfiler: You will find the total time of each step, accumulated over all cards in the benchmark. """ - def profiler_instantiate_recipe(self, **kwargs) -> StandardRecipe: + def profiler_instantiate_recipe(self, **kwargs) -> DatasetRecipe: return load_recipe(**kwargs) - def profiler_load_by_recipe(self, recipe: StandardRecipe) -> MultiStream: + def profiler_load_by_recipe(self, recipe: DatasetRecipe) -> MultiStream: ms = recipe.loading.process() assert isinstance(ms, MultiStream) return ms def profiler_metadata_and_standardization( - self, ms: MultiStream, recipe: StandardRecipe + self, ms: MultiStream, recipe: DatasetRecipe ) -> MultiStream: ms = recipe.metadata.process(ms) return recipe.standardization.process(ms) def profiler_processing_demos_metadata( - self, ms: MultiStream, recipe: StandardRecipe + self, ms: MultiStream, recipe: DatasetRecipe ) -> MultiStream: ms = recipe.processing.process(ms) return recipe.metadata.process(ms) def profiler_verbalize_and_finalize( - self, ms: MultiStream, recipe: StandardRecipe + self, ms: MultiStream, recipe: DatasetRecipe ) -> MultiStream: ms = recipe.verbalization.process(ms) return recipe.finalize.process(ms) diff --git a/prepare/benchmarks/glue.py b/prepare/benchmarks/glue.py index 34dd521522..328002553c 100644 --- a/prepare/benchmarks/glue.py +++ b/prepare/benchmarks/glue.py @@ -1,36 +1,36 @@ from unitxt.benchmark import Benchmark from unitxt.catalog import add_to_catalog -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe benchmark = Benchmark( subsets={ - "cola": StandardRecipe( + "cola": DatasetRecipe( card="cards.cola", template="templates.classification.multi_class.instruction", ), - "mnli": StandardRecipe( + "mnli": DatasetRecipe( card="cards.mnli", template="templates.classification.multi_class.relation.default", ), - "mrpc": StandardRecipe( + "mrpc": DatasetRecipe( card="cards.mrpc", template="templates.classification.multi_class.relation.default", ), - "qnli": StandardRecipe( + "qnli": DatasetRecipe( card="cards.qnli", template="templates.classification.multi_class.relation.default", ), - "rte": StandardRecipe( + "rte": DatasetRecipe( card="cards.rte", template="templates.classification.multi_class.relation.default", ), - "sst2": StandardRecipe( + "sst2": DatasetRecipe( card="cards.sst2", template="templates.classification.multi_class.title" ), - "stsb": StandardRecipe( + "stsb": DatasetRecipe( card="cards.stsb", template="templates.regression.two_texts.title" ), - "wnli": StandardRecipe( + "wnli": DatasetRecipe( card="cards.wnli", template="templates.classification.multi_class.relation.default", ), diff --git a/prepare/recipes/bluebench.py b/prepare/recipes/bluebench.py index 3368ea6374..eb63bbfc6d 100644 --- a/prepare/recipes/bluebench.py +++ b/prepare/recipes/bluebench.py @@ -1,5 +1,5 @@ from unitxt import add_to_catalog -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe subsets = { # the key must appear in the card name "cards.legalbench": [ @@ -82,7 +82,7 @@ def prepare_recipe(default_args, specific_args): if "template" in recipe and "template_card_index" in recipe: del recipe["template_card_index"] - return StandardRecipe(**recipe, format="formats.chat_api") + return DatasetRecipe(**recipe, format="formats.chat_api") ### Reasoning diff --git a/src/unitxt/api.py b/src/unitxt/api.py index 53b53a4637..40be96ee58 100644 --- a/src/unitxt/api.py +++ b/src/unitxt/api.py @@ -14,7 +14,7 @@ from .operator import SourceOperator from .schema import UNITXT_DATASET_SCHEMA, loads_instance from .settings_utils import get_constants, get_settings -from .standard import StandardRecipe +from .standard import DatasetRecipe logger = get_logger() constants = get_constants() @@ -30,7 +30,7 @@ def load(source: Union[SourceOperator, str]): return source().to_dataset() -def _get_recipe_from_query(dataset_query: str) -> StandardRecipe: +def _get_recipe_from_query(dataset_query: str) -> DatasetRecipe: dataset_query = dataset_query.replace("sys_prompt", "instruction") try: dataset_stream, _ = fetch_artifact(dataset_query) @@ -39,14 +39,14 @@ def _get_recipe_from_query(dataset_query: str) -> StandardRecipe: return dataset_stream -def _get_recipe_from_dict(dataset_params: Dict[str, Any]) -> StandardRecipe: - recipe_attributes = list(StandardRecipe.__dict__["__fields__"].keys()) +def _get_recipe_from_dict(dataset_params: Dict[str, Any]) -> DatasetRecipe: + recipe_attributes = list(DatasetRecipe.__dict__["__fields__"].keys()) for param in dataset_params.keys(): assert param in recipe_attributes, ( f"The parameter '{param}' is not an attribute of the 'StandardRecipe' class. " f"Please check if the name is correct. The available attributes are: '{recipe_attributes}'." ) - return StandardRecipe(**dataset_params) + return DatasetRecipe(**dataset_params) def _verify_dataset_args(dataset_query: Optional[str] = None, dataset_args=None): @@ -71,8 +71,8 @@ def _verify_dataset_args(dataset_query: Optional[str] = None, dataset_args=None) ) -def load_recipe(dataset_query: Optional[str] = None, **kwargs) -> StandardRecipe: - if isinstance(dataset_query, StandardRecipe): +def load_recipe(dataset_query: Optional[str] = None, **kwargs) -> DatasetRecipe: + if isinstance(dataset_query, DatasetRecipe): return dataset_query _verify_dataset_args(dataset_query, kwargs) diff --git a/src/unitxt/benchmark.py b/src/unitxt/benchmark.py index b1750eacb8..a3f4562d83 100644 --- a/src/unitxt/benchmark.py +++ b/src/unitxt/benchmark.py @@ -5,7 +5,7 @@ from .formats import Format from .fusion import FixedFusion, WeightedFusion from .operator import SourceOperator -from .standard import StandardRecipe +from .standard import DatasetRecipe from .stream import MultiStream from .system_prompts import SystemPrompt @@ -22,7 +22,7 @@ def reset(self): class Benchmark(BaseBenchmark): - subsets: Dict[str, Union[StandardRecipe, BaseBenchmark]] + subsets: Dict[str, Union[DatasetRecipe, BaseBenchmark]] max_total_samples: int = None max_samples_per_subset: int = None diff --git a/src/unitxt/catalog/benchmarks/glue.json b/src/unitxt/catalog/benchmarks/glue.json index f720ffdd05..10fd726603 100644 --- a/src/unitxt/catalog/benchmarks/glue.json +++ b/src/unitxt/catalog/benchmarks/glue.json @@ -2,42 +2,42 @@ "__type__": "benchmark", "subsets": { "cola": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.cola", "template": "templates.classification.multi_class.instruction" }, "mnli": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.mnli", "template": "templates.classification.multi_class.relation.default" }, "mrpc": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.mrpc", "template": "templates.classification.multi_class.relation.default" }, "qnli": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.qnli", "template": "templates.classification.multi_class.relation.default" }, "rte": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.rte", "template": "templates.classification.multi_class.relation.default" }, "sst2": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.sst2", "template": "templates.classification.multi_class.title" }, "stsb": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.stsb", "template": "templates.regression.two_texts.title" }, "wnli": { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.wnli", "template": "templates.classification.multi_class.relation.default" } diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_age.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_age.json index 5e68da8756..0bff65bac5 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_age.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_age.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_disability_status.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_disability_status.json index d173106dc6..ebfbaa92e7 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_disability_status.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_disability_status.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_gender_identity.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_gender_identity.json index f3fcc9208d..f23da00b29 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_gender_identity.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_gender_identity.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_nationality.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_nationality.json index 6a3eeaf00b..78cf913f10 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_nationality.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_nationality.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_physical_appearance.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_physical_appearance.json index d6bebf0925..aa19e829d4 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_physical_appearance.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_physical_appearance.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_ethnicity.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_ethnicity.json index 89e23a3d3c..e7123fb7b2 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_ethnicity.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_ethnicity.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_gender.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_gender.json index 508c8bf374..be5977efc9 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_gender.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_gender.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_ses.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_ses.json index 6ffc6e24f4..0eac47994b 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_ses.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_x_ses.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_religion.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_religion.json index 520f062480..5cac2697b5 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_religion.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_religion.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_ses.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_ses.json index e3700dd1d4..6309539561 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_ses.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_ses.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_sexual_orientation.json b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_sexual_orientation.json index 1863d20c81..ac5cc0a9cd 100644 --- a/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_sexual_orientation.json +++ b/src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_sexual_orientation.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/chatbot_abilities/arena_hard_generation_english_gpt_4_0314_reference.json b/src/unitxt/catalog/recipes/bluebench/chatbot_abilities/arena_hard_generation_english_gpt_4_0314_reference.json index 8259767cab..595ad5c404 100644 --- a/src/unitxt/catalog/recipes/bluebench/chatbot_abilities/arena_hard_generation_english_gpt_4_0314_reference.json +++ b/src/unitxt/catalog/recipes/bluebench/chatbot_abilities/arena_hard_generation_english_gpt_4_0314_reference.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 0, "num_demos": 0, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/entity_extraction/universal_ner_en_ewt.json b/src/unitxt/catalog/recipes/bluebench/entity_extraction/universal_ner_en_ewt.json index 4dab5b9be2..662dba9b3c 100644 --- a/src/unitxt/catalog/recipes/bluebench/entity_extraction/universal_ner_en_ewt.json +++ b/src/unitxt/catalog/recipes/bluebench/entity_extraction/universal_ner_en_ewt.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 10000, "num_demos": 5, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_biology.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_biology.json index 27908d8581..a6e1f335a7 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_biology.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_biology.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_business.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_business.json index dfeae0c7bb..c8ab7ce711 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_business.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_business.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_chemistry.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_chemistry.json index 828444520e..6d2379095f 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_chemistry.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_chemistry.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_computer_science.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_computer_science.json index 5b2c35fffc..7d9ab632d9 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_computer_science.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_computer_science.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_economics.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_economics.json index a9ddc4c927..d9534bc91a 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_economics.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_economics.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_engineering.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_engineering.json index 7cb3dfeaea..7199f5b1ff 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_engineering.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_engineering.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_health.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_health.json index ec54ce638b..fddb9854c9 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_health.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_health.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_history.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_history.json index 56e7143b82..9a6132008e 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_history.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_history.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_law.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_law.json index 7c053421f9..2228a1852a 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_law.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_law.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_math.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_math.json index befdf9defc..1aaa37e694 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_math.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_math.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_other.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_other.json index a050d42592..1bcbac2706 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_other.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_other.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_philosophy.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_philosophy.json index a90ece0cf6..1b9d4f4e3e 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_philosophy.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_philosophy.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_physics.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_physics.json index f3d69ebbc1..15cbfd85ae 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_physics.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_physics.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_psychology.json b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_psychology.json index 97c4b2b4aa..daf77596bf 100644 --- a/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_psychology.json +++ b/src/unitxt/catalog/recipes/bluebench/knowledge/mmlu_pro_psychology.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 20, "num_demos": 5, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_abercrombie.json b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_abercrombie.json index 98461e10b9..8a9ba754a5 100644 --- a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_abercrombie.json +++ b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_abercrombie.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 10, "num_demos": 1, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_corporate_lobbying.json b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_corporate_lobbying.json index b03a43787d..cf1248e1de 100644 --- a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_corporate_lobbying.json +++ b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_corporate_lobbying.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 10, "num_demos": 1, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_function_of_decision_section.json b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_function_of_decision_section.json index e3eab4ac9b..b9fb501388 100644 --- a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_function_of_decision_section.json +++ b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_function_of_decision_section.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 10, "num_demos": 1, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_international_citizenship_questions.json b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_international_citizenship_questions.json index ce0711eee6..ff74b89d2b 100644 --- a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_international_citizenship_questions.json +++ b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_international_citizenship_questions.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 10, "num_demos": 1, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_proa.json b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_proa.json index 3a40cae4b7..56277304c1 100644 --- a/src/unitxt/catalog/recipes/bluebench/legal/legalbench_proa.json +++ b/src/unitxt/catalog/recipes/bluebench/legal/legalbench_proa.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 10, "num_demos": 1, "demos_taken_from": "test", diff --git a/src/unitxt/catalog/recipes/bluebench/news_classification/20_newsgroups_short.json b/src/unitxt/catalog/recipes/bluebench/news_classification/20_newsgroups_short.json index 3694028d97..c9ab0a4975 100644 --- a/src/unitxt/catalog/recipes/bluebench/news_classification/20_newsgroups_short.json +++ b/src/unitxt/catalog/recipes/bluebench/news_classification/20_newsgroups_short.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 1, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_2023.json b/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_2023.json index 722badc306..029a89e4d0 100644 --- a/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_2023.json +++ b/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_2023.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_watsonx.json b/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_watsonx.json index 106fba290c..4266568ad9 100644 --- a/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_watsonx.json +++ b/src/unitxt/catalog/recipes/bluebench/product_help/cfpb_product_watsonx.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/qa_finance/fin_qa.json b/src/unitxt/catalog/recipes/bluebench/qa_finance/fin_qa.json index dafd58b260..7b5f125aae 100644 --- a/src/unitxt/catalog/recipes/bluebench/qa_finance/fin_qa.json +++ b/src/unitxt/catalog/recipes/bluebench/qa_finance/fin_qa.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 1, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/rag_general/rag_response_generation_clapnq.json b/src/unitxt/catalog/recipes/bluebench/rag_general/rag_response_generation_clapnq.json index 576811d64d..ce8916eadb 100644 --- a/src/unitxt/catalog/recipes/bluebench/rag_general/rag_response_generation_clapnq.json +++ b/src/unitxt/catalog/recipes/bluebench/rag_general/rag_response_generation_clapnq.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 1, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/reasoning/hellaswag.json b/src/unitxt/catalog/recipes/bluebench/reasoning/hellaswag.json index a8720501e2..8ebb6c983a 100644 --- a/src/unitxt/catalog/recipes/bluebench/reasoning/hellaswag.json +++ b/src/unitxt/catalog/recipes/bluebench/reasoning/hellaswag.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/reasoning/openbook_qa.json b/src/unitxt/catalog/recipes/bluebench/reasoning/openbook_qa.json index a0afa2058e..545bd4ac12 100644 --- a/src/unitxt/catalog/recipes/bluebench/reasoning/openbook_qa.json +++ b/src/unitxt/catalog/recipes/bluebench/reasoning/openbook_qa.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/safety/attaq_500.json b/src/unitxt/catalog/recipes/bluebench/safety/attaq_500.json index ebfe9cb615..ffb7d005a3 100644 --- a/src/unitxt/catalog/recipes/bluebench/safety/attaq_500.json +++ b/src/unitxt/catalog/recipes/bluebench/safety/attaq_500.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 0, "num_demos": 0, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/summarization/billsum_document_filtered_to_6000_chars.json b/src/unitxt/catalog/recipes/bluebench/summarization/billsum_document_filtered_to_6000_chars.json index 370b2a5017..e8386de0e9 100644 --- a/src/unitxt/catalog/recipes/bluebench/summarization/billsum_document_filtered_to_6000_chars.json +++ b/src/unitxt/catalog/recipes/bluebench/summarization/billsum_document_filtered_to_6000_chars.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 0, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/summarization/tldr_document_filtered_to_6000_chars.json b/src/unitxt/catalog/recipes/bluebench/summarization/tldr_document_filtered_to_6000_chars.json index 53ea41cd14..cf7cfb54c6 100644 --- a/src/unitxt/catalog/recipes/bluebench/summarization/tldr_document_filtered_to_6000_chars.json +++ b/src/unitxt/catalog/recipes/bluebench/summarization/tldr_document_filtered_to_6000_chars.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 0, "demos_taken_from": "train", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ara_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ara_eng.json index 56e2c4280c..d2488c7143 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ara_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ara_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_deu_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_deu_eng.json index dfac2ad756..eb2884d696 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_deu_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_deu_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ara.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ara.json index f8bec1ec6b..19d16e5b67 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ara.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ara.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_deu.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_deu.json index c5c9f590fc..58bca3fcb0 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_deu.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_deu.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_fra.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_fra.json index b56f10b4d1..49dc5df5d0 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_fra.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_fra.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_kor.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_kor.json index ae8a136df4..c0a296c09e 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_kor.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_kor.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_por.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_por.json index ac3c3edbcf..e059f9624c 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_por.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_por.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ron.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ron.json index 2290bc3592..d91dfdab5a 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ron.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_ron.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_spa.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_spa.json index a5e359bf56..e12bcefba2 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_spa.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_eng_spa.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_fra_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_fra_eng.json index 840eda4505..6006dc2002 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_fra_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_fra_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_jpn_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_jpn_eng.json index 6ab8c49028..f19c6de7d9 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_jpn_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_jpn_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_kor_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_kor_eng.json index 9aa2ba1756..72338d3f0d 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_kor_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_kor_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_por_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_por_eng.json index 5d251a3296..8dd3cf2553 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_por_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_por_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ron_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ron_eng.json index 324565f134..26e04f2ce6 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ron_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_ron_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_spa_eng.json b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_spa_eng.json index 790b55448d..8bc58a689a 100644 --- a/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_spa_eng.json +++ b/src/unitxt/catalog/recipes/bluebench/translation/mt_flores_101_spa_eng.json @@ -1,5 +1,5 @@ { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "demos_pool_size": 100, "num_demos": 5, "demos_taken_from": "validation", diff --git a/src/unitxt/settings_utils.py b/src/unitxt/settings_utils.py index ae3e00a9f6..8e35afe082 100644 --- a/src/unitxt/settings_utils.py +++ b/src/unitxt/settings_utils.py @@ -138,7 +138,7 @@ def __getattr__(self, key): settings.max_log_message_size = (int, 100000) settings.catalogs = None settings.artifactories = None - settings.default_recipe = "standard_recipe" + settings.default_recipe = "dataset_recipe" settings.default_verbosity = "info" settings.use_eager_execution = False settings.remote_metrics = [] diff --git a/src/unitxt/standard.py b/src/unitxt/standard.py index e4460200c7..eadd4ee407 100644 --- a/src/unitxt/standard.py +++ b/src/unitxt/standard.py @@ -682,5 +682,5 @@ class StandardRecipeWithIndexes(BaseRecipe): pass -class StandardRecipe(StandardRecipeWithIndexes): +class DatasetRecipe(StandardRecipeWithIndexes): pass diff --git a/src/unitxt/test_utils/card.py b/src/unitxt/test_utils/card.py index 9b09f7460f..26647adb2c 100644 --- a/src/unitxt/test_utils/card.py +++ b/src/unitxt/test_utils/card.py @@ -9,7 +9,7 @@ from ..logging_utils import get_logger from ..metric import _compute from ..settings_utils import get_settings -from ..standard import StandardRecipe +from ..standard import DatasetRecipe from ..text_utils import construct_dict_str from ..utils import deep_copy @@ -46,10 +46,10 @@ def test_loading_from_catalog(card): ), "Card loaded is not equal to card stored" -def load_examples_from_standard_recipe(card, template_card_index, debug, **kwargs): +def load_examples_from_dataset_recipe(card, template_card_index, debug, **kwargs): if settings.test_card_disable: logger.info( - "load_examples_from_standard_recipe() functionality is disabled because unitxt.settings.test_card_disable=True or UNITXT_TEST_CARD_DISABLE environment variable is set" + "load_examples_from_dataset_recipe() functionality is disabled because unitxt.settings.test_card_disable=True or UNITXT_TEST_CARD_DISABLE environment variable is set" ) return None @@ -58,7 +58,7 @@ def load_examples_from_standard_recipe(card, template_card_index, debug, **kwarg kwargs["loader_limit"] = 30 kwargs["template_card_index"] = template_card_index - recipe = StandardRecipe(card=card, **kwargs) + recipe = DatasetRecipe(card=card, **kwargs) logger.info(f"Using these card recipe parameters: {kwargs}") if debug: @@ -292,7 +292,7 @@ def test_card( template_card_indices = range(len(card.templates)) for template_card_index in template_card_indices: - examples = load_examples_from_standard_recipe( + examples = load_examples_from_dataset_recipe( card, template_card_index=template_card_index, debug=debug, **kwargs ) if test_exact_match_score_when_predictions_equal_references: diff --git a/src/unitxt/ui/ui_utils.py b/src/unitxt/ui/ui_utils.py index 4055d34828..9db3753f5d 100644 --- a/src/unitxt/ui/ui_utils.py +++ b/src/unitxt/ui/ui_utils.py @@ -6,7 +6,7 @@ from ..api import evaluate from ..logging_utils import get_logger -from ..standard import StandardRecipe +from ..standard import DatasetRecipe from ..text_utils import print_dict from . import settings as config from .load_catalog_data import get_catalog_items, load_cards_data @@ -122,7 +122,7 @@ def collect(dataset, split, n): def build_prompt(prompt_args): - recipe = StandardRecipe(**prompt_args) + recipe = DatasetRecipe(**prompt_args) logger.info("loading args:") print_dict(prompt_args) dataset = recipe() @@ -134,7 +134,7 @@ def build_prompt(prompt_args): prompt_args["demos_taken_from"] = "test" logger.info("trying againg with loading args:") print_dict(prompt_args) - recipe = StandardRecipe(**prompt_args) + recipe = DatasetRecipe(**prompt_args) dataset = recipe() prompt_list = collect(dataset, "test", config.PROMPT_SAMPLE_SIZE) return prompt_list diff --git a/tests/library/test_artifact.py b/tests/library/test_artifact.py index f3f7da91c1..f587d0be05 100644 --- a/tests/library/test_artifact.py +++ b/tests/library/test_artifact.py @@ -572,7 +572,7 @@ def test_artifact_link_in_recursive_load(self): def test_artifact_is_not_saving_if_artifact_has_changed(self): with self.assertRaises(UnitxtError) as e: args = { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.sst2", "template_card_index": 0, "demos_pool_size": 100, diff --git a/tests/library/test_artifact_recovery.py b/tests/library/test_artifact_recovery.py index 376baeb9f6..c074ad486b 100644 --- a/tests/library/test_artifact_recovery.py +++ b/tests/library/test_artifact_recovery.py @@ -13,7 +13,7 @@ class TestArtifactRecovery(UnitxtTestCase): def test_correct_artifact_recovery(self): args = { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.sst2", "template_card_index": 0, "demos_pool_size": 100, @@ -24,7 +24,7 @@ def test_correct_artifact_recovery(self): def test_correct_artifact_recovery_with_overwrite(self): args = { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.sst2", "template_card_index": 0, "demos_pool_size": 100, @@ -45,7 +45,7 @@ def test_bad_artifact_recovery_missing_type(self): def test_bad_artifact_recovery_bad_type(self): args = { - "__type__": "standard_recipe", + "__type__": "dataset_recipe", "card": "cards.sst2", "template_card_index": 1000, "demos_pool_size": 100, diff --git a/tests/library/test_benchmark.py b/tests/library/test_benchmark.py index 9a6d0056ad..3b9a2e4d3b 100644 --- a/tests/library/test_benchmark.py +++ b/tests/library/test_benchmark.py @@ -1,5 +1,5 @@ from unitxt.benchmark import Benchmark -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe from tests.utils import UnitxtTestCase @@ -11,11 +11,11 @@ def test_benchmark(self): max_samples_per_subset=2, loader_limit=300, subsets={ - "cola": StandardRecipe( + "cola": DatasetRecipe( card="cards.cola", template="templates.classification.multi_class.instruction", ), - "wnli": StandardRecipe( + "wnli": DatasetRecipe( card="cards.wnli", template="templates.classification.multi_class.relation.default", ), @@ -109,18 +109,18 @@ def test_benchmark_format_trickling(self): max_samples_per_subset=1, loader_limit=300, subsets={ - "cola": StandardRecipe( + "cola": DatasetRecipe( card="cards.cola", template="templates.classification.multi_class.instruction", ), - "wnli": StandardRecipe( + "wnli": DatasetRecipe( card="cards.wnli", format="formats.empty", template="templates.classification.multi_class.relation.default", ), }, ), - "wnli": StandardRecipe( + "wnli": DatasetRecipe( card="cards.wnli", template="templates.classification.multi_class.relation.default", ), diff --git a/tests/library/test_formats.py b/tests/library/test_formats.py index 598a2572d2..c433363e64 100644 --- a/tests/library/test_formats.py +++ b/tests/library/test_formats.py @@ -2,7 +2,7 @@ from unitxt.formats import ChatAPIFormat, HFSystemFormat, SystemFormat from unitxt.loaders import LoadFromDictionary from unitxt.settings_utils import get_constants -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe from unitxt.system_prompts import TextualSystemPrompt from unitxt.task import Task from unitxt.templates import InputOutputTemplate @@ -665,7 +665,7 @@ def test_system_format_with_demos_different_target_prefixes(self): templates=[template], ) - recipe = StandardRecipe( + recipe = DatasetRecipe( card=card, loader_limit=20, demos_pool_size=5, diff --git a/tests/library/test_fusion.py b/tests/library/test_fusion.py index a53be0622f..2b0c28454c 100644 --- a/tests/library/test_fusion.py +++ b/tests/library/test_fusion.py @@ -1,7 +1,7 @@ from unitxt.api import evaluate from unitxt.fusion import FixedFusion, WeightedFusion from unitxt.operators import IterableSource -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe from unitxt.test_utils.operators import check_operator from tests.utils import UnitxtTestCase, fillna, round_values @@ -309,22 +309,22 @@ def test_over_bounded_weighted_fusion(self): def test_end_to_end(self): dataset = WeightedFusion( subsets={ - "wnli": StandardRecipe( + "wnli": DatasetRecipe( card="cards.wnli", template="templates.classification.multi_class.relation.default", group_by=["template"], ), - "rte": StandardRecipe( + "rte": DatasetRecipe( card="cards.rte", template="templates.classification.multi_class.relation.default", ), "stsb": WeightedFusion( subsets={ - "regression": StandardRecipe( + "regression": DatasetRecipe( card="cards.stsb", template="templates.regression.two_texts.simple", ), - "classification": StandardRecipe( + "classification": DatasetRecipe( card="cards.stsb", template=[ "templates.regression.two_texts.similarity.flan", diff --git a/tests/library/test_recipe.py b/tests/library/test_recipe.py index a3225fae3b..3c4440daad 100644 --- a/tests/library/test_recipe.py +++ b/tests/library/test_recipe.py @@ -10,7 +10,7 @@ from unitxt.formats import SystemFormat from unitxt.loaders import LoadFromDictionary from unitxt.serializers import SingleTypeSerializer, TableSerializer -from unitxt.standard import StandardRecipe, StandardRecipeWithIndexes +from unitxt.standard import DatasetRecipe, StandardRecipeWithIndexes from unitxt.task import Task from unitxt.templates import InputOutputTemplate, TemplatesList from unitxt.text_utils import print_dict @@ -21,8 +21,8 @@ class TestRecipes(UnitxtTestCase): - def test_standard_recipe(self): - recipe = StandardRecipe( + def test_dataset_recipe(self): + recipe = DatasetRecipe( card="cards.wnli", template=InputOutputTemplate( input_format="{text_a}", @@ -59,8 +59,8 @@ def test_standard_recipe(self): ) break - def test_standard_recipe_with_catalog(self): - recipe = StandardRecipe( + def test_dataset_recipe_with_catalog(self): + recipe = DatasetRecipe( card="cards.mmlu.marketing", system_prompt="system_prompts.models.llama", template="templates.qa.multiple_choice.with_topic.lm_eval_harness", @@ -75,8 +75,8 @@ def test_standard_recipe_with_catalog(self): print_dict(instance) break - def test_standard_recipe_production_without_demos(self): - recipe = StandardRecipe( + def test_dataset_recipe_production_without_demos(self): + recipe = DatasetRecipe( card="cards.mmlu.marketing", system_prompt="system_prompts.models.llama", template="templates.qa.multiple_choice.with_topic.lm_eval_harness", @@ -107,8 +107,8 @@ def test_standard_recipe_production_without_demos(self): self.assertDictEqual(result, target) - def test_standard_recipe_production_consistency(self): - recipe = StandardRecipe( + def test_dataset_recipe_production_consistency(self): + recipe = DatasetRecipe( card="cards.mmlu.marketing", system_prompt="system_prompts.models.llama", template="templates.qa.multiple_choice.with_topic.lm_eval_harness", @@ -135,8 +135,8 @@ def test_standard_recipe_production_consistency(self): i2 = recipe.production_preprocess(recursive_copy(instances))[0] self.assertDictEqual(i1, i2) - def test_standard_recipe_production_with_demos(self): - recipe = StandardRecipe( + def test_dataset_recipe_production_with_demos(self): + recipe = DatasetRecipe( card="cards.mmlu.marketing", system_prompt="system_prompts.models.llama", template="templates.qa.multiple_choice.with_topic.lm_eval_harness", @@ -172,8 +172,8 @@ def test_standard_recipe_production_with_demos(self): self.assertDictEqual(result, target) self.assertDictEqual(target_task_data, result_task_data) - def test_standard_recipe_with_given_demos(self): - recipe = StandardRecipe( + def test_dataset_recipe_with_given_demos(self): + recipe = DatasetRecipe( card="cards.wnli", template_card_index=0, ) @@ -181,7 +181,7 @@ def test_standard_recipe_with_given_demos(self): for_demos = recipe.processing(for_demos) for_demos = recursive_copy(list(for_demos["validation"])) - recipe2 = StandardRecipe( + recipe2 = DatasetRecipe( card="cards.wnli", template_card_index=0, given_demos_pool=for_demos, @@ -190,7 +190,7 @@ def test_standard_recipe_with_given_demos(self): trains = list(recipe2()["train"]) assert "The entailment class is entailment" not in trains[0]["source"] - recipe3 = StandardRecipe( + recipe3 = DatasetRecipe( card="cards.wnli", template_card_index=0, given_demos_pool=for_demos, @@ -200,8 +200,8 @@ def test_standard_recipe_with_given_demos(self): trains = list(recipe3()["train"]) assert "The entailment class is entailment" in trains[0]["source"] - def test_standard_recipe_not_duplicating_demos_pool(self): - recipe = StandardRecipe( + def test_dataset_recipe_not_duplicating_demos_pool(self): + recipe = DatasetRecipe( card="cards.wnli", template_card_index=0, ) @@ -209,7 +209,7 @@ def test_standard_recipe_not_duplicating_demos_pool(self): for_demos = recipe.processing(for_demos) for_demos = recursive_copy(list(for_demos["validation"])) - recipe3 = StandardRecipe( + recipe3 = DatasetRecipe( card="cards.wnli", template_card_index=0, given_demos_pool=for_demos, @@ -237,8 +237,8 @@ def test_standard_recipe_not_duplicating_demos_pool(self): first_demo_of_second_instance["input_fields"]["text_a_type"], "hallelujah" ) - def test_standard_recipe_with_demoed_instances(self): - recipe = StandardRecipe( + def test_dataset_recipe_with_demoed_instances(self): + recipe = DatasetRecipe( card="cards.wnli", template_card_index=0, ) @@ -263,7 +263,7 @@ def test_standard_recipe_with_demoed_instances(self): a_standardized_input_instance ) - recipe2 = StandardRecipe( + recipe2 = DatasetRecipe( card="cards.wnli", template_card_index=0, demos_pool_size=3, @@ -277,7 +277,7 @@ def test_standard_recipe_with_demoed_instances(self): processed_input_instance["source"], ) - recipe3 = StandardRecipe( + recipe3 = DatasetRecipe( card="cards.wnli", template_card_index=0, demos_pool_size=3, @@ -293,8 +293,8 @@ def test_standard_recipe_with_demoed_instances(self): processed_input_instance["source"], ) - def test_standard_recipe_with_indexes_with_catalog(self): - recipe = StandardRecipe( + def test_dataset_recipe_with_indexes_with_catalog(self): + recipe = DatasetRecipe( card="cards.wnli", system_prompt="system_prompts.models.llama", template_card_index=0, @@ -309,8 +309,8 @@ def test_standard_recipe_with_indexes_with_catalog(self): print_dict(instance) break - def test_standard_recipe_with_demos_not_removed_from_data(self): - recipe = StandardRecipe( + def test_dataset_recipe_with_demos_not_removed_from_data(self): + recipe = DatasetRecipe( card="cards.wnli", template_card_index=0, demos_pool_size=100, @@ -547,7 +547,7 @@ def test_random_num_demos(self): self.assertEqual(len(lengths), 4) - def test_standard_recipe_with_balancer(self): + def test_dataset_recipe_with_balancer(self): recipe = StandardRecipeWithIndexes( card="cards.wnli", system_prompt="system_prompts.models.llama", @@ -565,7 +565,7 @@ def test_standard_recipe_with_balancer(self): self.assertEqual(counts["entailment"], counts["not entailment"]) - def test_standard_recipe_with_loader_limit(self): + def test_dataset_recipe_with_loader_limit(self): recipe = StandardRecipeWithIndexes( card="cards.wnli", system_prompt="system_prompts.models.llama", @@ -582,7 +582,7 @@ def test_standard_recipe_with_loader_limit(self): ) # 5 elements were moved to demo pool self.assertEqual(len(list(stream["test"])), 10) - def test_standard_recipe_with_loader_limit_errors(self): + def test_dataset_recipe_with_loader_limit_errors(self): with self.assertRaises(ValueError): StandardRecipeWithIndexes( card="cards.wnli", @@ -615,7 +615,7 @@ def test_standard_recipe_with_loader_limit_errors(self): loader_limit=9, ) - def test_standard_recipe_with_no_demos_to_take(self): + def test_dataset_recipe_with_no_demos_to_take(self): recipe = StandardRecipeWithIndexes( template="templates.key_val", card="cards.xwinogrande.pt", @@ -657,7 +657,7 @@ def test_standard_recipe_with_no_demos_to_take(self): "num_demos (got: 30) should not exceed demos_pool_size - 1 (got: 10), (-1: to always allow filtering of a demo identical to the processed instance).", ) - def test_standard_recipe_with_no_test(self): + def test_dataset_recipe_with_no_test(self): recipe = StandardRecipeWithIndexes( template="templates.key_val", card="cards.xwinogrande.pt", @@ -668,7 +668,7 @@ def test_standard_recipe_with_no_test(self): results = list(recipe()["test"]) self.assertTrue(len(results) > 0) - def test_standard_recipe_with_template_errors(self): + def test_dataset_recipe_with_template_errors(self): # Check either template or template index was specified , but not both with self.assertRaises(AssertionError) as cm: StandardRecipeWithIndexes( @@ -708,7 +708,7 @@ def test_standard_recipe_with_template_errors(self): StandardRecipeWithIndexes(card="cards.wnli", template_card_index=100) self.assertTrue("not defined in card." in str(cm.exception)) - def test_standard_recipe_with_balancer_and_size_limit(self): + def test_dataset_recipe_with_balancer_and_size_limit(self): recipe = StandardRecipeWithIndexes( card="cards.wnli", system_prompt="system_prompts.models.llama", @@ -728,7 +728,7 @@ def test_standard_recipe_with_balancer_and_size_limit(self): self.assertEqual(counts["entailment"], counts["not entailment"], 10) - def test_standard_recipe_with_augmentor_on_task_input(self): + def test_dataset_recipe_with_augmentor_on_task_input(self): recipe = StandardRecipeWithIndexes( card="cards.sst2", augmentor="augmentors.text.white_space", @@ -753,7 +753,7 @@ def test_standard_recipe_with_augmentor_on_task_input(self): normalized_output_source == normalized_input_source ), f"{normalized_output_source} is not equal to f{normalized_input_source}" - def test_standard_recipe_with_train_size_limit(self): + def test_dataset_recipe_with_train_size_limit(self): recipe = StandardRecipeWithIndexes( card="cards.wnli", system_prompt="system_prompts.models.llama", @@ -775,7 +775,7 @@ def test_recipe_with_hf_with_twice_the_same_instance_demos(self): d = load_dataset( dataset_file, - "__type__=standard_recipe_with_indexes,card=cards.wnli,template=templates.classification.multi_class.relation.default,system_prompt=system_prompts.models.llama,demos_pool_size=5,num_demos=1", + "__type__=dataset_recipe_with_indexes,card=cards.wnli,template=templates.classification.multi_class.relation.default,system_prompt=system_prompts.models.llama,demos_pool_size=5,num_demos=1", streaming=True, trust_remote_code=True, ) @@ -795,7 +795,7 @@ def test_recipe_loaded_from_arguments_and_overwrites_only(self): first_inst = next(iterator) self.assertListEqual(["metrics.accuracy"], first_inst["metrics"]) - def test_standard_recipe_with_a_missing_sampler(self): + def test_dataset_recipe_with_a_missing_sampler(self): """Check that initializing a recipe with a card that does not have a sampler raises an exception.""" task_card, _ = copy.deepcopy(fetch_artifact("cards.sst2")) task_card.sampler = None @@ -849,7 +849,7 @@ def serialize(self, value: Table, instance: Dict[str, Any]) -> str: task=task, ) - recipe = StandardRecipe( + recipe = DatasetRecipe( card=card, template=template, serializer=TableSerializer(), @@ -858,7 +858,7 @@ def serialize(self, value: Table, instance: Dict[str, Any]) -> str: target = "Solve: col1,col2\nval1,val2\nval3\nval4\nAnswer: \n" self.assertEqual(result, target) - recipe = StandardRecipe( + recipe = DatasetRecipe( card=card, template=template, serializer=MyTableSerializer(), diff --git a/utils/compare_unitxt_datasets_between_versions.py b/utils/compare_unitxt_datasets_between_versions.py index db1cd22d43..b1df39c794 100644 --- a/utils/compare_unitxt_datasets_between_versions.py +++ b/utils/compare_unitxt_datasets_between_versions.py @@ -21,7 +21,7 @@ from unitxt import register_local_catalog from unitxt.dataset_utils import fetch -from unitxt.standard import StandardRecipe +from unitxt.standard import DatasetRecipe DEMOS_POOL_SIZE = 100 TEST_SIZE = 100 @@ -134,7 +134,7 @@ def generate_examples_for_configuration( "demos_pool_size": DEMOS_POOL_SIZE, "loader_limit": 2 * TEST_SIZE + DEMOS_POOL_SIZE, } - recipe = StandardRecipe(**inputs) + recipe = DatasetRecipe(**inputs) stream = recipe() return list(itertools.islice(stream["test"], TEST_SIZE))